diff options
| author | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
|---|---|---|
| committer | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
| commit | 4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch) | |
| tree | 1682a647d4463397c119183369ae6f750d5fdcff /src/third_party/wiredtiger/test | |
| parent | aa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff) | |
| parent | 8f0827553e09872941945a093b647a4211a9db7f (diff) | |
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'src/third_party/wiredtiger/test')
82 files changed, 736 insertions, 6182 deletions
diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/README b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/README deleted file mode 100644 index e9d53be96d1..00000000000 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/README +++ /dev/null @@ -1,4 +0,0 @@ -We have modified concurrencytest.py from the original concurrencytest-0.1.2 to wait for the child processes -preventing defunct processes and to prefix output with the running PID for debuggability.o - -Future changes such as updating concurrencytest must take this into account. diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/PKG-INFO b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/PKG-INFO index edb06bf7a42..edb06bf7a42 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/PKG-INFO +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/PKG-INFO diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/PKG-INFO b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/PKG-INFO index edb06bf7a42..edb06bf7a42 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/PKG-INFO +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/PKG-INFO diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/SOURCES.txt b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/SOURCES.txt index bf9f692ad1f..bf9f692ad1f 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/SOURCES.txt +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/SOURCES.txt diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/dependency_links.txt b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/dependency_links.txt index 8b137891791..8b137891791 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/dependency_links.txt +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/dependency_links.txt diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/requires.txt b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/requires.txt index 537ebcbac33..537ebcbac33 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/requires.txt +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/requires.txt diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/top_level.txt b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/top_level.txt index cfc96e6db71..cfc96e6db71 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.egg-info/top_level.txt +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.egg-info/top_level.txt diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.py b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.py index f9049ce8518..b3cb52d48d3 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/concurrencytest.py +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/concurrencytest.py @@ -23,9 +23,7 @@ Unix only. import os import sys -import time import traceback -from threading import Thread import unittest from itertools import cycle from multiprocessing import cpu_count @@ -42,29 +40,9 @@ _all__ = [ 'partition_tests', ] -# This file has been modified from the original concurrencytest-0.1.2 to wait for the child processes -# preventing defunct processes and to prefix output with the running PID for debuggability. CPU_COUNT = cpu_count() -def wait_for_children(pids): - while pids: - try: - # As Windows doesn't support -1 for all children, loop through each child pid explicitly. - for child_pid in pids: - pid, exit_status = os.waitpid(child_pid, os.WNOHANG) - exit_code = os.waitstatus_to_exitcode(exit_status) - if exit_code != 0: - pids.remove(pid) - if exit_code > 0: - print("[pid:{}]: Unexpected exit ({}) for child process ({})".format(os.getpid(), exit_code, pid)) - else: - print("[pid:{}]: Unexpected exit by signal ({}) for child process ({})".format(os.getpid(), abs(exit_code), pid)) - except ChildProcessError: - # No children processes. - break - # Sleep as the waipid is non blocking. - time.sleep(5) def fork_for_tests(concurrency_num=CPU_COUNT): """Implementation of `make_tests` used to construct `ConcurrentTestSuite`. @@ -83,7 +61,6 @@ def fork_for_tests(concurrency_num=CPU_COUNT): test_blocks = partition_tests(suite, concurrency_num) # Clear the tests from the original suite so it doesn't keep them alive suite._tests[:] = [] - pids = [] for process_tests in test_blocks: process_suite = unittest.TestSuite(process_tests) # Also clear each split list so new suite has only reference @@ -102,8 +79,6 @@ def fork_for_tests(concurrency_num=CPU_COUNT): subunit_result = AutoTimingTestResultDecorator( TestProtocolClient(stream) ) - # Set the pid tag for the parent to log with this information. - subunit_result.tags(["pid:" + str(os.getpid())], []) process_suite.run(subunit_result) except: # Try and report traceback on stream, but exit with error @@ -112,7 +87,7 @@ def fork_for_tests(concurrency_num=CPU_COUNT): # written in one go to avoid interleaving lines from # multiple failing children. try: - print("[pid:{}]: {}".format(os.getpid(), traceback.format_exc())) + stream.write(traceback.format_exc()) finally: os._exit(1) os._exit(0) @@ -121,10 +96,6 @@ def fork_for_tests(concurrency_num=CPU_COUNT): stream = os.fdopen(c2pread, 'rb', 1) test = ProtocolTestCase(stream) result.append(test) - pids.append(pid) - # Monitor our children to prevent leaving <defunct> processes around. - wait_thread = Thread(target = wait_for_children, args = (pids, )) - wait_thread.start() return result return do_fork @@ -142,6 +113,7 @@ def partition_tests(suite, count): partition.append(test) return partitions + if __name__ == '__main__': import time diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/setup.cfg b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/setup.cfg index 861a9f55426..861a9f55426 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/setup.cfg +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/setup.cfg diff --git a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/setup.py b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/setup.py index 447dc4110fe..447dc4110fe 100644 --- a/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2-locally-modified/setup.py +++ b/src/third_party/wiredtiger/test/3rdparty/concurrencytest-0.1.2/setup.py diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index ccfa2764787..cc6964d7db3 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -68,7 +68,6 @@ main(int argc, char *argv[]) g.ntables = 3; g.nworkers = 1; g.sweep_stress = g.use_timestamps = false; - g.failpoint_eviction_fail_after_reconciliation = false; g.failpoint_hs_delete_key_from_ts = false; g.hs_checkpoint_timing_stress = g.reserved_txnid_timing_stress = false; g.checkpoint_slow_timing_stress = false; @@ -128,9 +127,6 @@ main(int argc, char *argv[]) case '5': g.checkpoint_slow_timing_stress = true; break; - case '7': - g.failpoint_eviction_fail_after_reconciliation = true; - break; default: return (usage()); } @@ -277,10 +273,7 @@ wt_connect(const char *config_open) g.reserved_txnid_timing_stress || g.checkpoint_slow_timing_stress) { timing_stress = true; testutil_check(__wt_snprintf(timing_stress_config, sizeof(timing_stress_config), - ",timing_stress_for_test=[%s%s%s%s%s%s]", g.sweep_stress ? "aggressive_sweep" : "", - g.failpoint_eviction_fail_after_reconciliation ? - "failpoint_eviction_fail_after_reconciliation" : - "", + ",timing_stress_for_test=[%s%s%s%s%s]", g.sweep_stress ? "aggressive_sweep" : "", g.failpoint_hs_delete_key_from_ts ? "failpoint_history_store_delete_key_from_ts" : "", g.hs_checkpoint_timing_stress ? "history_store_checkpoint_delay" : "", g.reserved_txnid_timing_stress ? "checkpoint_reserved_txnid_delay" : "", @@ -299,10 +292,8 @@ wt_connect(const char *config_open) */ if (g.sweep_stress) testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),error_" - "prefix=" - "\"%s\",file_manager=(close_handle_minimum=1,close_idle_time=1,close_scan_interval=1)" - "," + "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),error_prefix=" + "\"%s\",file_manager=(close_handle_minimum=1,close_idle_time=1,close_scan_interval=1)," "log=(enabled),cache_size=1GB, eviction_dirty_trigger=%i, " "eviction_dirty_target=%i,%s%s%s%s", progname, fast_eviction ? 5 : 20, fast_eviction ? 1 : 5, timing_stress_config, @@ -310,8 +301,7 @@ wt_connect(const char *config_open) config_open == NULL ? "" : config_open)); else testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),log=(" - "enabled)," + "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),log=(enabled)," "error_prefix=\"%s\",cache_size=1G, eviction_dirty_trigger=%i, " "eviction_dirty_target=%i,%s%s%s%s", progname, fast_eviction ? 5 : 20, fast_eviction ? 1 : 5, @@ -599,8 +589,6 @@ usage(void) "\t\t3: hs_checkpoint_timing_stress\n" "\t\t4: reserved_txnid_timing_stress\n" "\t\t5: checkpoint_slow_timing_stress\n" - "\t\t6: evict_reposition_timing_stress\n" - "\t\t7: failpoint_eviction_fail_after_reconciliation\n" "\t-T specify a table configuration\n" "\t-t set a file type ( col | mix | row | lsm )\n" "\t-v verify only\n" diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index ece2be67b1e..1d8bd72ab0c 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -55,35 +55,33 @@ typedef struct { } COOKIE; typedef struct { - char *home; /* Home directory */ - const char *checkpoint_name; /* Checkpoint name */ - WT_CONNECTION *conn; /* WiredTiger connection */ - bool debug_mode; /* History store stress test */ - u_int nkeys; /* Keys to load */ - u_int nops; /* Operations per thread */ - FILE *logfp; /* Message log file. */ - int nworkers; /* Number workers configured */ - int ntables; /* Number tables configured */ - int ntables_created; /* Number tables opened */ - volatile int running; /* Whether to stop */ - int status; /* Exit status */ - bool sweep_stress; /* Sweep stress test */ - bool failpoint_hs_delete_key_from_ts; /* Failpoint for hs key deletion. */ - bool hs_checkpoint_timing_stress; /* History store checkpoint timing stress */ - bool reserved_txnid_timing_stress; /* Reserved transaction id timing stress */ - bool checkpoint_slow_timing_stress; /* Checkpoint slow timing stress */ - uint64_t ts_oldest; /* Current oldest timestamp */ - uint64_t ts_stable; /* Current stable timestamp */ - bool failpoint_eviction_fail_after_reconciliation; /*Fail point for eviction fail after - reconciliation. */ - bool mixed_mode_deletes; /* Run with mixed mode deletes */ - bool use_timestamps; /* Use txn timestamps */ - bool race_timestamps; /* Async update to oldest timestamp */ - bool prepare; /* Use prepare transactions */ - COOKIE *cookies; /* Per-thread info */ - WT_RWLOCK clock_lock; /* Clock synchronization */ - wt_thread_t checkpoint_thread; /* Checkpoint thread */ - wt_thread_t clock_thread; /* Clock thread */ + char *home; /* Home directory */ + const char *checkpoint_name; /* Checkpoint name */ + WT_CONNECTION *conn; /* WiredTiger connection */ + bool debug_mode; /* History store stress test */ + u_int nkeys; /* Keys to load */ + u_int nops; /* Operations per thread */ + FILE *logfp; /* Message log file. */ + int nworkers; /* Number workers configured */ + int ntables; /* Number tables configured */ + int ntables_created; /* Number tables opened */ + volatile int running; /* Whether to stop */ + int status; /* Exit status */ + bool sweep_stress; /* Sweep stress test */ + bool failpoint_hs_delete_key_from_ts; /* Failpoint for hs key deletion. */ + bool hs_checkpoint_timing_stress; /* History store checkpoint timing stress */ + bool reserved_txnid_timing_stress; /* Reserved transaction id timing stress */ + bool checkpoint_slow_timing_stress; /* Checkpoint slow timing stress */ + uint64_t ts_oldest; /* Current oldest timestamp */ + uint64_t ts_stable; /* Current stable timestamp */ + bool mixed_mode_deletes; /* Run with mixed mode deletes */ + bool use_timestamps; /* Use txn timestamps */ + bool race_timestamps; /* Async update to oldest timestamp */ + bool prepare; /* Use prepare transactions */ + COOKIE *cookies; /* Per-thread info */ + WT_RWLOCK clock_lock; /* Clock synchronization */ + wt_thread_t checkpoint_thread; /* Checkpoint thread */ + wt_thread_t clock_thread; /* Clock thread */ } GLOBAL; extern GLOBAL g; diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt index b859727d4d9..bce5248222a 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt @@ -34,13 +34,14 @@ runtime_monitor= ), stat_cache_size= ( - max=200, - runtime=false, + max=110, + runtime=true, ), # The data files compress to around 25MB per table at the end of a run so 250MB total. # +1.4GB for the history store. With an additional 150MB margin. stat_db_size= ( + max=1900000000, save=true, ), ), diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/operations_test_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/operations_test_stress.txt index e5c25561925..5085f9ebe17 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/operations_test_stress.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/operations_test_stress.txt @@ -11,6 +11,11 @@ runtime_monitor= ( stat_db_size= ( + #At the end of the run the data files are approximately 2.3MB each. Which is a total of: + #1.15GB, the history store isn't significant. Give the workload an extra 200MB of margin. + max=1350000000, + # FIXME-WT-8886 - This check has been disabled to remove noisy failures in evergreen and + # will be properly corrected in WT-8886. runtime=false, save=true, ) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h index 1c4957e814b..0fce6bfeb45 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h @@ -43,7 +43,7 @@ class thread_manager { */ template <typename Callable, typename... Args> void - add_thread(Callable &&fct, Args &&...args) + add_thread(Callable &&fct, Args &&... args) { std::thread *t = new std::thread(fct, std::forward<Args>(args)...); _workers.push_back(t); diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index f4d6cc21afb..3f93ec25a02 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -57,7 +57,7 @@ static char home[1024]; /* Program working dir */ #define INVALID_KEY UINT64_MAX #define MAX_CKPT_INVL 2 /* Maximum interval between checkpoints */ /* Set large, some slow I/O systems take tens of seconds to fsync. */ -#define MAX_STARTUP 60 /* Seconds to start up and set stable */ +#define MAX_STARTUP 30 /* Seconds to start up and set stable */ #define MAX_TH 12 #define MAX_TIME 40 #define MAX_VAL 1024 @@ -635,10 +635,6 @@ thread_run(void *arg) printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->info, td->start); stable_ts = 0; for (i = td->start;; ++i) { - /* Give other threads a chance to run and move their timestamps forward. */ - if (use_ts && !stable_set && (i + 1) % 100 == 0) - __wt_sleep(2, 0); - /* * Allow some threads to skip schema operations so that they are generating sufficient dirty * data. diff --git a/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c b/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c index fe48721b789..1095077aab9 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c @@ -56,8 +56,7 @@ main(int argc, char *argv[]) size_t len; uint32_t hw, sw; uint8_t *data; - uint8_t data_ff[32]; - u_int i, j, length, misalignment; + u_int i, j; opts = &_opts; memset(opts, 0, sizeof(*opts)); @@ -70,7 +69,6 @@ main(int argc, char *argv[]) /* Allocate aligned memory for the data. */ data = dcalloc(DATASIZE, sizeof(uint8_t)); - memset(data_ff, 0xff, sizeof(data_ff)); /* * Some simple known checksums. @@ -132,7 +130,7 @@ main(int argc, char *argv[]) check(hw, sw, len, "random power-of-two"); len *= 2; - if (len > DATASIZE || len == 0) + if (len > DATASIZE) len = 512; } @@ -140,9 +138,7 @@ main(int argc, char *argv[]) * Checksums of random data chunks. */ for (i = 0; i < 1000; ++i) { - do { - len = __wt_random(&rnd) % DATASIZE; - } while (len == 0); + len = __wt_random(&rnd) % DATASIZE; for (j = 0; j < len; ++j) data[j] = __wt_random(&rnd) & 0xff; hw = __wt_checksum(data, len); @@ -150,17 +146,6 @@ main(int argc, char *argv[]) check(hw, sw, len, "random"); } - /* - * "Strobed" misalignments - test every combo of size/misalignment up to 16B. - */ - for (length = 0; length < 16; length++) { - for (misalignment = 0; misalignment < 16; misalignment++) { - hw = __wt_checksum(&data_ff[misalignment], length); - sw = __wt_checksum_sw(&data_ff[misalignment], length); - check(hw, sw, length, "0xff: strobed"); - } - } - free(data); testutil_cleanup(opts); return (EXIT_SUCCESS); diff --git a/src/third_party/wiredtiger/test/csuite/wt8246_compact_rts_data_correctness/main.c b/src/third_party/wiredtiger/test/csuite/wt8246_compact_rts_data_correctness/main.c index 2fc64a0547c..b055edf8fbb 100644 --- a/src/third_party/wiredtiger/test/csuite/wt8246_compact_rts_data_correctness/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt8246_compact_rts_data_correctness/main.c @@ -30,7 +30,6 @@ #include <signal.h> #define TIMEOUT 1 -#define MAX_RETRIES 5 #define NUM_RECORDS 800000 @@ -335,13 +334,11 @@ static void large_updates(WT_SESSION *session, const char *uri, char *value, int commit_ts) { WT_CURSOR *cursor; - WT_DECL_RET; WT_RAND_STATE rnd; uint64_t val; - int i, retry_attempts; + int i; char tscfg[64]; - retry_attempts = 0; __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); @@ -351,17 +348,7 @@ large_updates(WT_SESSION *session, const char *uri, char *value, int commit_ts) cursor->set_key(cursor, i + 1); val = (uint64_t)__wt_random(&rnd); cursor->set_value(cursor, val, val, val, value); - while (((ret = cursor->insert(cursor)) == WT_ROLLBACK) && retry_attempts < MAX_RETRIES) { - printf("Rollback transaction for key %d\n", i + 1); - testutil_check(session->rollback_transaction(session, NULL)); - testutil_check(session->begin_transaction(session, NULL)); - ++retry_attempts; - } - - if (retry_attempts == MAX_RETRIES) - testutil_die(ret, "Cursor insert returned WT_ROLLBACK for %d times", MAX_RETRIES); - - testutil_check(ret); + testutil_check(cursor->insert(cursor)); testutil_check(session->commit_transaction(session, tscfg)); } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index fcfcfa2430a..1eb0ac2ae5e 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -11,8 +11,7 @@ stepback: true pre: - func: "cleanup" post: - - func: "dump stacktraces" - - func: "upload stacktraces" + - func: "print python stacktrace" - func: "dump stderr/stdout" - func: "upload artifact" vars: @@ -108,40 +107,11 @@ functions: set -o errexit set -o verbose - MACOS_PYTHON_FLAGS= - if [ "${build_variant|}" = "macos-1014" ]; then - # For mac builds, we want explicitly tell cmake which python to use, as - # well as the matching library directory and header files. The find_libpython - # module gives us the library. - SYSPY=${python_binary} - $SYSPY -mvenv venv - source venv/bin/activate - pip3 install find_libpython - SYSPYLIB=`find_libpython` - SYSPYINCDEF= - - # We have the shared library to link to, it may be named simply 'Python3' or 'Python'. - # If that's the case, use the associated dylib symlink found in an expected relative - # location. Also get the location of the header files. We'll give this all to cmake. - base=$(basename $SYSPYLIB) - if [ "$base" = 'Python3' -o "$base" = 'Python' ]; then - SYSPYDIR=$(dirname $SYSPYLIB) - NSYSPYLIB=$(ls $SYSPYDIR/lib/libpython*.dylib 2>/dev/null | head -1) - if [ -f "$NSYSPYLIB" ]; then - SYSPYLIB=$NSYSPYLIB - fi - if [ -d "$SYSPYDIR/Headers" ]; then - SYSPYINCDEF="-DPYTHON_INCLUDE_DIR=$SYSPYDIR/Headers" - fi - fi - MACOS_PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=$SYSPY -DPYTHON_LIBRARY=$SYSPYLIB $SYSPYINCDEF" - fi - if [ "$OS" = "Windows_NT" ]; then # Use the Windows powershell script to configure the CMake build. # We execute it in a powershell environment as its easier to detect and source the Visual Studio # toolchain in a native Windows environment. We can't easily execute the build in a cygwin environment. - ${configure_env_vars|} powershell.exe -NonInteractive '.\test\evergreen\build_windows.ps1' -configure 1 ${windows_configure_flags|} + powershell.exe -NonInteractive '.\test\evergreen\build_windows.ps1' -configure 1 ${windows_configure_flags|} else # Compiling with CMake. . test/evergreen/find_cmake.sh @@ -151,34 +121,8 @@ functions: if [ -d cmake_build ]; then rm -r cmake_build; fi mkdir -p cmake_build cd cmake_build - ${configure_env_vars|} $CMAKE \ - ${posix_configure_flags|-DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_STATIC=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL} $MACOS_PYTHON_FLAGS -G "${cmake_generator|Ninja}" ./.. - fi - "python config check": - command: shell.exec - params: - working_dir: "wiredtiger/cmake_build" - shell: bash - script: | - set -o errexit - set -o verbose - # Note: ${var} is used for expansions of evergreen variables, whereas $var are /bin/sh variables. - python_configured=$(grep PYTHON_EXECUTABLE: ./CMakeCache.txt | sed -e 's/.*=//') - if [ "${python_binary}" = '' -o "$python_configured" = '' -o "$(${python_binary} -V -V)" != "$($python_configured -V -V)" ]; then - echo "Python versions do not match." - echo "Python version used by cmake builds ($python_configured):" - if [ "$python_configured" = '' ]; then - echo "No python configured" - else - $python_configured -V -V - fi - echo "Python version used in shell commands (${python_binary}):" - if [ "${python_binary}" = '' ]; then - echo "No python set in Expansions" - else - ${python_binary} -V -V - fi - exit 1 + $CMAKE \ + ${posix_configure_flags|-DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_STATIC=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL} -G "${cmake_generator|Ninja}" ./.. fi "make wiredtiger": &make_wiredtiger command: shell.exec @@ -194,7 +138,7 @@ functions: else # Compiling with CMake generated Ninja file. cd cmake_build - ${compile_env_vars|} ${make_command|ninja} ${smp_command|} 2>&1 + ${make_command|ninja} ${smp_command|} 2>&1 fi "compile wiredtiger": - *configure_wiredtiger @@ -300,25 +244,6 @@ functions: # Save generated documentation mv docs docs-$branch done - "dump stacktraces": &dump_stacktraces - command: shell.exec - params: - working_dir: "wiredtiger/cmake_build" - script: | - set -o errexit - set -o verbose - ${python_binary|python3} ../test/evergreen/print_stack_trace.py - "upload stacktraces": &upload_stacktraces - command: s3.put - params: - aws_secret: ${aws_secret} - aws_key: ${aws_key} - local_files_include_filter: - - wiredtiger/cmake_build/*stacktrace.txt - bucket: build_external - permissions: public-read - content_type: text/plain - remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${task_name}_${build_id}/ "update wiredtiger docs": - command: shell.exec @@ -443,15 +368,8 @@ functions: script: | set -o errexit set -o verbose - # Fail, show the configuration file. - fail() { - echo "======= FAILURE ==========" - [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG - exit 1 - } - for i in $(seq ${times|1}); do - ./t -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} || fail + ./t -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} || ( [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG ) 2>&1 done "format test script": command: shell.exec @@ -564,7 +482,7 @@ functions: wt_hang_analyzer_option="-c -o file -o stdout" echo "Calling the wt hang analyzer ..." - PATH="/opt/mongodbtoolchain/v4/bin:$PATH" ${python_binary|python3} ../test/wt_hang_analyzer/wt_hang_analyzer.py $wt_hang_analyzer_option + PATH="/opt/mongodbtoolchain/gdb/bin:$PATH" ${python_binary|python3} ../test/wt_hang_analyzer/wt_hang_analyzer.py $wt_hang_analyzer_option "save wt hang analyzer core/debugger files": - command: archive.targz_pack @@ -586,6 +504,17 @@ functions: display_name: WT Hang Analyzer Output - Execution ${execution} remote_file: wiredtiger/${build_variant}/${revision}/wt_hang_analyzer/wt-hang-analyzer_${task_name}_${build_id}${postfix|}.tgz + "print python stacktrace": + command: shell.exec + params: + working_dir: "wiredtiger/cmake_build" + script: | + set -o errexit + set -o verbose + if [ -d "WT_TEST" ]; then + ${python_binary|python3} ../test/evergreen/print_python_stack_trace.py -e ${python_binary|python3} -c WT_TEST -l . + fi + "dump stderr/stdout": command: shell.exec params: @@ -753,22 +682,6 @@ variables: vars: format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 360 - - &format-stress-sanitizer-ppc-test - exec_timeout_secs: 25200 - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v4_clang.cmake -DCMAKE_BUILD_TYPE=ASan -DHAVE_BUILTIN_EXTENSION_LZ4=1 -DHAVE_BUILTIN_EXTENSION_SNAPPY=1 -DHAVE_BUILTIN_EXTENSION_ZLIB=1 - - func: "format test script" - vars: - # Always disable mmap for PPC due to issues on variant setup. - # See https://bugzilla.redhat.com/show_bug.cgi?id=1686261#c10 for the potential cause. - format_test_script_args: -t 360 -- -C "mmap=false,mmap_all=false" - test_env_vars: - ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1" - ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer - - &format-stress-sanitizer-test exec_timeout_secs: 25200 commands: @@ -811,7 +724,7 @@ variables: ####################################### tasks: - # Check the python configuration + # Base compile task on posix flavours - name: compile tags: ["pull_request"] @@ -884,12 +797,12 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -DENABLE_STRICT=1 -DHAVE_DIAGNOSTIC=0 - func: "compile wiredtiger" vars: + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -DCLANG_C_VERSION=6.0 -DCLANG_CXX_VERSION=6.0 -DCMAKE_C_FLAGS="-ggdb" -DWITH_PIC=1 + - func: "compile wiredtiger" + vars: posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -DCLANG_C_VERSION=7 -DCLANG_CXX_VERSION=7 -DCMAKE_C_FLAGS="-ggdb" -DWITH_PIC=1 - func: "compile wiredtiger" vars: @@ -1658,17 +1571,6 @@ tasks: - func: "fetch artifacts" - func: "unit test" - - name: unit-test-macos - tags: ["python"] - depends_on: - - name: compile - commands: - - func: "fetch artifacts" - - func: "python config check" - - func: "unit test" - vars: - smp_command: -j $(echo $(sysctl -n hw.logicalcpu) / 2 | bc) - - name: unit-test-zstd tags: ["python"] depends_on: @@ -2034,8 +1936,7 @@ tasks: script: | set -o errexit set -o verbose - # FIXME-WT-12144 Turn off noisy test failures until s_clang_scan is fixed. - # sh dist/s_clang_scan 2>&1 + sh dist/s_clang-scan 2>&1 - name: configure-combinations commands: @@ -2280,11 +2181,10 @@ tasks: vars: posix_configure_flags: -DENABLE_PYTHON=1 -DSPINLOCK_TYPE=gcc -DENABLE_STRICT=1 - func: "make check all" - - func: "unit test" - func: "format test" vars: times: 3 - extra_args: runs.rows=1000000:2500000 + - func: "unit test" - name: spinlock-pthread-adaptive-test commands: @@ -2293,11 +2193,10 @@ tasks: vars: posix_configure_flags: -DENABLE_PYTHON=1 -DSPINLOCK_TYPE=pthread_adaptive -DENABLE_STRICT=1 - func: "make check all" - - func: "unit test" - func: "format test" vars: times: 3 - extra_args: runs.rows=1000000:2500000 + - func: "unit test" - name: wtperf-test depends_on: @@ -2639,20 +2538,6 @@ tasks: set -o errexit set -o verbose ${test_env_vars|} ../../../tools/run_parallel.sh 'nice ./recovery-test.sh "${data_validation_stress_test_args} -s 5" WT_TEST.$t test_checkpoint' 120 - - - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 - depends_on: - - name: compile - commands: - - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/cmake_build/test/checkpoint" - shell: bash - script: | - set -o errexit - set -o verbose - ${test_env_vars|} ../../../tools/run_parallel.sh 'nice ./recovery-test.sh "${data_validation_stress_test_args} -s 7" WT_TEST.$t test_checkpoint' 120 - name: format-failure-configs-test depends_on: @@ -2700,7 +2585,7 @@ tasks: - name: format-stress-sanitizer-lsm-test # FIXME-WT-6258: Re-enable the test once the outstanding issues with LSM are resolved. - # tags: ["stress-test"] + # tags: ["stress-test-1"] commands: - func: "get project" - func: "compile wiredtiger address sanitizer" @@ -2745,8 +2630,9 @@ tasks: set -o verbose for i in {1..10}; do ${test_env_vars|} ${python_binary|python3} split_stress.py; done - - name: format-stress-zseries-test - tags: ["stress-test-zseries-1"] + # The task name is ppc-zseries because this task will be used in both buildVariants + - name: format-stress-ppc-zseries-test + tags: ["stress-test-ppc-1", "stress-test-zseries-1"] # Set 2.5 hours timeout (60 * 60 * 2.5) exec_timeout_secs: 9000 commands: @@ -2757,22 +2643,6 @@ tasks: #run for 2 hours ( 2 * 60 = 120 minutes), use default config format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 120 - - name: format-stress-ppc-test - tags: ["stress-test-ppc-1"] - # Set 2.5 hours timeout (60 * 60 * 2.5) - exec_timeout_secs: 9000 - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DHAVE_BUILTIN_EXTENSION_LZ4=1 -DHAVE_BUILTIN_EXTENSION_SNAPPY=1 -DHAVE_BUILTIN_EXTENSION_ZLIB=1 - - func: "format test script" - vars: - #run for 2 hours ( 2 * 60 = 120 minutes), use default config - # Always disable mmap for PPC due to issues on variant setup. - # See https://bugzilla.redhat.com/show_bug.cgi?id=1686261#c10 for the potential cause. - format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 120 -- -C "mmap=false,mmap_all=false" - - <<: *format-stress-test name: format-stress-test-1 tags: ["stress-test-1"] @@ -2785,36 +2655,30 @@ tasks: - <<: *format-stress-test name: format-stress-test-4 tags: ["stress-test-4"] - - <<: *format-stress-sanitizer-ppc-test - name: format-stress-sanitizer-ppc-test-1 - tags: ["stress-test-ppc-1"] - - <<: *format-stress-sanitizer-ppc-test - name: format-stress-sanitizer-ppc-test-2 - tags: ["stress-test-ppc-2"] - <<: *format-stress-sanitizer-test name: format-stress-sanitizer-test-1 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-1", "stress-test-ppc-1"] - <<: *format-stress-sanitizer-test name: format-stress-sanitizer-test-2 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-2", "stress-test-ppc-2"] - <<: *format-stress-sanitizer-test name: format-stress-sanitizer-test-3 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-3"] - <<: *format-stress-sanitizer-test name: format-stress-sanitizer-test-4 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-4"] - <<: *race-condition-stress-sanitizer-test name: race-condition-stress-sanitizer-test-1 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-1"] - <<: *race-condition-stress-sanitizer-test name: race-condition-stress-sanitizer-test-2 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-2"] - <<: *race-condition-stress-sanitizer-test name: race-condition-stress-sanitizer-test-3 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-3"] - <<: *race-condition-stress-sanitizer-test name: race-condition-stress-sanitizer-test-4 - tags: ["stress-test-sanitizer"] + tags: ["stress-test-4"] - <<: *recovery-stress-test name: recovery-stress-test-1 tags: ["stress-test-1", "stress-test-zseries-1"] @@ -2838,7 +2702,7 @@ tasks: format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 360 - name: format-stress-sanitizer-test-no-barrier - tags: ["stress-test-sanitizer"] + tags: ["stress-test-no-barrier"] exec_timeout_secs: 25200 commands: - func: "get project" @@ -2853,7 +2717,7 @@ tasks: ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer - name: race-condition-stress-sanitizer-test-no-barrier - tags: ["stress-test-sanitizer"] + tags: ["stress-test-no-barrier"] exec_timeout_secs: 25200 commands: - func: "get project" @@ -2909,22 +2773,29 @@ tasks: params: working_dir: mongo-tests/largescale shell: bash - include_expansions_in_env: - - atlas_wt_perf_test_user - - atlas_wt_perf_pass script: | set -o errexit set -o verbose export "PATH=/opt/mongodbtoolchain/v3/bin:$PATH" virtualenv -p python3 venv source venv/bin/activate - # Need both pymongo and pymongo[srv] as upload-results-atlas.py uses mongo+srv for the URI. - pip3 install lorem pymongo==3.12.2 "pymongo[srv]==3.12.2" + pip3 install lorem pymongo==3.12.2 mongod_path=$(find ../../mongo/build -executable -type f -path \*/bin/mongod) ./run_many_coll.sh $mongod_path mongodb.log config/many-collection-testing many-collection clean-and-populate - # upload the results + + - command: shell.exec + params: + working_dir: mongo-tests/largescale + shell: bash + silent: true + script: | + set -o errexit + set -o verbose + virtualenv -p python3 venv + source venv/bin/activate + pip3 install "pymongo[srv]==3.12.2" res_dir=`find ./ -type d -name "many-collection-artifacts" -print` - ./upload-results-atlas.py wt-perf-tests many-collection-test ${branch_name} $res_dir/results/results.json + ./upload-results-atlas.py ${atlas_wt_perf_test_user} ${atlas_wt_perf_pass} wt-perf-tests many-collection-test ${branch_name} $res_dir/results/results.json - name: cyclomatic-complexity commands: @@ -3124,21 +2995,6 @@ tasks: - func: "upload-perf-test-stats" vars: perf-test-name: parallel-pop-btree.wtperf - - - name: perf-test-parallel-pop-btree-long - tags: ["btree-perf"] - depends_on: - - name: compile - commands: - - func: "fetch artifacts" - - func: "run-perf-test" - vars: - perf-test-name: parallel-pop-btree-long.wtperf - maxruns: 1 - wtarg: -ops ['"load"'] - - func: "upload-perf-test-stats" - vars: - perf-test-name: parallel-pop-btree-long.wtperf - name: perf-test-update-only-btree tags: ["btree-perf"] @@ -3819,10 +3675,7 @@ buildvariants: run_on: - ubuntu2004-test expansions: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib @@ -3833,7 +3686,6 @@ buildvariants: -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 - -DENABLE_ZSTD=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DENABLE_TCMALLOC=1 @@ -3875,7 +3727,6 @@ buildvariants: - name: data-validation-stress-test-checkpoint-fp-hs-insert-s4 - name: data-validation-stress-test-checkpoint-fp-hs-insert-s5 - name: data-validation-stress-test-checkpoint-fp-hs-insert-s5-no-timestamp - - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 - name: data-validation-stress-test-checkpoint-no-timestamp - name: unittest-test - name: tiered-storage-extensions-test @@ -3898,10 +3749,7 @@ buildvariants: -DCMAKE_CXX_FLAGS="-ggdb" smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: ninja - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1" @@ -3909,6 +3757,7 @@ buildvariants: ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer TESTUTIL_BYPASS_ASAN=1 LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib tasks: - name: ".pull_request !.pull_request_compilers !.python" @@ -3931,13 +3780,11 @@ buildvariants: -DCMAKE_C_FLAGS="-ggdb" smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: ninja - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH MSAN_OPTIONS="abort_on_error=1:disable_coredump=0:print_stacktrace=1" MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR @@ -3972,12 +3819,10 @@ buildvariants: -DCMAKE_CXX_FLAGS="-ggdb" smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: ninja - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH UBSAN_OPTIONS="detect_leaks=1:disable_coredump=0:external_symbolizer_path=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer:abort_on_error=1:print_stacktrace=1" LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib @@ -4020,15 +3865,13 @@ buildvariants: - name: ubuntu2004-stress-tests display_name: Ubuntu 20.04 Stress tests run_on: - - ubuntu2004-small + - ubuntu2004-test expansions: smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: ninja - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib @@ -4049,9 +3892,6 @@ buildvariants: - name: ".stress-test-3" - name: ".stress-test-4" - name: ".stress-test-no-barrier" - - name: ".stress-test-sanitizer" - run_on: - - ubuntu2004-medium - name: format-abort-recovery-stress-test - name: ubuntu2004-stress-tests-arm64 @@ -4065,11 +3905,9 @@ buildvariants: expansions: smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` / 2" | bc) make_command: ninja - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib @@ -4095,16 +3933,107 @@ buildvariants: - name: ".stress-test-no-barrier" - name: format-abort-recovery-stress-test +- name: ubuntu2004-perf-tests + display_name: Ubuntu 20.04 Performance tests + # Perf tests are NOT expected to be triggered on Evergreen projects + # other than "WiredTiger (develop)". + activate: false + batchtime: 1440 # 1 day + run_on: + - ubuntu2004-medium + expansions: + test_env_vars: + WT_TOPDIR=$(git rev-parse --show-toplevel) + WT_BUILDDIR=$WT_TOPDIR/cmake_build + LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_BUILDDIR/test/utility/ + posix_configure_flags: + -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake + -DCMAKE_C_FLAGS="-ggdb" + -DHAVE_DIAGNOSTIC=1 + -DENABLE_PYTHON=1 + -DENABLE_ZLIB=1 + -DENABLE_SNAPPY=1 + -DENABLE_STRICT=1 + -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' + pip3_binary: '/opt/mongodbtoolchain/v3/bin/pip3' + virtualenv_binary: '/opt/mongodbtoolchain/v3/bin/virtualenv' + smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` * 2" | bc) + cmake_generator: Ninja + make_command: ninja + tasks: + - name: compile + - name: ".btree-perf" + # Disable LSM perf tests till the support for LSM is restored. + # - name: ".lsm-perf" + - name: ".stress-perf" + - name: ".checkpoint-perf" + - name: ".evict-perf" + - name: ".log-perf" + - name: ".long-perf" + - name: many-dhandle-stress + - name: ".wt2853-perf" + display_tasks: + - name: Wiredtiger-perf-btree-jobs + execution_tasks: + - ".btree-perf" + # Disable LSM perf tests till the support for LSM is restored. + # - name: Wiredtiger-perf-lsm-jobs + # execution_tasks: + # - ".lsm-perf" + - name: Wiredtiger-perf-stress-jobs + execution_tasks: + - ".stress-perf" + - name: Wiredtiger-perf-checkpoint-jobs + execution_tasks: + - ".checkpoint-perf" + - name: Wiredtiger-perf-evict-jobs + execution_tasks: + - ".evict-perf" + - name: Wiredtiger-perf-log-jobs + execution_tasks: + - ".log-perf" + - name: Wiredtiger-perf-long-jobs + execution_tasks: + - ".long-perf" + - name: WiredTiger-wt2853-perf + execution_tasks: + - ".wt2853-perf" + +- name: large-scale-tests + display_name: "Large scale tests" + batchtime: 480 # 3 times a day + run_on: + - ubuntu2004-test + expansions: + make_command: ninja + cmake_generator: Ninja + posix_configure_flags: + -DENABLE_PYTHON=1 + -DENABLE_ZLIB=1 + -DENABLE_SNAPPY=1 + -DENABLE_STRICT=1 + -DENABLE_STATIC=1 + -DENABLE_TCMALLOC=1 + -DCMAKE_PREFIX_PATH="$(pwd)/../TCMALLOC_LIB" + test_env_vars: + PATH=/opt/mongodbtoolchain/v3/bin:$PATH + WT_TOPDIR=$(git rev-parse --show-toplevel) + WT_BUILDDIR=$WT_TOPDIR/cmake_build + LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib + upload_source_dir: mongo-tests/largescale/many-collection-artifacts + upload_filename: many-collection.tgz + tasks: + - name: many-collection-test + distros: ubuntu2004-wt-large + - name: cppsuite-stress-tests display_name: "Cppsuite Stress Tests" batchtime: 480 # 3 times a day run_on: - - ubuntu2004-medium + - ubuntu2004-test expansions: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib @@ -4136,17 +4065,28 @@ buildvariants: tasks: - name: package +- name: documentation-update + display_name: "~ Documentation update" + # Documentation update is NOT expected to be triggered on Evergreen projects + # other than "WiredTiger (develop)" + activate: false + batchtime: 10080 # 7 days + run_on: + - ubuntu2004-test + expansions: + doc_update_branches: develop,mongodb-5.0,mongodb-4.4,mongodb-4.2,mongodb-4.0 + tasks: + - name: doc-update + - name: linux-no-ftruncate display_name: Linux no ftruncate batchtime: 1440 # 1 day run_on: - ubuntu2004-test expansions: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib @@ -4165,11 +4105,9 @@ buildvariants: run_on: - rhel80-test expansions: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_PRELOAD=/usr/local/lib/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib @@ -4198,18 +4136,72 @@ buildvariants: - name: long-test - name: configure-combinations +- name: code-statistics + display_name: "Code statistics" + # Code coverage and complexity metrics are NOT expected to be triggered on Evergreen projects + # other than "WiredTiger (develop)". + activate: false + batchtime: 10080 # 7 days + run_on: + - ubuntu2004-test + expansions: + test_env_vars: + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so + PATH=/opt/mongodbtoolchain/v3/bin:$PATH + WT_TOPDIR=$(git rev-parse --show-toplevel) + WT_BUILDDIR=$WT_TOPDIR/cmake_build + LD_LIBRARY_PATH=$WT_BUILDDIR:$WT_TOPDIR/TCMALLOC_LIB/lib + make_command: ninja + cmake_generator: Ninja + tasks: + - name: coverage-report + - name: cyclomatic-complexity + +- name: compatibility-tests-less-frequent + display_name: Compatibility tests (less frequent) + # Compatibility tests are NOT expected to be triggered on Evergreen projects + # other than "WiredTiger (develop)". + activate: false + batchtime: 10080 # 7 days + run_on: + - ubuntu1804-test + tasks: + - name: compatibility-test-for-older-releases + - name: compatibility-test-for-wt-standalone-releases + +- name: compatibility-tests-daily + display_name: Compatibility tests (daily) + # Compatibility tests are NOT expected to be triggered on Evergreen projects + # other than "WiredTiger (develop)". + activate: false + batchtime: 1440 # 1 day + run_on: + - ubuntu2004-test + tasks: + - name: compatibility-test-upgrade-to-latest + +- name: compatibility-tests + display_name: Compatibility tests + # Compatibility tests are NOT expected to be triggered on Evergreen projects + # other than "WiredTiger (develop)". + activate: false + run_on: + - ubuntu2004-test + tasks: + - name: compatibility-test-for-newer-releases + - name: compatibility-test-for-patch-releases + - name: import-compatibility-test + - name: windows-64 display_name: "! Windows 64-bit" run_on: - windows-64-vs2017-test expansions: - python_binary: '/cygdrive/c/python/Python311/python' - configure_env_vars: PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH - compile_env_vars: PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH + python_binary: '/cygdrive/c/Python39/python' test_env_vars: - PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build + PATH=/cygdrive/c/Python39:/cygdrive/c/Python39/Scripts:$PATH PYTHONPATH=($WT_TOPDIR/lang/python/wiredtiger):$(cygpath -w $WT_TOPDIR/lang/python) windows_configure_flags: -vcvars_bat "'C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\VC\Auxiliary\Build\vcvars64.bat'" tasks: @@ -4227,25 +4219,21 @@ buildvariants: posix_configure_flags: -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 - -DCC_OPTIMIZE_LEVEL=-O0 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL - python_binary: '/opt/mongodbtoolchain/v3/bin/python3' + python_binary: 'python3' smp_command: -j $(sysctl -n hw.logicalcpu) cmake_generator: "Unix Makefiles" make_command: make - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_BUILDDIR=$(git rev-parse --show-toplevel)/cmake_build DYLD_LIBRARY_PATH=$WT_BUILDDIR tasks: - name: compile - name: make-check-test - - name: unit-test-macos + - name: unit-test - name: fops - name: little-endian @@ -4256,8 +4244,6 @@ buildvariants: expansions: python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) @@ -4273,14 +4259,14 @@ buildvariants: - name: big-endian display_name: "~ Big-endian (s390x/zSeries)" + modules: + - enterprise run_on: - ubuntu1804-zseries-build batchtime: 4320 # 3 days expansions: python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) @@ -4296,15 +4282,14 @@ buildvariants: - name: rhel8-ppc display_name: "~ RHEL8 PPC" + # FIXME-WT-8981 + activate: false run_on: - rhel81-power8-small batchtime: 120 # 2 hours expansions: format_test_setting: ulimit -c unlimited - configure_env_vars: PATH=/opt/mongodbtoolchain/v4/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v4/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v4/bin:$PATH WT_BUILDDIR=$(git rev-parse --show-toplevel)/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR posix_configure_flags: @@ -4317,8 +4302,8 @@ buildvariants: -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v4/bin/python3' - # Use quarter of the vCPUs to avoid OOM kill failure and disk issues on this variant. - smp_command: -j $(echo $(grep -c ^processor /proc/cpuinfo) / 4 | bc) + # Use half number of vCPU to avoid OOM kill failure + smp_command: -j $(echo $(grep -c ^processor /proc/cpuinfo) / 2 | bc) cmake_generator: Ninja make_command: ninja tasks: @@ -4336,10 +4321,7 @@ buildvariants: - rhel80-zseries-test batchtime: 120 # 2 hours expansions: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_BUILDDIR=$(git rev-parse --show-toplevel)/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR posix_configure_flags: @@ -4370,10 +4352,7 @@ buildvariants: - ubuntu2004-arm64-small batchtime: 1440 # 24 hours expansions: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - compile_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH test_env_vars: - PATH=/opt/mongodbtoolchain/v3/bin:$PATH WT_TOPDIR=$(git rev-parse --show-toplevel) WT_BUILDDIR=$WT_TOPDIR/cmake_build LD_LIBRARY_PATH=$WT_BUILDDIR @@ -4399,7 +4378,6 @@ buildvariants: - name: checkpoint-filetypes-test - name: unit-test-zstd - name: unit-test-long - distros: ubuntu2004-arm64-large - name: spinlock-gcc-test - name: spinlock-pthread-adaptive-test - name: compile-wtperf diff --git a/src/third_party/wiredtiger/test/evergreen/print_python_stack_trace.py b/src/third_party/wiredtiger/test/evergreen/print_python_stack_trace.py new file mode 100644 index 00000000000..4dd81654ac3 --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/print_python_stack_trace.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import argparse +import itertools +import os +import re +import subprocess +import sys +from shutil import which + + +def border_msg(msg: str): + count = len(msg) + 2 + dash = "-" * count + return "+{dash}+\n| {msg} |\n+{dash}+".format(dash=dash, msg=msg) + + +class LLDBDumper: + """LLDBDumper class - prints stack traces on macOS""" + def __init__(self): + self.dbg = self._find_debugger("lldb") + + @staticmethod + def _find_debugger(debugger: str): + """Find the installed debugger.""" + return which(debugger) + + def dump(self, exe_path: str, core_path: str): + """Dump stack trace.""" + if self.dbg is None: + sys.exit("Debugger lldb not found," + "skipping dumping of {}".format(core_path)) + + cmds = [ + "thread backtrace all -c 30", + "quit" + ] + + subprocess.run([self.dbg, "--batch"] + [exe_path, "-c", core_path] + + list(itertools.chain.from_iterable([['-o', b] for b in cmds])), + check=True) + + +class GDBDumper: + """GDBDumper class - prints stack traces on Linux""" + def __init__(self): + self.dbg = self._find_debugger("gdb") + + @staticmethod + def _find_debugger(debugger: str): + """Find the installed debugger.""" + return which(debugger) + + def dump(self, exe_path: str, core_path: str, lib_path: str): + """Dump stack trace.""" + if self.dbg is None: + sys.exit("Debugger gdb not found," + "skipping dumping of {}".format(core_path)) + + cmds = [] + if lib_path: + cmds.append("set solib-search-path " + lib_path) + cmds.extend([ + "thread apply all backtrace 30", + "quit" + ]) + + subprocess.run([self.dbg, "--batch", "--quiet"] + + list(itertools.chain.from_iterable([['-ex', b] for b in cmds])) + + [exe_path, core_path], + check=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-e', '--executable_path', + help='path to the executable', + required=True) + parser.add_argument('-c', '--core_path', + help='directory path to the core dumps', + required=True) + parser.add_argument('-l', '--lib_path', help='library path') + args = parser.parse_args() + + # Store the path of the core files as a list. + core_files = [] + regex = re.compile(r'.*dump.*python.*', re.IGNORECASE) + for root, _, files in os.walk(args.core_path): + for file in files: + if regex.match(file): + core_files.extend([os.path.join(root, file)]) + + for core_file_path in core_files: + print(border_msg(core_file_path), flush=True) + if sys.platform.startswith('linux'): + dbg = GDBDumper() + dbg.dump(args.executable_path, core_file_path, args.lib_path) + elif sys.platform.startswith('darwin'): + # FIXME - macOS to be supported in WT-8976 + # dbg = LLDBDumper() + # dbg.dump(args.executable_path, core_file_path) + pass + elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'): + # FIXME - Windows to be supported in WT-8937 + pass + +if __name__ == "__main__": + main() diff --git a/src/third_party/wiredtiger/test/evergreen/print_stack_trace.py b/src/third_party/wiredtiger/test/evergreen/print_stack_trace.py deleted file mode 100644 index 9de0a0c4aca..00000000000 --- a/src/third_party/wiredtiger/test/evergreen/print_stack_trace.py +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env python3 -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import argparse -import itertools -import os -import re -import subprocess -import sys -from shutil import which - - -def border_msg(msg: str): - count = len(msg) + 2 - dash = "-" * count - return "+{dash}+\n| {msg} |\n+{dash}+".format(dash=dash, msg=msg) - - -class LLDBDumper: - """LLDBDumper class - prints stack traces on macOS""" - def __init__(self): - self.dbg = self._find_debugger("lldb") - - @staticmethod - def _find_debugger(debugger: str): - """Find the installed debugger.""" - return which(debugger) - - def dump(self, exe_path: str, core_path: str, dump_all: bool, output_file: str): - """Dump stack trace.""" - if self.dbg is None: - sys.exit("Debugger lldb not found," - "skipping dumping of {}".format(core_path)) - - cmds = [] - if dump_all: - cmds.append("thread apply all backtrace -c 30") - else: - cmds.append("backtrace -c 30") - cmds.append("quit") - - output = None - if (output_file): - try: - output = open(output_file, "w") - except OSError as e : - raise e - subprocess.run([self.dbg, "--batch"] + [exe_path, "-c", core_path] + - list(itertools.chain.from_iterable([['-o', b] for b in cmds])), - check=True, stdout=output) - - -class GDBDumper: - """GDBDumper class - prints stack traces on Linux""" - def __init__(self): - self.dbg = self._find_debugger("gdb") - - @staticmethod - def _find_debugger(debugger: str): - """Find the installed debugger.""" - return which(debugger) - - def dump(self, exe_path: str, core_path: str, lib_path: str, dump_all: bool, output_file: str): - """Dump stack trace.""" - if self.dbg is None: - sys.exit("Debugger gdb not found," - "skipping dumping of {}".format(core_path)) - - cmds = [] - if lib_path: - cmds.append("set solib-search-path " + lib_path) - - if dump_all: - cmds.append("thread apply all backtrace 30") - else: - cmds.append("backtrace 30") - cmds.append("quit") - - output = None - if (output_file): - try: - output = open(output_file, "w") - except OSError as e : - raise e - subprocess.run([self.dbg, "--batch", "--quiet"] + - list(itertools.chain.from_iterable([['-ex', b] for b in cmds])) + - [exe_path, core_path], - check=True, stdout=output) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--core_path', - help='directory path to the core dumps') - parser.add_argument('-l', '--lib_path', help='library path') - args = parser.parse_args() - - # If the lib_path is not provided then search the current dir. - lib_path = "." if args.lib_path is None else args.lib_path - - # If the core_path is not provided then search the current dir. - core_path = "." if args.core_path is None else args.core_path - - # Append the path of the core files present in the core path in a list. - core_files = [] - regex = re.compile(r'dump.*core', re.IGNORECASE) - for root, _, files in os.walk(core_path): - for file in files: - if regex.match(file): - core_files.append(os.path.join(root, file)) - - for core_file_path in core_files: - print(border_msg(core_file_path), flush=True) - - # Get the executable from the core file itself. - proc = subprocess.Popen(["file", core_file_path], stdout=subprocess.PIPE) - # The file command prints something similar to: - # WT_TEST/test_practice.0/dump_python3.16562.core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), - # SVR4-style, from 'python3 /home/ubuntu/wiredtiger/test/suite/run.py -j 1 -p test_practice', - # real uid: 1000, effective uid: 1000, real gid: 1000, effective gid: 1000, execfn: '/usr/bin/python3', platform: 'x86_64' - output = str(proc.communicate()) - - # The field of interest is execfn: '/usr/bin/python3' from the example above. - start = output.find('execfn: ') - if start < 0: - print("The 'execfn' is missing, skipping core...") - continue - # Fetch the value of execfn. This is the executable path! - executable_path = re.search(r'\'.*?\'', output[start:]).group(0) - executable_path = executable_path.replace("'", "") - - # If the core dump comes from a Python test, we don't need to construct the executable path. - if "python" not in executable_path.lower(): - # We may have the executable path already. If not, we need to find it. - if not os.access(executable_path, os.X_OK): - # The executable is where the core dump is. - dirname = core_file_path.rsplit('/', 1)[0] - executable_path = dirname + '/' + executable_path - - if sys.platform.startswith('linux'): - dbg = GDBDumper() - dbg.dump(executable_path, core_file_path, lib_path, False, None) - - # Extract the filename from the core file path, to create a stacktrace output file. - file_name, _ = os.path.splitext(os.path.basename(core_file_path)) - dbg.dump(executable_path, core_file_path, lib_path, True, file_name + ".stacktrace.txt") - elif sys.platform.startswith('darwin'): - # FIXME - macOS to be supported in WT-8976 - # dbg = LLDBDumper() - # dbg.dump(args.executable_path, core_file_path, False, None) - - # Extract the filename from the core file path, to create a stacktrace output file. - # file_name, _ = os.path.splitext(os.path.basename(core_file_path)) - # dbg.dump(executable_path, core_file_path, True, file_name + ".stacktrace.txt") - pass - elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'): - # FIXME - Windows to be supported in WT-8937 - pass - -if __name__ == "__main__": - main() diff --git a/src/third_party/wiredtiger/test/evergreen/run_format_configs.sh b/src/third_party/wiredtiger/test/evergreen/run_format_configs.sh index 99a2766c57d..a29fb30edab 100755 --- a/src/third_party/wiredtiger/test/evergreen/run_format_configs.sh +++ b/src/third_party/wiredtiger/test/evergreen/run_format_configs.sh @@ -26,7 +26,7 @@ failure=0 for config in $(find ../../../test/format/failure_configs/ -name "CONFIG.*" | sort) do echo -e "\nTesting CONFIG $config ...\n" - if (./t -1 -c $config); then + if (./t -c $config); then let "success++" else let "failure++" diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 40f8053afb8..bbff4a1ab17 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -256,18 +256,8 @@ config_table(TABLE *table, void *arg) * direct I/O can be so slow the additional I/O for overflow items causes eviction to stall). */ if (GV(RUNS_IN_MEMORY) || GV(DISK_DIRECT_IO)) { - /* - * Always limit the row count if its greater that 1,000,000 and in memory wasn't explicitly - * set. Direct IO is always explicitly set, never limit the row count because the user has - * taken control. - */ - if (GV(RUNS_IN_MEMORY) && TV(RUNS_ROWS) > WT_MILLION && - config_explicit(NULL, "runs.in_memory")) { - WARN("limiting table%" PRIu32 - ".runs.rows to 1,000,000 as runs.in_memory has been automatically enabled", - table->id) + if (!config_explicit(table, "runs.rows") && TV(RUNS_ROWS) > 1000000) config_single(table, "runs.rows=1000000", false); - } if (!config_explicit(table, "btree.key_max")) config_single(table, "btree.key_max=32", false); if (!config_explicit(table, "btree.key_min")) @@ -856,15 +846,8 @@ config_in_memory(void) if (config_explicit(NULL, "ops.verify")) return; - if (!config_explicit(NULL, "runs.in_memory") && mmrand(NULL, 1, 20) == 1) { + if (!config_explicit(NULL, "runs.in_memory") && mmrand(NULL, 1, 20) == 1) config_single(NULL, "runs.in_memory=1", false); - /* Use table[0] to access the global value (RUNS_ROWS is a table value). */ - if ((tables[0]->v[V_TABLE_RUNS_ROWS].v) > WT_MILLION) { - WARN("%s", - "limiting runs.rows to 1,000,000 as runs.in_memory has been automatically enabled"); - config_single(NULL, "runs.rows=1000000", true); - } - } } /* diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index bb80b82bc9d..14fb612d31d 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -69,72 +69,67 @@ typedef struct { #define V_TABLE_DISK_FIRSTFIT 37 #define V_GLOBAL_DISK_MMAP 38 #define V_GLOBAL_DISK_MMAP_ALL 39 -#define V_GLOBAL_FILE_MANAGER_CLOSE_HANDLE_MINIMUM 40 -#define V_GLOBAL_FILE_MANAGER_CLOSE_IDLE_TIME 41 -#define V_GLOBAL_FILE_MANAGER_CLOSE_SCAN_INTERVAL 42 -#define V_GLOBAL_FORMAT_ABORT 43 -#define V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG 44 -#define V_GLOBAL_FORMAT_MAJOR_TIMEOUT 45 -#define V_GLOBAL_IMPORT 46 -#define V_GLOBAL_LOGGING 47 -#define V_GLOBAL_LOGGING_COMPRESSION 48 -#define V_GLOBAL_LOGGING_FILE_MAX 49 -#define V_GLOBAL_LOGGING_PREALLOC 50 -#define V_GLOBAL_LOGGING_REMOVE 51 -#define V_TABLE_LSM_AUTO_THROTTLE 52 -#define V_TABLE_LSM_BLOOM 53 -#define V_TABLE_LSM_BLOOM_BIT_COUNT 54 -#define V_TABLE_LSM_BLOOM_HASH_COUNT 55 -#define V_TABLE_LSM_BLOOM_OLDEST 56 -#define V_TABLE_LSM_CHUNK_SIZE 57 -#define V_TABLE_LSM_MERGE_MAX 58 -#define V_GLOBAL_LSM_WORKER_THREADS 59 -#define V_GLOBAL_OPS_ALTER 60 -#define V_GLOBAL_OPS_COMPACTION 61 -#define V_GLOBAL_OPS_HS_CURSOR 62 -#define V_TABLE_OPS_PCT_DELETE 63 -#define V_TABLE_OPS_PCT_INSERT 64 -#define V_TABLE_OPS_PCT_MODIFY 65 -#define V_TABLE_OPS_PCT_READ 66 -#define V_TABLE_OPS_PCT_WRITE 67 -#define V_GLOBAL_OPS_PREPARE 68 -#define V_GLOBAL_OPS_RANDOM_CURSOR 69 -#define V_GLOBAL_OPS_SALVAGE 70 -#define V_TABLE_OPS_TRUNCATE 71 -#define V_GLOBAL_OPS_VERIFY 72 -#define V_GLOBAL_QUIET 73 -#define V_GLOBAL_RUNS_IN_MEMORY 74 -#define V_GLOBAL_RUNS_OPS 75 -#define V_TABLE_RUNS_ROWS 76 -#define V_TABLE_RUNS_SOURCE 77 -#define V_GLOBAL_RUNS_TABLES 78 -#define V_GLOBAL_RUNS_THREADS 79 -#define V_GLOBAL_RUNS_TIMER 80 -#define V_TABLE_RUNS_TYPE 81 -#define V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP 82 -#define V_GLOBAL_STATISTICS 83 -#define V_GLOBAL_STATISTICS_SERVER 84 -#define V_GLOBAL_STRESS_AGGRESSIVE_SWEEP 85 -#define V_GLOBAL_STRESS_CHECKPOINT 86 -#define V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 87 -#define V_GLOBAL_STRESS_CHECKPOINT_PREPARE 88 -#define V_GLOBAL_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION 89 -#define V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS 90 -#define V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY 91 -#define V_GLOBAL_STRESS_HS_SEARCH 92 -#define V_GLOBAL_STRESS_HS_SWEEP 93 -#define V_GLOBAL_STRESS_PREPARE_RESOLUTION 94 -#define V_GLOBAL_STRESS_SPLIT_1 95 -#define V_GLOBAL_STRESS_SPLIT_2 96 -#define V_GLOBAL_STRESS_SPLIT_3 97 -#define V_GLOBAL_STRESS_SPLIT_4 98 -#define V_GLOBAL_STRESS_SPLIT_5 99 -#define V_GLOBAL_STRESS_SPLIT_6 100 -#define V_GLOBAL_STRESS_SPLIT_7 101 -#define V_GLOBAL_TRANSACTION_IMPLICIT 102 -#define V_GLOBAL_TRANSACTION_TIMESTAMPS 103 -#define V_GLOBAL_WIREDTIGER_CONFIG 104 -#define V_GLOBAL_WIREDTIGER_RWLOCK 105 -#define V_GLOBAL_WIREDTIGER_LEAK_MEMORY 106 +#define V_GLOBAL_FORMAT_ABORT 40 +#define V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG 41 +#define V_GLOBAL_FORMAT_MAJOR_TIMEOUT 42 +#define V_GLOBAL_IMPORT 43 +#define V_GLOBAL_LOGGING 44 +#define V_GLOBAL_LOGGING_COMPRESSION 45 +#define V_GLOBAL_LOGGING_FILE_MAX 46 +#define V_GLOBAL_LOGGING_PREALLOC 47 +#define V_GLOBAL_LOGGING_REMOVE 48 +#define V_TABLE_LSM_AUTO_THROTTLE 49 +#define V_TABLE_LSM_BLOOM 50 +#define V_TABLE_LSM_BLOOM_BIT_COUNT 51 +#define V_TABLE_LSM_BLOOM_HASH_COUNT 52 +#define V_TABLE_LSM_BLOOM_OLDEST 53 +#define V_TABLE_LSM_CHUNK_SIZE 54 +#define V_TABLE_LSM_MERGE_MAX 55 +#define V_GLOBAL_LSM_WORKER_THREADS 56 +#define V_GLOBAL_OPS_ALTER 57 +#define V_GLOBAL_OPS_COMPACTION 58 +#define V_GLOBAL_OPS_HS_CURSOR 59 +#define V_TABLE_OPS_PCT_DELETE 60 +#define V_TABLE_OPS_PCT_INSERT 61 +#define V_TABLE_OPS_PCT_MODIFY 62 +#define V_TABLE_OPS_PCT_READ 63 +#define V_TABLE_OPS_PCT_WRITE 64 +#define V_GLOBAL_OPS_PREPARE 65 +#define V_GLOBAL_OPS_RANDOM_CURSOR 66 +#define V_GLOBAL_OPS_SALVAGE 67 +#define V_TABLE_OPS_TRUNCATE 68 +#define V_GLOBAL_OPS_VERIFY 69 +#define V_GLOBAL_QUIET 70 +#define V_GLOBAL_RUNS_IN_MEMORY 71 +#define V_GLOBAL_RUNS_OPS 72 +#define V_TABLE_RUNS_ROWS 73 +#define V_TABLE_RUNS_SOURCE 74 +#define V_GLOBAL_RUNS_TABLES 75 +#define V_GLOBAL_RUNS_THREADS 76 +#define V_GLOBAL_RUNS_TIMER 77 +#define V_TABLE_RUNS_TYPE 78 +#define V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP 79 +#define V_GLOBAL_STATISTICS 80 +#define V_GLOBAL_STATISTICS_SERVER 81 +#define V_GLOBAL_STRESS_AGGRESSIVE_SWEEP 82 +#define V_GLOBAL_STRESS_CHECKPOINT 83 +#define V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 84 +#define V_GLOBAL_STRESS_CHECKPOINT_PREPARE 85 +#define V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS 86 +#define V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY 87 +#define V_GLOBAL_STRESS_HS_SEARCH 88 +#define V_GLOBAL_STRESS_HS_SWEEP 89 +#define V_GLOBAL_STRESS_SPLIT_1 90 +#define V_GLOBAL_STRESS_SPLIT_2 91 +#define V_GLOBAL_STRESS_SPLIT_3 92 +#define V_GLOBAL_STRESS_SPLIT_4 93 +#define V_GLOBAL_STRESS_SPLIT_5 94 +#define V_GLOBAL_STRESS_SPLIT_6 95 +#define V_GLOBAL_STRESS_SPLIT_7 96 +#define V_GLOBAL_TRANSACTION_IMPLICIT 97 +#define V_GLOBAL_TRANSACTION_TIMESTAMPS 98 +#define V_GLOBAL_WIREDTIGER_CONFIG 99 +#define V_GLOBAL_WIREDTIGER_RWLOCK 100 +#define V_GLOBAL_WIREDTIGER_LEAK_MEMORY 101 -#define V_ELEMENT_COUNT 107 +#define V_ELEMENT_COUNT 102 diff --git a/src/third_party/wiredtiger/test/format/config.sh b/src/third_party/wiredtiger/test/format/config.sh index 7f51c9879d8..23af56e2724 100644 --- a/src/third_party/wiredtiger/test/format/config.sh +++ b/src/third_party/wiredtiger/test/format/config.sh @@ -151,13 +151,6 @@ CONFIG configuration_list[] = { {"disk.mmap_all", "configure mmap operations (read and write)", C_BOOL, 5, 0, 0} -/* Test format can only handle 32 tables so we use a maximum value of 32 here. */ -{"file_manager.close_handle_minimum", "number of handles open before the file manager will look for handles to close", 0x0, 0, 32, 32} - -{"file_manager.close_idle_time", "amount of time in seconds a file handle needs to be idle before attempting to close it. A setting of 0 means that idle handles are not closed", 0x0, 0, 60, 100000} - -{"file_manager.close_scan_interval", "interval in seconds at which to check for files that are inactive and close them", 0x0, 0, 30, 100000} - {"format.abort", "drop core during timed run", C_BOOL, 0, 0, 0} {"format.independent_thread_rng", "configure independent thread RNG space", C_BOOL, 75, 0, 0} @@ -255,8 +248,6 @@ CONFIG configuration_list[] = { {"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0} -{"stress.failpoint_eviction_fail_after_reconciliation", "stress failpoint eviction fail after reconciliation", C_BOOL, 30, 0, 0} - {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", C_BOOL, 30, 0, 0} {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0} @@ -265,8 +256,6 @@ CONFIG configuration_list[] = { {"stress.hs_sweep", "stress history store sweep", C_BOOL, 2, 0, 0} -{"stress.prepare_resolution", "stress prepare resolution", C_BOOL, 2, 0, 0} - {"stress.split_1", "stress splits (#1)", C_BOOL, 2, 0, 0} {"stress.split_2", "stress splits (#2)", C_BOOL, 2, 0, 0} diff --git a/src/third_party/wiredtiger/test/format/config_def.c b/src/third_party/wiredtiger/test/format/config_def.c index c33e7697dea..78613c1c1ab 100644 --- a/src/third_party/wiredtiger/test/format/config_def.c +++ b/src/third_party/wiredtiger/test/format/config_def.c @@ -2,36 +2,32 @@ #include "format.h" -CONFIG configuration_list[] = { - {"assert.read_timestamp", "assert read_timestamp", - C_BOOL, 2, 0, 0, V_GLOBAL_ASSERT_READ_TIMESTAMP}, +CONFIG configuration_list[] = {{"assert.read_timestamp", "assert read_timestamp", C_BOOL, 2, 0, 0, + V_GLOBAL_ASSERT_READ_TIMESTAMP}, - {"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", - C_BOOL, 2, 0, 0, V_GLOBAL_ASSERT_WRITE_TIMESTAMP}, + {"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", C_BOOL, 2, 0, + 0, V_GLOBAL_ASSERT_WRITE_TIMESTAMP}, - {"backup", "configure backups", - C_BOOL, 20, 0, 0, V_GLOBAL_BACKUP}, + {"backup", "configure backups", C_BOOL, 20, 0, 0, V_GLOBAL_BACKUP}, - {"backup.incremental", "backup type (off | block | log)", - C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_BACKUP_INCREMENTAL}, + {"backup.incremental", "backup type (off | block | log)", C_IGNORE | C_STRING, 0, 0, 0, + V_GLOBAL_BACKUP_INCREMENTAL}, - {"backup.incr_granularity", "incremental backup block granularity (KB)", - 0x0, 4, 16384, 16384, V_GLOBAL_BACKUP_INCR_GRANULARITY}, + {"backup.incr_granularity", "incremental backup block granularity (KB)", 0x0, 4, 16384, 16384, + V_GLOBAL_BACKUP_INCR_GRANULARITY}, - {"block_cache", "enable the block cache", - C_BOOL, 10, 0, 0, V_GLOBAL_BLOCK_CACHE}, + {"block_cache", "enable the block cache", C_BOOL, 10, 0, 0, V_GLOBAL_BLOCK_CACHE}, - {"block_cache.cache_on_checkpoint", "block cache: cache checkpoint writes", - C_BOOL, 30, 0, 0, V_GLOBAL_BLOCK_CACHE_CACHE_ON_CHECKPOINT}, + {"block_cache.cache_on_checkpoint", "block cache: cache checkpoint writes", C_BOOL, 30, 0, 0, + V_GLOBAL_BLOCK_CACHE_CACHE_ON_CHECKPOINT}, - {"block_cache.cache_on_writes", "block cache: populate the cache on writes", - C_BOOL, 60, 0, 0, V_GLOBAL_BLOCK_CACHE_CACHE_ON_WRITES}, + {"block_cache.cache_on_writes", "block cache: populate the cache on writes", C_BOOL, 60, 0, 0, + V_GLOBAL_BLOCK_CACHE_CACHE_ON_WRITES}, - {"block_cache.size", "block cache size (MB)", - 0x0, 1, 100, 100 * 1024, V_GLOBAL_BLOCK_CACHE_SIZE}, + {"block_cache.size", "block cache size (MB)", 0x0, 1, 100, 100 * 1024, V_GLOBAL_BLOCK_CACHE_SIZE}, - {"btree.bitcnt", "fixed-length column-store object size (number of bits)", - C_TABLE | C_TYPE_FIX, 1, 8, 8, V_TABLE_BTREE_BITCNT}, + {"btree.bitcnt", "fixed-length column-store object size (number of bits)", C_TABLE | C_TYPE_FIX, + 1, 8, 8, V_TABLE_BTREE_BITCNT}, {"btree.compression", "data compression (off | lz4 | snappy | zlib | zstd)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_BTREE_COMPRESSION}, @@ -42,293 +38,244 @@ CONFIG configuration_list[] = { {"btree.huffman_value", "configure huffman encoded values", C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0, V_TABLE_BTREE_HUFFMAN_VALUE}, - {"btree.internal_key_truncation", "truncate internal keys", - C_BOOL | C_TABLE, 95, 0, 0, V_TABLE_BTREE_INTERNAL_KEY_TRUNCATION}, + {"btree.internal_key_truncation", "truncate internal keys", C_BOOL | C_TABLE, 95, 0, 0, + V_TABLE_BTREE_INTERNAL_KEY_TRUNCATION}, - {"btree.internal_page_max", "btree internal node maximum size", - C_TABLE, 9, 17, 27, V_TABLE_BTREE_INTERNAL_PAGE_MAX}, + {"btree.internal_page_max", "btree internal node maximum size", C_TABLE, 9, 17, 27, + V_TABLE_BTREE_INTERNAL_PAGE_MAX}, - {"btree.key_max", "maximum key size", - C_TABLE | C_TYPE_ROW, 20, 128, MEGABYTE(10), V_TABLE_BTREE_KEY_MAX}, + {"btree.key_max", "maximum key size", C_TABLE | C_TYPE_ROW, 20, 128, MEGABYTE(10), + V_TABLE_BTREE_KEY_MAX}, - {"btree.key_min", "minimum key size", - C_TABLE | C_TYPE_ROW, KEY_LEN_CONFIG_MIN, 32, 256, V_TABLE_BTREE_KEY_MIN}, + {"btree.key_min", "minimum key size", C_TABLE | C_TYPE_ROW, KEY_LEN_CONFIG_MIN, 32, 256, + V_TABLE_BTREE_KEY_MIN}, - {"btree.leaf_page_max", "btree leaf node maximum size", - C_TABLE, 9, 17, 27, V_TABLE_BTREE_LEAF_PAGE_MAX}, + {"btree.leaf_page_max", "btree leaf node maximum size", C_TABLE, 9, 17, 27, + V_TABLE_BTREE_LEAF_PAGE_MAX}, - {"btree.memory_page_max", "maximum cache page size", - C_TABLE, 1, 10, 128, V_TABLE_BTREE_MEMORY_PAGE_MAX}, + {"btree.memory_page_max", "maximum cache page size", C_TABLE, 1, 10, 128, + V_TABLE_BTREE_MEMORY_PAGE_MAX}, - {"btree.prefix_len", "common key prefix", - C_TABLE | C_TYPE_ROW | C_ZERO_NOTSET, PREFIX_LEN_CONFIG_MIN, PREFIX_LEN_CONFIG_MAX, PREFIX_LEN_CONFIG_MAX, V_TABLE_BTREE_PREFIX_LEN}, + {"btree.prefix_len", "common key prefix", C_TABLE | C_TYPE_ROW | C_ZERO_NOTSET, + PREFIX_LEN_CONFIG_MIN, PREFIX_LEN_CONFIG_MAX, PREFIX_LEN_CONFIG_MAX, V_TABLE_BTREE_PREFIX_LEN}, - {"btree.prefix_compression", "configure prefix compressed keys", - C_BOOL | C_TABLE | C_TYPE_ROW, 80, 0, 0, V_TABLE_BTREE_PREFIX_COMPRESSION}, + {"btree.prefix_compression", "configure prefix compressed keys", C_BOOL | C_TABLE | C_TYPE_ROW, + 80, 0, 0, V_TABLE_BTREE_PREFIX_COMPRESSION}, {"btree.prefix_compression_min", "minimum gain before prefix compression is used (bytes)", C_TABLE | C_TYPE_ROW, 0, 8, 256, V_TABLE_BTREE_PREFIX_COMPRESSION_MIN}, - {"btree.repeat_data_pct", "duplicate values (percentage)", - C_TABLE | C_TYPE_VAR, 0, 90, 90, V_TABLE_BTREE_REPEAT_DATA_PCT}, + {"btree.repeat_data_pct", "duplicate values (percentage)", C_TABLE | C_TYPE_VAR, 0, 90, 90, + V_TABLE_BTREE_REPEAT_DATA_PCT}, - {"btree.reverse", "reverse order collation", - C_BOOL | C_TABLE | C_TYPE_ROW, 10, 0, 0, V_TABLE_BTREE_REVERSE}, + {"btree.reverse", "reverse order collation", C_BOOL | C_TABLE | C_TYPE_ROW, 10, 0, 0, + V_TABLE_BTREE_REVERSE}, - {"btree.split_pct", "page split size as a percentage of the maximum page size", - C_TABLE, 50, 100, 100, V_TABLE_BTREE_SPLIT_PCT}, + {"btree.split_pct", "page split size as a percentage of the maximum page size", C_TABLE, 50, 100, + 100, V_TABLE_BTREE_SPLIT_PCT}, - {"btree.value_max", "maximum value size", - C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 32, 4096, MEGABYTE(10), V_TABLE_BTREE_VALUE_MAX}, + {"btree.value_max", "maximum value size", C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 32, 4096, + MEGABYTE(10), V_TABLE_BTREE_VALUE_MAX}, - {"btree.value_min", "minimum value size", - C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 0, 20, 4096, V_TABLE_BTREE_VALUE_MIN}, + {"btree.value_min", "minimum value size", C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 0, 20, 4096, + V_TABLE_BTREE_VALUE_MIN}, - {"cache", "cache size (MB)", - 0x0, 1, 100, 100 * 1024, V_GLOBAL_CACHE}, + {"cache", "cache size (MB)", 0x0, 1, 100, 100 * 1024, V_GLOBAL_CACHE}, - {"cache.evict_max", "maximum number of eviction workers", - 0x0, 0, 5, 100, V_GLOBAL_CACHE_EVICT_MAX}, + {"cache.evict_max", "maximum number of eviction workers", 0x0, 0, 5, 100, + V_GLOBAL_CACHE_EVICT_MAX}, - {"cache.minimum", "minimum cache size (MB)", - C_IGNORE, 0, 0, 100 * 1024, V_GLOBAL_CACHE_MINIMUM}, + {"cache.minimum", "minimum cache size (MB)", C_IGNORE, 0, 0, 100 * 1024, V_GLOBAL_CACHE_MINIMUM}, - {"checkpoint", "checkpoint type (on | off | wiredtiger)", - C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_CHECKPOINT}, + {"checkpoint", "checkpoint type (on | off | wiredtiger)", C_IGNORE | C_STRING, 0, 0, 0, + V_GLOBAL_CHECKPOINT}, - {"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", - 0x0, 20, 200, 1024, V_GLOBAL_CHECKPOINT_LOG_SIZE}, + {"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", 0x0, 20, 200, + 1024, V_GLOBAL_CHECKPOINT_LOG_SIZE}, - {"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", - 0x0, 5, 100, 3600, V_GLOBAL_CHECKPOINT_WAIT}, + {"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", 0x0, 5, 100, 3600, + V_GLOBAL_CHECKPOINT_WAIT}, {"disk.checksum", "checksum type (on | off | uncompressed | unencrypted)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_DISK_CHECKSUM}, - {"disk.data_extend", "configure data file extension", - C_BOOL, 5, 0, 0, V_GLOBAL_DISK_DATA_EXTEND}, + {"disk.data_extend", "configure data file extension", C_BOOL, 5, 0, 0, V_GLOBAL_DISK_DATA_EXTEND}, - {"disk.direct_io", "configure direct I/O for data objects", - C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_DISK_DIRECT_IO}, + {"disk.direct_io", "configure direct I/O for data objects", C_BOOL | C_IGNORE, 0, 0, 1, + V_GLOBAL_DISK_DIRECT_IO}, - {"disk.encryption", "encryption type (off | rotn-7)", - C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_DISK_ENCRYPTION}, + {"disk.encryption", "encryption type (off | rotn-7)", C_IGNORE | C_STRING, 0, 0, 0, + V_GLOBAL_DISK_ENCRYPTION}, - {"disk.firstfit", "configure first-fit allocation", - C_BOOL | C_TABLE, 10, 0, 0, V_TABLE_DISK_FIRSTFIT}, + {"disk.firstfit", "configure first-fit allocation", C_BOOL | C_TABLE, 10, 0, 0, + V_TABLE_DISK_FIRSTFIT}, - {"disk.mmap", "configure mmap operations (reads only)", - C_BOOL, 90, 0, 0, V_GLOBAL_DISK_MMAP}, + {"disk.mmap", "configure mmap operations (reads only)", C_BOOL, 90, 0, 0, V_GLOBAL_DISK_MMAP}, - {"disk.mmap_all", "configure mmap operations (read and write)", - C_BOOL, 5, 0, 0, V_GLOBAL_DISK_MMAP_ALL}, + {"disk.mmap_all", "configure mmap operations (read and write)", C_BOOL, 5, 0, 0, + V_GLOBAL_DISK_MMAP_ALL}, -/* Test format can only handle 32 tables so we use a maximum value of 32 here. */ - {"file_manager.close_handle_minimum", "number of handles open before the file manager will look for handles to close", - 0x0, 0, 32, 32, V_GLOBAL_FILE_MANAGER_CLOSE_HANDLE_MINIMUM}, + {"format.abort", "drop core during timed run", C_BOOL, 0, 0, 0, V_GLOBAL_FORMAT_ABORT}, - {"file_manager.close_idle_time", "amount of time in seconds a file handle needs to be idle before attempting to close it. A setting of 0 means that idle handles are not closed", - 0x0, 0, 60, 100000, V_GLOBAL_FILE_MANAGER_CLOSE_IDLE_TIME}, + {"format.independent_thread_rng", "configure independent thread RNG space", C_BOOL, 75, 0, 0, + V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG}, - {"file_manager.close_scan_interval", "interval in seconds at which to check for files that are inactive and close them", - 0x0, 0, 30, 100000, V_GLOBAL_FILE_MANAGER_CLOSE_SCAN_INTERVAL}, + {"format.major_timeout", "long-running operations timeout (minutes)", C_IGNORE, 0, 0, 1000, + V_GLOBAL_FORMAT_MAJOR_TIMEOUT}, - {"format.abort", "drop core during timed run", - C_BOOL, 0, 0, 0, V_GLOBAL_FORMAT_ABORT}, + /* + * 0% + * FIXME-WT-7418: Temporarily disable import until WT_ROLLBACK error and wt_copy_and_sync error is + * fixed. It should be (C_BOOL, 20, 0, 0). + */ + {"import", "import table from newly created database", C_BOOL, 0, 0, 0, V_GLOBAL_IMPORT}, - {"format.independent_thread_rng", "configure independent thread RNG space", - C_BOOL, 75, 0, 0, V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG}, - - {"format.major_timeout", "long-running operations timeout (minutes)", - C_IGNORE, 0, 0, 1000, V_GLOBAL_FORMAT_MAJOR_TIMEOUT}, - -/* - * 0% - * FIXME-WT-7418: Temporarily disable import until WT_ROLLBACK error and wt_copy_and_sync error is - * fixed. It should be (C_BOOL, 20, 0, 0). - */ - {"import", "import table from newly created database", - C_BOOL, 0, 0, 0, V_GLOBAL_IMPORT}, - - {"logging", "configure logging", - C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING}, + {"logging", "configure logging", C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING}, {"logging.compression", "logging compression (off | lz4 | snappy | zlib | zstd)", C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_LOGGING_COMPRESSION}, - {"logging.file_max", "maximum log file size (KB)", - 0x0, 100, 512000, 2097152, V_GLOBAL_LOGGING_FILE_MAX}, + {"logging.file_max", "maximum log file size (KB)", 0x0, 100, 512000, 2097152, + V_GLOBAL_LOGGING_FILE_MAX}, - {"logging.prealloc", "configure log file pre-allocation", - C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING_PREALLOC}, + {"logging.prealloc", "configure log file pre-allocation", C_BOOL, 50, 0, 0, + V_GLOBAL_LOGGING_PREALLOC}, - {"logging.remove", "configure log file removal", - C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING_REMOVE}, + {"logging.remove", "configure log file removal", C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING_REMOVE}, - {"lsm.auto_throttle", "throttle LSM inserts", - C_BOOL | C_TABLE | C_TYPE_LSM, 90, 0, 0, V_TABLE_LSM_AUTO_THROTTLE}, + {"lsm.auto_throttle", "throttle LSM inserts", C_BOOL | C_TABLE | C_TYPE_LSM, 90, 0, 0, + V_TABLE_LSM_AUTO_THROTTLE}, - {"lsm.bloom", "configure bloom filters", - C_BOOL | C_TABLE | C_TYPE_LSM, 95, 0, 0, V_TABLE_LSM_BLOOM}, + {"lsm.bloom", "configure bloom filters", C_BOOL | C_TABLE | C_TYPE_LSM, 95, 0, 0, + V_TABLE_LSM_BLOOM}, - {"lsm.bloom_bit_count", "number of bits per item for bloom filters", - C_TABLE | C_TYPE_LSM, 4, 64, 1000, V_TABLE_LSM_BLOOM_BIT_COUNT}, + {"lsm.bloom_bit_count", "number of bits per item for bloom filters", C_TABLE | C_TYPE_LSM, 4, 64, + 1000, V_TABLE_LSM_BLOOM_BIT_COUNT}, - {"lsm.bloom_hash_count", "number of hash values per item for bloom filters", - C_TABLE | C_TYPE_LSM, 4, 32, 100, V_TABLE_LSM_BLOOM_HASH_COUNT}, + {"lsm.bloom_hash_count", "number of hash values per item for bloom filters", C_TABLE | C_TYPE_LSM, + 4, 32, 100, V_TABLE_LSM_BLOOM_HASH_COUNT}, - {"lsm.bloom_oldest", "configure bloom_oldest=true", - C_BOOL | C_TABLE | C_TYPE_LSM, 10, 0, 0, V_TABLE_LSM_BLOOM_OLDEST}, + {"lsm.bloom_oldest", "configure bloom_oldest=true", C_BOOL | C_TABLE | C_TYPE_LSM, 10, 0, 0, + V_TABLE_LSM_BLOOM_OLDEST}, - {"lsm.chunk_size", "LSM chunk size (MB)", - C_TABLE | C_TYPE_LSM, 1, 10, 100, V_TABLE_LSM_CHUNK_SIZE}, + {"lsm.chunk_size", "LSM chunk size (MB)", C_TABLE | C_TYPE_LSM, 1, 10, 100, + V_TABLE_LSM_CHUNK_SIZE}, {"lsm.merge_max", "maximum number of chunks to include in an LSM merge operation", C_TABLE | C_TYPE_LSM, 4, 20, 100, V_TABLE_LSM_MERGE_MAX}, - {"lsm.worker_threads", "number of LSM worker threads", - C_TYPE_LSM, 3, 4, 20, V_GLOBAL_LSM_WORKER_THREADS}, + {"lsm.worker_threads", "number of LSM worker threads", C_TYPE_LSM, 3, 4, 20, + V_GLOBAL_LSM_WORKER_THREADS}, - {"ops.alter", "configure table alterations", - C_BOOL, 10, 0, 0, V_GLOBAL_OPS_ALTER}, + {"ops.alter", "configure table alterations", C_BOOL, 10, 0, 0, V_GLOBAL_OPS_ALTER}, - {"ops.compaction", "configure compaction", - C_BOOL, 10, 0, 0, V_GLOBAL_OPS_COMPACTION}, + {"ops.compaction", "configure compaction", C_BOOL, 10, 0, 0, V_GLOBAL_OPS_COMPACTION}, - {"ops.hs_cursor", "configure history store cursor reads", - C_BOOL, 50, 0, 0, V_GLOBAL_OPS_HS_CURSOR}, + {"ops.hs_cursor", "configure history store cursor reads", C_BOOL, 50, 0, 0, + V_GLOBAL_OPS_HS_CURSOR}, - {"ops.pct.delete", "delete operations (percentage)", - C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_DELETE}, + {"ops.pct.delete", "delete operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100, + V_TABLE_OPS_PCT_DELETE}, - {"ops.pct.insert", "insert operations (percentage)", - C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_INSERT}, + {"ops.pct.insert", "insert operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100, + V_TABLE_OPS_PCT_INSERT}, - {"ops.pct.modify", "modify operations (percentage)", - C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_MODIFY}, + {"ops.pct.modify", "modify operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100, + V_TABLE_OPS_PCT_MODIFY}, - {"ops.pct.read", "read operations (percentage)", - C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_READ}, + {"ops.pct.read", "read operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100, + V_TABLE_OPS_PCT_READ}, - {"ops.pct.write", "update operations (percentage)", - C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_WRITE}, + {"ops.pct.write", "update operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100, + V_TABLE_OPS_PCT_WRITE}, - {"ops.prepare", "configure transaction prepare", - C_BOOL, 5, 0, 0, V_GLOBAL_OPS_PREPARE}, + {"ops.prepare", "configure transaction prepare", C_BOOL, 5, 0, 0, V_GLOBAL_OPS_PREPARE}, - {"ops.random_cursor", "configure random cursor reads", - C_BOOL, 10, 0, 0, V_GLOBAL_OPS_RANDOM_CURSOR}, + {"ops.random_cursor", "configure random cursor reads", C_BOOL, 10, 0, 0, + V_GLOBAL_OPS_RANDOM_CURSOR}, - {"ops.salvage", "configure salvage", - C_BOOL, 100, 1, 0, V_GLOBAL_OPS_SALVAGE}, + {"ops.salvage", "configure salvage", C_BOOL, 100, 1, 0, V_GLOBAL_OPS_SALVAGE}, - {"ops.truncate", "configure truncation", - C_BOOL | C_TABLE, 100, 0, 0, V_TABLE_OPS_TRUNCATE}, + {"ops.truncate", "configure truncation", C_BOOL | C_TABLE, 100, 0, 0, V_TABLE_OPS_TRUNCATE}, - {"ops.verify", "configure verify", - C_BOOL, 100, 1, 0, V_GLOBAL_OPS_VERIFY}, + {"ops.verify", "configure verify", C_BOOL, 100, 1, 0, V_GLOBAL_OPS_VERIFY}, - {"quiet", "quiet run (same as -q)", - C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_QUIET}, + {"quiet", "quiet run (same as -q)", C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_QUIET}, - {"runs.in_memory", "configure in-memory", - C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_RUNS_IN_MEMORY}, + {"runs.in_memory", "configure in-memory", C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_RUNS_IN_MEMORY}, - {"runs.ops", "operations per run", - 0x0, 0, M(2), M(100), V_GLOBAL_RUNS_OPS}, + {"runs.ops", "operations per run", 0x0, 0, M(2), M(100), V_GLOBAL_RUNS_OPS}, - {"runs.rows", "number of rows", - C_TABLE, 10, M(1), M(100), V_TABLE_RUNS_ROWS}, + {"runs.rows", "number of rows", C_TABLE, 10, M(1), M(100), V_TABLE_RUNS_ROWS}, - {"runs.source", "data source type (file | lsm | table)", - C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_RUNS_SOURCE}, + {"runs.source", "data source type (file | lsm | table)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, + V_TABLE_RUNS_SOURCE}, - {"runs.tables", "number of tables", - 0x0, 1, 32, V_MAX_TABLES_CONFIG, V_GLOBAL_RUNS_TABLES}, + {"runs.tables", "number of tables", 0x0, 1, 32, V_MAX_TABLES_CONFIG, V_GLOBAL_RUNS_TABLES}, - {"runs.threads", "number of worker threads", - 0x0, 1, 32, 128, V_GLOBAL_RUNS_THREADS}, + {"runs.threads", "number of worker threads", 0x0, 1, 32, 128, V_GLOBAL_RUNS_THREADS}, - {"runs.timer", "run time (minutes)", - C_IGNORE, 0, 0, UINT_MAX, V_GLOBAL_RUNS_TIMER}, + {"runs.timer", "run time (minutes)", C_IGNORE, 0, 0, UINT_MAX, V_GLOBAL_RUNS_TIMER}, - {"runs.type", "object type (fix | row | var)", - C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_RUNS_TYPE}, + {"runs.type", "object type (fix | row | var)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, + V_TABLE_RUNS_TYPE}, - {"runs.verify_failure_dump", "configure page dump on repeatable read error", - C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP}, + {"runs.verify_failure_dump", "configure page dump on repeatable read error", C_BOOL | C_IGNORE, 0, + 0, 1, V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP}, - {"statistics", "configure statistics", - C_BOOL, 20, 0, 0, V_GLOBAL_STATISTICS}, + {"statistics", "configure statistics", C_BOOL, 20, 0, 0, V_GLOBAL_STATISTICS}, - {"statistics.server", "configure statistics server thread", - C_BOOL, 5, 0, 0, V_GLOBAL_STATISTICS_SERVER}, + {"statistics.server", "configure statistics server thread", C_BOOL, 5, 0, 0, + V_GLOBAL_STATISTICS_SERVER}, - {"stress.aggressive_sweep", "stress aggressive sweep", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_AGGRESSIVE_SWEEP}, + {"stress.aggressive_sweep", "stress aggressive sweep", C_BOOL, 2, 0, 0, + V_GLOBAL_STRESS_AGGRESSIVE_SWEEP}, - {"stress.checkpoint", "stress checkpoints", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT}, + {"stress.checkpoint", "stress checkpoints", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT}, {"stress.checkpoint_reserved_txnid_delay", "stress checkpoint invisible transaction id delay", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY}, - {"stress.checkpoint_prepare", "stress checkpoint prepare", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT_PREPARE}, - - {"stress.failpoint_eviction_fail_after_reconciliation", "stress failpoint eviction fail after reconciliation", - C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION}, + {"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0, + V_GLOBAL_STRESS_CHECKPOINT_PREPARE}, {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS}, - {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY}, - - {"stress.hs_search", "stress history store search", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SEARCH}, + {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0, + V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY}, - {"stress.hs_sweep", "stress history store sweep", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SWEEP}, + {"stress.hs_search", "stress history store search", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SEARCH}, - {"stress.prepare_resolution", "stress prepare resolution", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_PREPARE_RESOLUTION}, + {"stress.hs_sweep", "stress history store sweep", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SWEEP}, - {"stress.split_1", "stress splits (#1)", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_1}, + {"stress.split_1", "stress splits (#1)", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_1}, - {"stress.split_2", "stress splits (#2)", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_2}, + {"stress.split_2", "stress splits (#2)", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_2}, - {"stress.split_3", "stress splits (#3)", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_3}, + {"stress.split_3", "stress splits (#3)", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_3}, - {"stress.split_4", "stress splits (#4)", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_4}, + {"stress.split_4", "stress splits (#4)", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_4}, - {"stress.split_5", "stress splits (#5)", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_5}, + {"stress.split_5", "stress splits (#5)", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_5}, - {"stress.split_6", "stress splits (#6)", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_6}, + {"stress.split_6", "stress splits (#6)", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_6}, - {"stress.split_7", "stress splits (#7)", - C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_7}, + {"stress.split_7", "stress splits (#7)", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_7}, - {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", - 0, 0, 100, 100, V_GLOBAL_TRANSACTION_IMPLICIT}, + {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", 0, 0, 100, + 100, V_GLOBAL_TRANSACTION_IMPLICIT}, - {"transaction.timestamps", "all transactions (or none), have timestamps", - C_BOOL, 80, 0, 0, V_GLOBAL_TRANSACTION_TIMESTAMPS}, + {"transaction.timestamps", "all transactions (or none), have timestamps", C_BOOL, 80, 0, 0, + V_GLOBAL_TRANSACTION_TIMESTAMPS}, - {"wiredtiger.config", "wiredtiger_open API configuration string", - C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_WIREDTIGER_CONFIG}, + {"wiredtiger.config", "wiredtiger_open API configuration string", C_IGNORE | C_STRING, 0, 0, 0, + V_GLOBAL_WIREDTIGER_CONFIG}, - {"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", - C_BOOL, 80, 0, 0, V_GLOBAL_WIREDTIGER_RWLOCK}, + {"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", C_BOOL, 80, 0, 0, + V_GLOBAL_WIREDTIGER_RWLOCK}, - {"wiredtiger.leak_memory", "leak memory on wiredtiger shutdown", - C_BOOL, 0, 0, 0, V_GLOBAL_WIREDTIGER_LEAK_MEMORY}, + {"wiredtiger.leak_memory", "leak memory on wiredtiger shutdown", C_BOOL, 0, 0, 0, + V_GLOBAL_WIREDTIGER_LEAK_MEMORY}, - {NULL, NULL, 0x0, 0, 0, 0, 0} -}; + {NULL, NULL, 0x0, 0, 0, 0, 0}}; diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10017 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10017 deleted file mode 100644 index efec568184e..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10017 +++ /dev/null @@ -1,179 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -backup=0 -backup.incremental=off -backup.incr_granularity=856 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=0 -block_cache.size=47 -btree.huffman_value=0 -buffer_alignment=0 -cache=1452 -cache.evict_max=3 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=66 -checkpoint.wait=33 -debug.checkpoint_retention=10 -debug.cursor_reposition=0 -debug.eviction=0 -debug.log_retention=4 -debug.realloc_exact=0 -debug.realloc_malloc=0 -debug.slow_checkpoint=0 -debug.table_logging=0 -debug.update_restore_evict=0 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=1 -file_manager.close_handle_minimum=7 -file_manager.close_idle_time=29 -file_manager.close_scan_interval=21 -format.abort=0 -format.independent_thread_rng=0 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=63955 -logging.prealloc=1 -logging.remove=1 -ops.alter=0 -ops.compaction=0 -ops.hs_cursor=0 -ops.bound_cursor=0 -ops.prepare=1 -ops.random_cursor=1 -ops.salvage=0 -ops.verify=1 -quiet=1 -random.data_seed=4173770 -random.extra_seed=12599215 -runs.in_memory=0 -runs.ops=0 -runs.predictable_replay=0 -runs.rows=1800000 -runs.tables=3 -runs.threads=32 -runs.timer=6 -runs.verify_failure_dump=0 -statistics.mode=all -statistics_log.sources=off -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_prepare=0 -stress.evict_reposition=0 -stress.failpoint_eviction_fail_after_reconciliation=0 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.sleep_before_read_overflow_onpage=0 -stress.split_1=0 -stress.split_2=0 -stress.split_3=0 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -stress.split_8=0 -tiered_storage.flush_frequency=0 -tiered_storage.storage_source=off -transaction.implicit=0 -transaction.timestamps=1 -wiredtiger.config=off -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.dictionary=1 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=10 -table1.btree.leaf_page_max=12 -table1.btree.memory_page_max=8 -table1.btree.repeat_data_pct=42 -table1.btree.split_pct=50 -table1.btree.value_max=3971 -table1.btree.value_min=5 -table1.disk.checksum=on -table1.disk.firstfit=0 -table1.ops.pareto=0 -table1.ops.pareto.skew=48 -table1.ops.pct.delete=35 -table1.ops.pct.insert=9 -table1.ops.pct.modify=49 -table1.ops.pct.read=3 -table1.ops.pct.write=4 -table1.ops.truncate=1 -table1.runs.mirror=0 -table1.runs.source=table -table1.runs.type=variable-length column-store -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.compression=zstd -table2.btree.dictionary=1 -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=15 -table2.btree.key_max=109 -table2.btree.key_min=15 -table2.btree.leaf_page_max=9 -table2.btree.memory_page_max=8 -table2.btree.prefix_len=0 -table2.btree.prefix_compression=1 -table2.btree.prefix_compression_min=3 -table2.btree.reverse=0 -table2.btree.split_pct=95 -table2.btree.value_max=2650 -table2.btree.value_min=10 -table2.disk.checksum=on -table2.disk.firstfit=0 -table2.ops.pareto=1 -table2.ops.pareto.skew=56 -table2.ops.pct.delete=0 -table2.ops.pct.insert=1 -table2.ops.pct.modify=0 -table2.ops.pct.read=0 -table2.ops.pct.write=99 -table2.ops.truncate=1 -table2.runs.mirror=0 -table2.runs.source=table -table2.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.compression=snappy -table3.btree.dictionary=0 -table3.btree.internal_key_truncation=0 -table3.btree.internal_page_max=12 -table3.btree.key_max=33 -table3.btree.key_min=20 -table3.btree.leaf_page_max=9 -table3.btree.memory_page_max=6 -table3.btree.prefix_len=0 -table3.btree.prefix_compression=1 -table3.btree.prefix_compression_min=1 -table3.btree.reverse=0 -table3.btree.split_pct=87 -table3.btree.value_max=478 -table3.btree.value_min=9 -table3.disk.checksum=unencrypted -table3.disk.firstfit=0 -table3.ops.pareto=0 -table3.ops.pareto.skew=21 -table3.ops.pct.delete=63 -table3.ops.pct.insert=24 -table3.ops.pct.modify=0 -table3.ops.pct.read=5 -table3.ops.pct.write=8 -table3.ops.truncate=1 -table3.runs.mirror=0 -table3.runs.source=table -table3.runs.type=row-store diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10384 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10384 deleted file mode 100644 index 9ce716f3524..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10384 +++ /dev/null @@ -1,163 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -backup=1 -backup.incremental=block -backup.incr_granularity=16384 -block_cache=1 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=0 -block_cache.size=26 -btree.huffman_value=0 -buffer_alignment=0 -cache=128 -cache.evict_max=3 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=117 -checkpoint.wait=25 -debug.realloc_exact=0 -debug.realloc_malloc=0 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=0 -format.abort=0 -format.independent_thread_rng=1 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=393645 -logging.prealloc=1 -logging.remove=1 -ops.alter=0 -ops.compaction=0 -ops.hs_cursor=0 -ops.bound_cursor=0 -ops.prepare=0 -ops.random_cursor=0 -ops.salvage=0 -ops.verify=1 -quiet=1 -runs.in_memory=0 -runs.ops=0 -runs.rows=1000000 -runs.tables=3 -runs.threads=4 -runs.timer=11 -runs.verify_failure_dump=0 -statistics.mode=all -statistics_log.sources=off -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_prepare=0 -stress.evict_reposition=0 -stress.failpoint_eviction_fail_after_reconciliation=0 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.sleep_before_read_overflow_onpage=0 -stress.split_1=1 -stress.split_2=1 -stress.split_3=1 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=1 -transaction.implicit=0 -transaction.timestamps=1 -wiredtiger.config=off -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.compression=snappy -table1.btree.dictionary=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=15 -table1.btree.key_max=95 -table1.btree.key_min=18 -table1.btree.leaf_page_max=10 -table1.btree.memory_page_max=7 -table1.btree.prefix_len=0 -table1.btree.prefix_compression=0 -table1.btree.prefix_compression_min=1 -table1.btree.reverse=0 -table1.btree.split_pct=81 -table1.btree.value_max=3914 -table1.btree.value_min=4 -table1.disk.checksum=on -table1.disk.firstfit=0 -table1.ops.pct.delete=35 -table1.ops.pct.insert=9 -table1.ops.pct.modify=18 -table1.ops.pct.read=36 -table1.ops.pct.write=2 -table1.ops.truncate=1 -table1.runs.mirror=1 -table1.runs.source=table -table1.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.compression=zlib -table2.btree.dictionary=0 -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=9 -table2.btree.key_max=126 -table2.btree.key_min=23 -table2.btree.leaf_page_max=9 -table2.btree.memory_page_max=8 -table2.btree.prefix_len=0 -table2.btree.prefix_compression=1 -table2.btree.prefix_compression_min=3 -table2.btree.reverse=0 -table2.btree.split_pct=70 -table2.btree.value_max=1847 -table2.btree.value_min=6 -table2.disk.checksum=unencrypted -table2.disk.firstfit=0 -table2.ops.pct.delete=1 -table2.ops.pct.insert=3 -table2.ops.pct.modify=16 -table2.ops.pct.read=78 -table2.ops.pct.write=2 -table2.ops.truncate=1 -table2.runs.mirror=1 -table2.runs.source=table -table2.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.compression=zstd -table3.btree.dictionary=1 -table3.btree.internal_key_truncation=1 -table3.btree.internal_page_max=16 -table3.btree.key_max=112 -table3.btree.key_min=29 -table3.btree.leaf_page_max=15 -table3.btree.memory_page_max=1 -table3.btree.prefix_len=0 -table3.btree.prefix_compression=1 -table3.btree.prefix_compression_min=5 -table3.btree.reverse=0 -table3.btree.split_pct=63 -table3.btree.value_max=3771 -table3.btree.value_min=13 -table3.disk.checksum=unencrypted -table3.disk.firstfit=1 -table3.ops.pct.delete=2 -table3.ops.pct.insert=0 -table3.ops.pct.modify=9 -table3.ops.pct.read=21 -table3.ops.pct.write=68 -table3.ops.truncate=1 -table3.runs.mirror=1 -table3.runs.source=table -table3.runs.type=row-store diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10961 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10961 deleted file mode 100644 index 254c16c392e..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-10961 +++ /dev/null @@ -1,166 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -backup=0 -backup.incremental=off -backup.incr_granularity=1487 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=1 -block_cache.size=75 -btree.huffman_value=0 -buffer_alignment=0 -cache=1320 -cache.evict_max=5 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=155 -checkpoint.wait=39 -debug.checkpoint_retention=2 -debug.cursor_reposition=0 -debug.eviction=0 -debug.log_retention=8 -debug.realloc_exact=0 -debug.realloc_malloc=0 -debug.slow_checkpoint=0 -debug.table_logging=0 -debug.update_restore_evict=0 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=off -disk.mmap=1 -disk.mmap_all=0 -file_manager.close_handle_minimum=18 -file_manager.close_idle_time=1 -file_manager.close_scan_interval=2 -format.abort=0 -format.independent_thread_rng=1 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=226874 -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -backup=0 -backup.incremental=off -backup.incr_granularity=1487 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=1 -block_cache.size=75 -btree.huffman_value=0 -buffer_alignment=0 -cache=1320 -cache.evict_max=5 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=155 -checkpoint.wait=39 -debug.checkpoint_retention=2 -debug.cursor_reposition=0 -debug.eviction=0 -debug.log_retention=8 -debug.realloc_exact=0 -debug.realloc_malloc=0 -debug.slow_checkpoint=0 -debug.table_logging=0 -debug.update_restore_evict=0 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=off -disk.mmap=1 -disk.mmap_all=0 -file_manager.close_handle_minimum=18 -file_manager.close_idle_time=1 -file_manager.close_scan_interval=2 -format.abort=0 -format.independent_thread_rng=1 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=226874 -logging.prealloc=1 -logging.remove=1 -ops.alter=0 -ops.compaction=0 -ops.hs_cursor=1 -ops.bound_cursor=0 -ops.prepare=0 -ops.random_cursor=0 -ops.salvage=1 -ops.verify=1 -quiet=1 -random.data_seed=680556 -random.extra_seed=4131426 -runs.in_memory=0 -runs.ops=0 -runs.predictable_replay=0 -runs.rows=1000000 -runs.tables=1 -runs.threads=32 -runs.timer=6 -runs.verify_failure_dump=0 -statistics.mode=fast -statistics_log.sources=off -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_prepare=0 -stress.evict_reposition=0 -stress.failpoint_eviction_fail_after_reconciliation=0 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.sleep_before_read_overflow_onpage=0 -stress.split_1=0 -stress.split_2=0 -stress.split_3=0 -stress.split_4=1 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -stress.split_8=0 -tiered_storage.flush_frequency=0 -tiered_storage.storage_source=off -transaction.implicit=40 -transaction.timestamps=0 -wiredtiger.config=off -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.compression=none -table1.btree.dictionary=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=10 -table1.btree.key_max=75 -table1.btree.key_min=20 -table1.btree.leaf_page_max=11 -table1.btree.memory_page_max=1 -table1.btree.prefix_len=0 -table1.btree.prefix_compression=0 -table1.btree.prefix_compression_min=7 -table1.btree.reverse=0 -table1.btree.split_pct=66 -table1.btree.value_max=1418 -table1.btree.value_min=19 -table1.disk.checksum=on -table1.disk.firstfit=0 -table1.ops.pareto=1 -table1.ops.pareto.skew=61 -table1.ops.pct.delete=96 -table1.ops.pct.insert=0 -table1.ops.pct.modify=1 -table1.ops.pct.read=1 -table1.ops.pct.write=2 -table1.ops.truncate=1 -table1.runs.mirror=0 -table1.runs.source=table -table1.runs.type=row-store diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11060 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11060 deleted file mode 100644 index 852f2f6507d..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11060 +++ /dev/null @@ -1,188 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -backup=0 -backup.incremental=off -backup.incr_granularity=2981 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=0 -block_cache.size=27 -btree.huffman_value=0 -buffer_alignment=0 -cache=120 -cache.evict_max=4 -cache.eviction_dirty_target=0 -cache.eviction_dirty_trigger=0 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=170 -checkpoint.wait=25 -debug.checkpoint_retention=10 -debug.cursor_reposition=0 -debug.eviction=0 -debug.log_retention=6 -debug.realloc_exact=0 -debug.realloc_malloc=0 -debug.slow_checkpoint=0 -debug.table_logging=0 -debug.update_restore_evict=0 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=0 -file_manager.close_handle_minimum=4 -file_manager.close_idle_time=9 -file_manager.close_scan_interval=22 -format.abort=0 -format.independent_thread_rng=1 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=211679 -logging.prealloc=0 -logging.remove=1 -ops.alter=0 -ops.compaction=0 -ops.hs_cursor=0 -ops.bound_cursor=0 -ops.prepare=1 -ops.random_cursor=1 -ops.salvage=0 -ops.verify=1 -quiet=1 -random.data_seed=16418622 -random.extra_seed=10025607 -runs.in_memory=0 -runs.ops=50000 -runs.predictable_replay=0 -runs.rows=10000 -runs.tables=3 -runs.threads=4 -runs.timer=6 -runs.verify_failure_dump=0 -statistics.mode=fast -statistics_log.sources=off -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_prepare=0 -stress.evict_reposition=0 -stress.failpoint_eviction_fail_after_reconciliation=0 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.prepare_resolution=0 -stress.sleep_before_read_overflow_onpage=0 -stress.split_1=0 -stress.split_2=0 -stress.split_3=0 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -stress.split_8=0 -tiered_storage.flush_frequency=0 -tiered_storage.storage_source=off -transaction.implicit=0 -transaction.timestamps=1 -wiredtiger.config=off -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.compression=none -table1.btree.dictionary=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=11 -table1.btree.key_max=62 -table1.btree.key_min=14 -table1.btree.leaf_page_max=14 -table1.btree.memory_page_max=3 -table1.btree.prefix_len=0 -table1.btree.prefix_compression=1 -table1.btree.prefix_compression_min=7 -table1.btree.reverse=0 -table1.btree.split_pct=88 -table1.btree.value_max=3339 -table1.btree.value_min=15 -table1.disk.checksum=unencrypted -table1.disk.firstfit=0 -table1.ops.pareto=1 -table1.ops.pareto.skew=74 -table1.ops.pct.delete=18 -table1.ops.pct.insert=21 -table1.ops.pct.modify=15 -table1.ops.pct.read=21 -table1.ops.pct.write=25 -table1.ops.truncate=1 -table1.runs.mirror=1 -table1.runs.source=table -table1.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.compression=none -table2.btree.dictionary=0 -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=11 -table2.btree.key_max=82 -table2.btree.key_min=21 -table2.btree.leaf_page_max=16 -table2.btree.memory_page_max=8 -table2.btree.prefix_len=0 -table2.btree.prefix_compression=1 -table2.btree.prefix_compression_min=3 -table2.btree.reverse=0 -table2.btree.split_pct=81 -table2.btree.value_max=179 -table2.btree.value_min=19 -table2.disk.checksum=on -table2.disk.firstfit=0 -table2.ops.pareto=0 -table2.ops.pareto.skew=94 -table2.ops.pct.delete=14 -table2.ops.pct.insert=75 -table2.ops.pct.modify=2 -table2.ops.pct.read=8 -table2.ops.pct.write=1 -table2.ops.truncate=1 -table2.runs.mirror=1 -table2.runs.source=table -table2.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.compression=none -table3.btree.dictionary=1 -table3.btree.internal_key_truncation=1 -table3.btree.internal_page_max=10 -table3.btree.key_max=112 -table3.btree.key_min=21 -table3.btree.leaf_page_max=13 -table3.btree.memory_page_max=1 -table3.btree.prefix_len=0 -table3.btree.prefix_compression=1 -table3.btree.prefix_compression_min=1 -table3.btree.reverse=0 -table3.btree.split_pct=75 -table3.btree.value_max=2733 -table3.btree.value_min=16 -table3.disk.checksum=on -table3.disk.firstfit=1 -table3.ops.pareto=0 -table3.ops.pareto.skew=20 -table3.ops.pct.delete=13 -table3.ops.pct.insert=22 -table3.ops.pct.modify=14 -table3.ops.pct.read=47 -table3.ops.pct.write=4 -table3.ops.truncate=1 -table3.runs.mirror=0 -table3.runs.source=table -table3.runs.type=row-store diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11064 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11064 deleted file mode 100644 index 7a9a8d3e7ce..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11064 +++ /dev/null @@ -1,229 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -backup=0 -backup.incremental=off -backup.incr_granularity=13161 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=1 -block_cache.size=84 -btree.huffman_value=0 -buffer_alignment=0 -cache=880 -cache.evict_max=2 -cache.eviction_dirty_target=0 -cache.eviction_dirty_trigger=0 -cache.minimum=20 -checkpoint=off -checkpoint.log_size=74 -checkpoint.wait=35 -debug.checkpoint_retention=8 -debug.cursor_reposition=0 -debug.eviction=0 -debug.log_retention=5 -debug.realloc_exact=0 -debug.realloc_malloc=0 -debug.slow_checkpoint=0 -debug.table_logging=0 -debug.update_restore_evict=0 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=0 -file_manager.close_handle_minimum=24 -file_manager.close_idle_time=11 -file_manager.close_scan_interval=16 -format.abort=0 -format.independent_thread_rng=0 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=353587 -logging.prealloc=0 -logging.remove=0 -ops.alter=0 -ops.compaction=0 -ops.hs_cursor=1 -ops.bound_cursor=0 -ops.prepare=1 -ops.random_cursor=0 -ops.salvage=0 -ops.verify=1 -quiet=1 -random.data_seed=4492643 -random.extra_seed=8537229 -runs.in_memory=0 -runs.ops=0 -runs.predictable_replay=0 -runs.rows=1000000 -runs.tables=5 -runs.threads=10 -runs.timer=16 -runs.verify_failure_dump=0 -statistics.mode=fast -statistics_log.sources=off -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_prepare=0 -stress.evict_reposition=0 -stress.failpoint_eviction_fail_after_reconciliation=0 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=1 -stress.hs_sweep=0 -stress.prepare_resolution=0 -stress.sleep_before_read_overflow_onpage=0 -stress.split_1=0 -stress.split_2=0 -stress.split_3=0 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -stress.split_8=0 -tiered_storage.flush_frequency=0 -tiered_storage.storage_source=off -transaction.implicit=0 -transaction.timestamps=1 -wiredtiger.config= -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.compression=zstd -table1.btree.dictionary=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=17 -table1.btree.key_max=52 -table1.btree.key_min=20 -table1.btree.leaf_page_max=13 -table1.btree.memory_page_max=7 -table1.btree.prefix_len=0 -table1.btree.prefix_compression=1 -table1.btree.prefix_compression_min=3 -table1.btree.reverse=0 -table1.btree.split_pct=97 -table1.btree.value_max=1389 -table1.btree.value_min=7 -table1.disk.checksum=on -table1.disk.firstfit=1 -table1.ops.pareto=0 -table1.ops.pareto.skew=91 -table1.ops.pct.delete=0 -table1.ops.pct.insert=35 -table1.ops.pct.modify=0 -table1.ops.pct.read=65 -table1.ops.pct.write=0 -table1.ops.truncate=0 -table1.runs.mirror=0 -table1.runs.source=table -table1.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.bitcnt=4 -table2.btree.compression=none -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=9 -table2.btree.leaf_page_max=13 -table2.btree.memory_page_max=10 -table2.btree.split_pct=76 -table2.disk.checksum=unencrypted -table2.disk.firstfit=0 -table2.ops.pareto=1 -table2.ops.pareto.skew=74 -table2.ops.pct.delete=36 -table2.ops.pct.insert=48 -table2.ops.pct.modify=16 -table2.ops.pct.read=0 -table2.ops.pct.write=0 -table2.ops.truncate=0 -table2.runs.mirror=0 -table2.runs.source=table -table2.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.bitcnt=8 -table3.btree.compression=zstd -table3.btree.internal_key_truncation=1 -table3.btree.internal_page_max=17 -table3.btree.leaf_page_max=9 -table3.btree.memory_page_max=9 -table3.btree.split_pct=74 -table3.disk.checksum=off -table3.disk.firstfit=0 -table3.ops.pareto=0 -table3.ops.pareto.skew=28 -table3.ops.pct.delete=14 -table3.ops.pct.insert=14 -table3.ops.pct.modify=2 -table3.ops.pct.read=68 -table3.ops.pct.write=2 -table3.ops.truncate=0 -table3.runs.mirror=0 -table3.runs.source=file -table3.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 4 -############################################ -table4.btree.compression=none -table4.btree.dictionary=0 -table4.btree.internal_key_truncation=1 -table4.btree.internal_page_max=12 -table4.btree.leaf_page_max=11 -table4.btree.memory_page_max=8 -table4.btree.repeat_data_pct=39 -table4.btree.split_pct=57 -table4.btree.value_max=1453 -table4.btree.value_min=19 -table4.disk.checksum=uncompressed -table4.disk.firstfit=0 -table4.ops.pareto=0 -table4.ops.pareto.skew=47 -table4.ops.pct.delete=11 -table4.ops.pct.insert=12 -table4.ops.pct.modify=74 -table4.ops.pct.read=0 -table4.ops.pct.write=3 -table4.ops.truncate=0 -table4.runs.mirror=0 -table4.runs.source=file -table4.runs.type=variable-length column-store -############################################ -# TABLE PARAMETERS: table 5 -############################################ -table5.btree.compression=none -table5.btree.dictionary=0 -table5.btree.internal_key_truncation=1 -table5.btree.internal_page_max=11 -table5.btree.key_max=114 -table5.btree.key_min=13 -table5.btree.leaf_page_max=14 -table5.btree.memory_page_max=6 -table5.btree.prefix_len=0 -table5.btree.prefix_compression=1 -table5.btree.prefix_compression_min=5 -table5.btree.reverse=0 -table5.btree.split_pct=86 -table5.btree.value_max=805 -table5.btree.value_min=14 -table5.disk.checksum=on -table5.disk.firstfit=0 -table5.ops.pareto=0 -table5.ops.pareto.skew=84 -table5.ops.pct.delete=1 -table5.ops.pct.insert=2 -table5.ops.pct.modify=6 -table5.ops.pct.read=84 -table5.ops.pct.write=7 -table5.ops.truncate=0 -table5.runs.mirror=0 -table5.runs.source=table -table5.runs.type=row-store diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11121 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11121 deleted file mode 100644 index ed46a06d435..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11121 +++ /dev/null @@ -1,206 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=1 -assert.write_timestamp=0 -backup=0 -backup.incremental=off -backup.incr_granularity=5027 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=1 -block_cache.size=8 -btree.huffman_value=0 -cache=380 -cache.evict_max=2 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=67 -checkpoint.wait=58 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=0 -format.abort=0 -format.independent_thread_rng=1 -format.major_timeout=0 -import=0 -logging=1 -logging.compression=zlib -logging.file_max=354713 -logging.prealloc=0 -logging.remove=1 -ops.alter=0 -ops.compaction=0 -ops.hs_cursor=0 -ops.prepare=0 -ops.random_cursor=0 -ops.salvage=1 -ops.verify=1 -quiet=1 -runs.in_memory=0 -runs.ops=0 -runs.rows=5000000 -runs.tables=5 -runs.threads=10 -runs.timer=6 -runs.type=variable-length column-store -runs.verify_failure_dump=0 -statistics=0 -statistics.server=0 -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_reserved_txnid_delay=0 -stress.checkpoint_prepare=0 -stress.failpoint_eviction_fail_after_reconciliation=1 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.split_1=0 -stress.split_2=0 -stress.split_3=0 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -transaction.implicit=53 -transaction.timestamps=0 -wiredtiger.config=off -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.compression=none -table1.btree.dictionary=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=17 -table1.btree.key_max=103 -table1.btree.key_min=31 -table1.btree.leaf_page_max=15 -table1.btree.memory_page_max=1 -table1.btree.prefix_len=0 -table1.btree.prefix_compression=1 -table1.btree.prefix_compression_min=8 -table1.btree.reverse=0 -table1.btree.split_pct=51 -table1.btree.value_max=1055 -table1.btree.value_min=9 -table1.disk.checksum=on -table1.disk.firstfit=0 -table1.ops.pct.delete=39 -table1.ops.pct.insert=0 -table1.ops.pct.modify=54 -table1.ops.pct.read=0 -table1.ops.pct.write=7 -table1.ops.truncate=1 -table1.runs.source=file -table1.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.compression=none -table2.btree.dictionary=1 -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=10 -table2.btree.key_max=77 -table2.btree.key_min=32 -table2.btree.leaf_page_max=15 -table2.btree.memory_page_max=1 -table2.btree.prefix_len=0 -table2.btree.prefix_compression=1 -table2.btree.prefix_compression_min=5 -table2.btree.reverse=0 -table2.btree.split_pct=92 -table2.btree.value_max=2647 -table2.btree.value_min=10 -table2.disk.checksum=unencrypted -table2.disk.firstfit=0 -table2.ops.pct.delete=26 -table2.ops.pct.insert=11 -table2.ops.pct.modify=3 -table2.ops.pct.read=28 -table2.ops.pct.write=32 -table2.ops.truncate=1 -table2.runs.source=table -table2.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.compression=snappy -table3.btree.dictionary=0 -table3.btree.internal_key_truncation=1 -table3.btree.internal_page_max=12 -table3.btree.key_max=84 -table3.btree.key_min=21 -table3.btree.leaf_page_max=10 -table3.btree.memory_page_max=1 -table3.btree.prefix_len=0 -table3.btree.prefix_compression=0 -table3.btree.prefix_compression_min=8 -table3.btree.reverse=0 -table3.btree.split_pct=69 -table3.btree.value_max=413 -table3.btree.value_min=19 -table3.disk.checksum=on -table3.disk.firstfit=0 -table3.ops.pct.delete=0 -table3.ops.pct.insert=65 -table3.ops.pct.modify=0 -table3.ops.pct.read=34 -table3.ops.pct.write=1 -table3.ops.truncate=1 -table3.runs.source=table -table3.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 4 -############################################ -table4.btree.compression=snappy -table4.btree.dictionary=0 -table4.btree.internal_key_truncation=1 -table4.btree.internal_page_max=10 -table4.btree.leaf_page_max=10 -table4.btree.memory_page_max=10 -table4.btree.repeat_data_pct=20 -table4.btree.split_pct=68 -table4.btree.value_max=52 -table4.btree.value_min=12 -table4.disk.checksum=unencrypted -table4.disk.firstfit=0 -table4.ops.pct.delete=0 -table4.ops.pct.insert=50 -table4.ops.pct.modify=0 -table4.ops.pct.read=0 -table4.ops.pct.write=50 -table4.ops.truncate=1 -table4.runs.source=table -############################################ -# TABLE PARAMETERS: table 5 -############################################ -table5.btree.compression=none -table5.btree.dictionary=0 -table5.btree.internal_key_truncation=1 -table5.btree.internal_page_max=11 -table5.btree.key_max=127 -table5.btree.key_min=20 -table5.btree.leaf_page_max=9 -table5.btree.memory_page_max=6 -table5.btree.prefix_len=0 -table5.btree.prefix_compression=0 -table5.btree.prefix_compression_min=7 -table5.btree.reverse=0 -table5.btree.split_pct=85 -table5.btree.value_max=4094 -table5.btree.value_min=12 -table5.disk.checksum=unencrypted -table5.disk.firstfit=1 -table5.ops.pct.delete=93 -table5.ops.pct.insert=4 -table5.ops.pct.modify=1 -table5.ops.pct.read=2 -table5.ops.pct.write=0 -table5.ops.truncate=1 -table5.runs.source=table -table5.runs.type=row-store diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11845 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11845 deleted file mode 100644 index 7fa922bb0b7..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-11845 +++ /dev/null @@ -1,289 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -background_compact=0 -background_compact.free_space_target=11 -backup=0 -backup.incremental=off -backup.incr_granularity=512 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=0 -block_cache.size=7 -btree.huffman_value=0 -buffer_alignment=0 -cache=2310 -cache.evict_max=4 -cache.eviction_dirty_target=0 -cache.eviction_dirty_trigger=0 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=103 -checkpoint.wait=85 -chunk_cache=0 -chunk_cache.capacity=265 -chunk_cache.chunk_size=39 -chunk_cache.storage_path=off -chunk_cache.type=off -compact.free_space_target=72 -debug.checkpoint_retention=7 -debug.cursor_reposition=0 -debug.eviction=0 -debug.log_retention=0 -debug.realloc_exact=0 -debug.realloc_malloc=0 -debug.slow_checkpoint=0 -debug.table_logging=0 -debug.update_restore_evict=0 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=0 -file_manager.close_handle_minimum=29 -file_manager.close_idle_time=5 -file_manager.close_scan_interval=27 -format.abort=0 -format.independent_thread_rng=1 -format.major_timeout=0 -import=0 -logging=1 -logging.compression=none -logging.file_max=271721 -logging.prealloc=0 -logging.remove=1 -ops.alter=1 -ops.compaction=0 -ops.hs_cursor=1 -ops.bound_cursor=0 -ops.prepare=0 -ops.random_cursor=0 -ops.salvage=0 -ops.throttle=0 -ops.throttle.sleep_us=286067 -ops.verify=1 -quiet=1 -random.data_seed=1478940 -random.extra_seed=15668881 -runs.in_memory=0 -runs.ops=0 -runs.predictable_replay=0 -runs.rows=1000000 -runs.tables=7 -runs.threads=32 -runs.timer=11 -runs.verify_failure_dump=0 -statistics.mode=fast -statistics_log.sources=off -stress.aggressive_stash_free=0 -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_prepare=0 -stress.compact_slow=0 -stress.evict_reposition=0 -stress.failpoint_eviction_split=0 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.prepare_resolution_1=0 -stress.sleep_before_read_overflow_onpage=0 -stress.split_1=0 -stress.split_2=0 -stress.split_3=0 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -stress.split_8=0 -tiered_storage.flush_frequency=0 -tiered_storage.storage_source=off -transaction.implicit=0 -transaction.operation_timeout_ms=2000 -transaction.timestamps=0 -wiredtiger.config=off -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.compression=none -table1.btree.dictionary=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=13 -table1.btree.key_max=20 -table1.btree.key_min=18 -table1.btree.leaf_page_max=11 -table1.btree.memory_page_max=3 -table1.btree.prefix_len=0 -table1.btree.prefix_compression=1 -table1.btree.prefix_compression_min=2 -table1.btree.reverse=0 -table1.btree.split_pct=99 -table1.btree.value_max=2392 -table1.btree.value_min=9 -table1.disk.checksum=unencrypted -table1.disk.firstfit=0 -table1.ops.pareto=0 -table1.ops.pareto.skew=8 -table1.ops.pct.delete=16 -table1.ops.pct.insert=10 -table1.ops.pct.modify=5 -table1.ops.pct.read=28 -table1.ops.pct.write=41 -table1.ops.truncate=1 -table1.runs.mirror=1 -table1.runs.source=table -table1.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.compression=none -table2.btree.dictionary=0 -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=15 -table2.btree.leaf_page_max=17 -table2.btree.memory_page_max=9 -table2.btree.split_pct=78 -table2.btree.value_max=469 -table2.btree.value_min=13 -table2.disk.checksum=unencrypted -table2.disk.firstfit=0 -table2.ops.pareto=0 -table2.ops.pareto.skew=93 -table2.ops.pct.delete=25 -table2.ops.pct.insert=10 -table2.ops.pct.modify=1 -table2.ops.pct.read=57 -table2.ops.pct.write=7 -table2.ops.truncate=1 -table2.runs.mirror=1 -table2.runs.source=table -table2.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.compression=none -table3.btree.dictionary=0 -table3.btree.internal_key_truncation=1 -table3.btree.internal_page_max=15 -table3.btree.leaf_page_max=17 -table3.btree.memory_page_max=9 -table3.btree.split_pct=78 -table3.btree.value_max=469 -table3.btree.value_min=13 -table3.disk.checksum=unencrypted -table3.disk.firstfit=0 -table3.ops.pareto=0 -table3.ops.pareto.skew=93 -table3.ops.pct.delete=25 -table3.ops.pct.insert=10 -table3.ops.pct.modify=1 -table3.ops.pct.read=57 -table3.ops.pct.write=7 -table3.ops.truncate=1 -table3.runs.mirror=1 -table3.runs.source=table -table3.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 4 -############################################ -table4.btree.compression=none -table4.btree.dictionary=0 -table4.btree.internal_key_truncation=1 -table4.btree.internal_page_max=15 -table4.btree.leaf_page_max=17 -table4.btree.memory_page_max=9 -table4.btree.split_pct=78 -table4.btree.value_max=469 -table4.btree.value_min=13 -table4.disk.checksum=unencrypted -table4.disk.firstfit=0 -table4.ops.pareto=0 -table4.ops.pareto.skew=93 -table4.ops.pct.delete=25 -table4.ops.pct.insert=10 -table4.ops.pct.modify=1 -table4.ops.pct.read=57 -table4.ops.pct.write=7 -table4.ops.truncate=1 -table4.runs.mirror=1 -table4.runs.source=table -table4.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 5 -############################################ -table5.btree.compression=none -table5.btree.dictionary=0 -table5.btree.internal_key_truncation=1 -table5.btree.internal_page_max=15 -table5.btree.leaf_page_max=17 -table5.btree.memory_page_max=9 -table5.btree.split_pct=78 -table5.btree.value_max=469 -table5.btree.value_min=13 -table5.disk.checksum=unencrypted -table5.disk.firstfit=0 -table5.ops.pareto=0 -table5.ops.pareto.skew=93 -table5.ops.pct.delete=25 -table5.ops.pct.insert=10 -table5.ops.pct.modify=1 -table5.ops.pct.read=57 -table5.ops.pct.write=7 -table5.ops.truncate=1 -table5.runs.mirror=1 -table5.runs.source=table -table5.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 6 -############################################ -table6.btree.compression=none -table6.btree.dictionary=0 -table6.btree.internal_key_truncation=1 -table6.btree.internal_page_max=15 -table6.btree.leaf_page_max=17 -table6.btree.memory_page_max=9 -table6.btree.split_pct=78 -table6.btree.value_max=469 -table6.btree.value_min=13 -table6.disk.checksum=unencrypted -table6.disk.firstfit=0 -table6.ops.pareto=0 -table6.ops.pareto.skew=93 -table6.ops.pct.delete=25 -table6.ops.pct.insert=10 -table6.ops.pct.modify=1 -table6.ops.pct.read=57 -table6.ops.pct.write=7 -table6.ops.truncate=1 -table6.runs.mirror=1 -table6.runs.source=table -table6.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 7 -############################################ -table7.btree.compression=none -table7.btree.dictionary=0 -table7.btree.internal_key_truncation=1 -table7.btree.internal_page_max=15 -table7.btree.leaf_page_max=17 -table7.btree.memory_page_max=9 -table7.btree.split_pct=78 -table7.btree.value_max=469 -table7.btree.value_min=13 -table7.disk.checksum=unencrypted -table7.disk.firstfit=0 -table7.ops.pareto=0 -table7.ops.pareto.skew=93 -table7.ops.pct.delete=25 -table7.ops.pct.insert=10 -table7.ops.pct.modify=1 -table7.ops.pct.read=57 -table7.ops.pct.write=7 -table7.ops.truncate=1 -table7.runs.mirror=1 -table7.runs.source=table -table7.runs.type=row-store diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-9751 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-9751 deleted file mode 100644 index 8ff2a91c49d..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-9751 +++ /dev/null @@ -1,138 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -assert.write_timestamp=0 -backup=0 -backup.incremental=off -backup.incr_granularity=8115 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=0 -block_cache.size=53 -btree.compression=none -cache=336 -cache.evict_max=2 -cache.minimum=40 -checkpoint=on -checkpoint.log_size=79 -checkpoint.wait=40 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=0 -format.abort=0 -format.independent_thread_rng=0 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=481991 -logging.prealloc=0 -logging.remove=1 -ops.alter=0 -ops.compaction=0 -ops.hs_cursor=0 -ops.prepare=0 -ops.random_cursor=1 -ops.salvage=0 -ops.verify=1 -quiet=1 -runs.in_memory=0 -runs.ops=0 -runs.rows=100000 -runs.source=table -runs.tables=3 -runs.threads=6 -runs.timer=1 -runs.type=variable-length column-store -runs.verify_failure_dump=0 -statistics=0 -statistics.server=1 -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_reserved_txnid_delay=0 -stress.checkpoint_prepare=0 -stress.evict_reposition=0 -stress.failpoint_eviction_fail_after_reconciliation=1 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.split_1=0 -stress.split_2=0 -stress.split_3=0 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -transaction.implicit=0 -transaction.timestamps=1 -wiredtiger.config=off -wiredtiger.rwlock=0 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.dictionary=1 -table1.btree.huffman_value=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=17 -table1.btree.leaf_page_max=9 -table1.btree.memory_page_max=8 -table1.btree.repeat_data_pct=87 -table1.btree.split_pct=60 -table1.btree.value_max=1299 -table1.btree.value_min=12 -table1.disk.checksum=on -table1.disk.firstfit=0 -table1.ops.pct.delete=77 -table1.ops.pct.insert=4 -table1.ops.pct.modify=12 -table1.ops.pct.read=1 -table1.ops.pct.write=6 -table1.ops.truncate=1 -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.dictionary=0 -table2.btree.huffman_value=0 -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=14 -table2.btree.leaf_page_max=11 -table2.btree.memory_page_max=7 -table2.btree.repeat_data_pct=89 -table2.btree.split_pct=67 -table2.btree.value_max=3716 -table2.btree.value_min=11 -table2.disk.checksum=unencrypted -table2.disk.firstfit=0 -table2.ops.pct.delete=20 -table2.ops.pct.insert=1 -table2.ops.pct.modify=7 -table2.ops.pct.read=72 -table2.ops.pct.write=0 -table2.ops.truncate=1 -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.dictionary=0 -table3.btree.huffman_value=0 -table3.btree.internal_key_truncation=1 -table3.btree.internal_page_max=17 -table3.btree.leaf_page_max=12 -table3.btree.memory_page_max=9 -table3.btree.repeat_data_pct=12 -table3.btree.split_pct=54 -table3.btree.value_max=2929 -table3.btree.value_min=8 -table3.disk.checksum=on -table3.disk.firstfit=0 -table3.ops.pct.delete=34 -table3.ops.pct.insert=45 -table3.ops.pct.modify=1 -table3.ops.pct.read=0 -table3.ops.pct.write=20 -table3.ops.truncate=1 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-9792 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-9792 deleted file mode 100644 index 4889602e518..00000000000 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-9792 +++ /dev/null @@ -1,160 +0,0 @@ -############################################ -# RUN PARAMETERS: V3 -############################################ -assert.read_timestamp=0 -backup=1 -backup.incremental=off -backup.incr_granularity=11461 -block_cache=0 -block_cache.cache_on_checkpoint=0 -block_cache.cache_on_writes=0 -block_cache.size=47 -btree.huffman_value=0 -cache=612 -cache.evict_max=1 -cache.minimum=20 -checkpoint=on -checkpoint.log_size=167 -checkpoint.wait=32 -disk.data_extend=0 -disk.direct_io=0 -disk.encryption=none -disk.mmap=1 -disk.mmap_all=0 -format.abort=0 -format.independent_thread_rng=0 -format.major_timeout=0 -import=0 -logging=0 -logging.compression=none -logging.file_max=199668 -logging.prealloc=0 -logging.remove=0 -ops.alter=0 -ops.compaction=1 -ops.hs_cursor=1 -ops.prepare=0 -ops.random_cursor=0 -ops.salvage=0 -ops.verify=1 -quiet=1 -runs.in_memory=0 -runs.ops=0 -runs.rows=1000000 -runs.tables=3 -runs.threads=16 -runs.timer=6 -runs.verify_failure_dump=0 -statistics=0 -statistics.server=0 -stress.aggressive_sweep=0 -stress.checkpoint=0 -stress.checkpoint_evict_page=0 -stress.checkpoint_reserved_txnid_delay=0 -stress.checkpoint_prepare=0 -stress.evict_reposition=1 -stress.failpoint_eviction_fail_after_reconciliation=0 -stress.failpoint_hs_delete_key_from_ts=0 -stress.hs_checkpoint_delay=0 -stress.hs_search=0 -stress.hs_sweep=0 -stress.split_1=0 -stress.split_2=1 -stress.split_3=0 -stress.split_4=0 -stress.split_5=0 -stress.split_6=0 -stress.split_7=0 -transaction.implicit=0 -transaction.timestamps=1 -wiredtiger.config=off -wiredtiger.rwlock=1 -wiredtiger.leak_memory=0 -############################################ -# TABLE PARAMETERS: table 1 -############################################ -table1.btree.compression=snappy -table1.btree.dictionary=0 -table1.btree.internal_key_truncation=1 -table1.btree.internal_page_max=15 -table1.btree.key_max=24 -table1.btree.key_min=14 -table1.btree.leaf_page_max=11 -table1.btree.memory_page_max=8 -table1.btree.prefix_len=0 -table1.btree.prefix_compression=0 -table1.btree.prefix_compression_min=7 -table1.btree.reverse=0 -table1.btree.split_pct=94 -table1.btree.value_max=1513 -table1.btree.value_min=1 -table1.disk.checksum=unencrypted -table1.disk.firstfit=0 -table1.ops.pct.delete=3 -table1.ops.pct.insert=63 -table1.ops.pct.modify=3 -table1.ops.pct.read=9 -table1.ops.pct.write=22 -table1.ops.truncate=1 -table1.runs.mirror=0 -table1.runs.source=file -table1.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 2 -############################################ -table2.btree.compression=zlib -table2.btree.dictionary=0 -table2.btree.internal_key_truncation=1 -table2.btree.internal_page_max=12 -table2.btree.key_max=115 -table2.btree.key_min=22 -table2.btree.leaf_page_max=12 -table2.btree.memory_page_max=9 -table2.btree.prefix_len=0 -table2.btree.prefix_compression=1 -table2.btree.prefix_compression_min=7 -table2.btree.reverse=0 -table2.btree.split_pct=86 -table2.btree.value_max=37 -table2.btree.value_min=16 -table2.disk.checksum=on -table2.disk.firstfit=1 -table2.ops.pct.delete=12 -table2.ops.pct.insert=0 -table2.ops.pct.modify=2 -table2.ops.pct.read=60 -table2.ops.pct.write=26 -table2.ops.truncate=1 -table2.runs.mirror=0 -table2.runs.source=table -table2.runs.type=row-store -############################################ -# TABLE PARAMETERS: table 3 -############################################ -table3.btree.compression=zstd -table3.btree.dictionary=0 -table3.btree.internal_key_truncation=1 -table3.btree.internal_page_max=11 -table3.btree.key_max=68 -table3.btree.key_min=21 -table3.btree.leaf_page_max=13 -table3.btree.memory_page_max=1 -table3.btree.prefix_len=41 -table3.btree.prefix_compression=1 -table3.btree.prefix_compression_min=2 -table3.btree.reverse=0 -table3.btree.split_pct=86 -table3.btree.value_max=239 -table3.btree.value_min=9 -table3.disk.checksum=unencrypted -table3.disk.firstfit=0 -table3.ops.pct.delete=50 -table3.ops.pct.insert=2 -table3.ops.pct.modify=1 -table3.ops.pct.read=3 -table3.ops.pct.write=44 -table3.ops.truncate=1 -table3.runs.mirror=0 -table3.runs.source=table -table3.runs.type=row-store - diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index dd9e10ab89d..861ded0890a 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -367,28 +367,6 @@ report_failure() echo "$name: failure status reported" > $dir/$status } -# Wait for a process to die. Handle both child and non-child processes. -# $1 pid -# Return <exit code> of process if child or 127 if non-child -wait_for_process() -{ - pid=$1 - ret=127 - - if [ `pstree -p $$ | grep -w $pid | wc -l` -gt "0" ]; then - # Can still produce "wait: pid XXXX is not a child of this shell" due to process - # ending between the steps, can be safely ignored. - wait $pid - ret=$? - else - while [ -d "/proc/$pid/" ]; do - sleep 1 - done - fi - - return $ret -} - # Resolve/cleanup completed jobs. resolve() { @@ -421,28 +399,17 @@ resolve() } # Kill the process group to catch any child processes. - if [ `ps -eo ppid | grep -w $pid | wc -l` -gt "0" ]; then - kill -KILL -- -$pid - fi - # Kill the process. - kill -KILL $pid - wait_for_process $pid - msg "job in $dir killed" + kill -KILL -- -$pid + wait $pid # Remove jobs we killed, they count as neither success or failure. rm -rf $dir $log continue } - wait_for_process $pid + wait $pid eret=$? - # Check for Sanitizer failures, have to do this prior to success because both can be reported. - grep -E -i 'Sanitizer' $log > /dev/null && { - report_failure $dir - continue - } - # Remove successful jobs. grep 'successful run completed' $log > /dev/null && { rm -rf $dir $log @@ -501,7 +468,7 @@ resolve() continue } - # Check for the library abort message or an error from format. + # Check for the library abort message, or an error from format. grep -E \ 'aborting WiredTiger library|format alarm timed out|run FAILED' \ $log > /dev/null && { diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 00762181a44..dc91c759bf1 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -147,61 +147,40 @@ static WT_EVENT_HANDLER event_handler = { * Configure stressing settings. */ static void -configure_timing_stress(char **p, size_t max) +configure_timing_stress(char *p, size_t max) { - CONFIG_APPEND(*p, ",timing_stress_for_test=["); + CONFIG_APPEND(p, ",timing_stress_for_test=["); if (GV(STRESS_AGGRESSIVE_SWEEP)) - CONFIG_APPEND(*p, ",aggressive_sweep"); + CONFIG_APPEND(p, ",aggressive_sweep"); if (GV(STRESS_CHECKPOINT)) - CONFIG_APPEND(*p, ",checkpoint_slow"); + CONFIG_APPEND(p, ",checkpoint_slow"); if (GV(STRESS_CHECKPOINT_PREPARE)) - CONFIG_APPEND(*p, ",prepare_checkpoint_delay"); + CONFIG_APPEND(p, ",prepare_checkpoint_delay"); if (GV(STRESS_CHECKPOINT_RESERVED_TXNID_DELAY)) - CONFIG_APPEND(*p, ",checkpoint_reserved_txnid_delay"); - if (GV(STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION)) - CONFIG_APPEND(*p, ",failpoint_eviction_fail_after_reconciliation"); + CONFIG_APPEND(p, ",checkpoint_reserved_txnid_delay"); if (GV(STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS)) - CONFIG_APPEND(*p, ",failpoint_history_store_delete_key_from_ts"); + CONFIG_APPEND(p, ",failpoint_history_store_delete_key_from_ts"); if (GV(STRESS_HS_CHECKPOINT_DELAY)) - CONFIG_APPEND(*p, ",history_store_checkpoint_delay"); + CONFIG_APPEND(p, ",history_store_checkpoint_delay"); if (GV(STRESS_HS_SEARCH)) - CONFIG_APPEND(*p, ",history_store_search"); + CONFIG_APPEND(p, ",history_store_search"); if (GV(STRESS_HS_SWEEP)) - CONFIG_APPEND(*p, ",history_store_sweep_race"); - if (GV(STRESS_PREPARE_RESOLUTION)) - CONFIG_APPEND(*p, ",prepare_resolution"); + CONFIG_APPEND(p, ",history_store_sweep_race"); if (GV(STRESS_SPLIT_1)) - CONFIG_APPEND(*p, ",split_1"); + CONFIG_APPEND(p, ",split_1"); if (GV(STRESS_SPLIT_2)) - CONFIG_APPEND(*p, ",split_2"); + CONFIG_APPEND(p, ",split_2"); if (GV(STRESS_SPLIT_3)) - CONFIG_APPEND(*p, ",split_3"); + CONFIG_APPEND(p, ",split_3"); if (GV(STRESS_SPLIT_4)) - CONFIG_APPEND(*p, ",split_4"); + CONFIG_APPEND(p, ",split_4"); if (GV(STRESS_SPLIT_5)) - CONFIG_APPEND(*p, ",split_5"); + CONFIG_APPEND(p, ",split_5"); if (GV(STRESS_SPLIT_6)) - CONFIG_APPEND(*p, ",split_6"); + CONFIG_APPEND(p, ",split_6"); if (GV(STRESS_SPLIT_7)) - CONFIG_APPEND(*p, ",split_7"); - CONFIG_APPEND(*p, "]"); -} - -/* - * configure_file_manager -- - * Configure file manager settings. - */ -static void -configure_file_manager(char **p, size_t max) -{ - CONFIG_APPEND(*p, ",file_manager=["); - if (GV(FILE_MANAGER_CLOSE_HANDLE_MINIMUM) != 0) - CONFIG_APPEND(*p, ",close_handle_minimum=%" PRIu32, GV(FILE_MANAGER_CLOSE_HANDLE_MINIMUM)); - if (GV(FILE_MANAGER_CLOSE_IDLE_TIME) != 0) - CONFIG_APPEND(*p, ",close_idle_time=%" PRIu32, GV(FILE_MANAGER_CLOSE_IDLE_TIME)); - if (GV(FILE_MANAGER_CLOSE_SCAN_INTERVAL) != 0) - CONFIG_APPEND(*p, ",close_scan_interval=%" PRIu32, GV(FILE_MANAGER_CLOSE_SCAN_INTERVAL)); - CONFIG_APPEND(*p, "]"); + CONFIG_APPEND(p, ",split_7"); + CONFIG_APPEND(p, "]"); } /* @@ -296,10 +275,7 @@ create_database(const char *home, WT_CONNECTION **connp) CONFIG_APPEND(p, ",statistics=(%s)", GV(STATISTICS) ? "fast" : "none"); /* Optional timing stress. */ - configure_timing_stress(&p, max); - - /* Optional file manager. */ - configure_file_manager(&p, max); + configure_timing_stress(p, max); /* Extensions. */ CONFIG_APPEND(p, ",extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", @@ -499,10 +475,7 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al CONFIG_APPEND(p, ",error_prefix=\"%s\"", progname); /* Optional timing stress. */ - configure_timing_stress(&p, max); - - /* Optional file manager. */ - configure_file_manager(&p, max); + configure_timing_stress(p, max); /* If in-memory, there's only a single, shared WT_CONNECTION handle. */ if (GV(RUNS_IN_MEMORY) != 0) diff --git a/src/third_party/wiredtiger/test/packing/intpack-test3.c b/src/third_party/wiredtiger/test/packing/intpack-test3.c index 6e6ceec158e..4c168f8a83e 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test3.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test3.c @@ -72,7 +72,7 @@ test_value(int64_t val) ", got %" WT_SIZET_FMT "\n", sinput, used_len, cp > p ? used_len + (size_t)(cp - p) : /* More than buf used */ - used_len - (size_t)(p - cp)); /* Less than buf used */ + used_len - (size_t)(p - cp)); /* Less than buf used */ abort(); } diff --git a/src/third_party/wiredtiger/test/suite/test_app_thread_evict01.py b/src/third_party/wiredtiger/test/suite/test_app_thread_evict01.py deleted file mode 100644 index cf0c573bf49..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_app_thread_evict01.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios - -# test_app_thread_evict01.py -# Test to trigger application threads to perform eviction. -class test_app_thread_evict01(wttest.WiredTigerTestCase): - uri = "table:test_app_thread_evict001" - format_values = [ - ('row_integer', dict(key_format='i', value_format='S')), - ] - - # 100MB cache, 52MB trigger, 50MB target - conn_config = "cache_size=100MB,statistics=(all),statistics_log=(wait=1,json=true,on_close=true)," \ - "eviction=(threads_max=1)," \ - "eviction_updates_trigger=52,eviction_dirty_trigger=52,eviction_trigger=52," \ - "eviction_updates_target=50,eviction_dirty_target=50,eviction_target=50," - - scenarios = make_scenarios(format_values) - - def get_stat(self, stat): - stat_cursor = self.session.open_cursor('statistics:') - val = stat_cursor[stat][2] - stat_cursor.close() - return val - - def test_app_thread_evict01(self): - format='key_format={},value_format={}'.format(self.key_format, self.value_format) - self.session.create(self.uri, format) - - # For our target stat to be incremented we need our application thread to evict a page, but - # this is probabilistic as the application thread is always racing against the internal - # eviction threads. Give the application thread a few chances to beat the internal thread - for _ in range(0, 20): - # Insert 40MB of data and perform lots of small inserts so we'll have a lot of - # pages to evict. We are below target levels so no eviction takes place - cursor = self.session.open_cursor(self.uri) - for i in range(40 * 1024): - cursor[i+1] = 'a' * 1024 - - # Perform two large updates. The first causes us to exceed trigger levels, - # and on the second insert the app thread is pulled into eviction since - # trigger levels are exceeded. - self.session.begin_transaction() - cursor[100001] = 'a' * 20 * 1024 * 1024 - cursor[100002] = 'a' * 20 * 1024 * 1024 - self.session.commit_transaction() - - num_app_evict_snapshot_refreshed = self.get_stat(wiredtiger.stat.conn.application_evict_snapshot_refreshed) - cursor.close() - - if num_app_evict_snapshot_refreshed > 0: - break - - self.assertGreater(self.get_stat(wiredtiger.stat.conn.application_evict_snapshot_refreshed), 0) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup29.py b/src/third_party/wiredtiger/test/suite/test_backup29.py deleted file mode 100644 index 5cbce78e321..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_backup29.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import os, re, time -from wtscenario import make_scenarios -from wtbackup import backup_base -from wiredtiger import stat - -# test_backup29.py -# Test interaction between checkpoint and incremental backup. There was a bug in -# maintaining the incremental backup bitmaps correctly after opening an uncached dhandle. -# This test reconstructs the failure scenario and verifies correct behavior both when a -# restart and when dhandle sweep lead to opening an uncached dhandle. -class test_backup29(backup_base): - conn_config = 'file_manager=(close_handle_minimum=0,' + \ - 'close_idle_time=3,close_scan_interval=1),' + \ - 'statistics=(fast)' - create_config = 'allocation_size=512,key_format=i,value_format=S' - # Backup directory name. Uncomment if actually taking a backup. - # dir='backup.dir' - uri1 = 'test_first' - uri2 = 'test_second' - file1_uri = 'file:' + uri1 + '.wt' - file2_uri = 'file:' + uri2 + '.wt' - table1_uri = 'table:' + uri1 - table2_uri = 'table:' + uri2 - active_uri = 'table:active.wt' - - value_base = '-abcdefghijkl' - - few = 100 - nentries = 5000 - - def get_open_file_count(self): - stat_cursor = self.session.open_cursor('statistics:', None, None) - n = stat_cursor[stat.conn.file_open][2] - stat_cursor.close() - return n - - def parse_blkmods(self, uri): - meta_cursor = self.session.open_cursor('metadata:') - config = meta_cursor[uri] - meta_cursor.close() - # The search string looks like: ,blocks=feffff1f000000000000000000000000 - # Obtain just the hex string. - b = re.search(',blocks=(\w+)', config) - self.assertTrue(b is not None) - # The bitmap string after the = is in group 1. - blocks = b.group(1) - self.pr("block bitmap: " + blocks) - return blocks - - def compare_bitmap(self, orig, new): - # Compare the bitmaps from the metadata. Once a bit is set, it should never - # be cleared. But new bits could be set. So the check is only: if the original - # bitmap has a bit set then the current bitmap must be set for that bit also. - # - # First convert both bitmaps to a binary string, accounting for any possible leading - # zeroes (that would be truncated off). Then compare bit by bit. - orig_bits = bin(int('1'+orig, 16))[3:] - new_bits = bin(int('1'+new, 16))[3:] - self.pr("Original bitmap in binary: " + orig_bits) - self.pr("Reopened bitmap in binary: " + new_bits) - for o_bit, n_bit in zip(orig_bits, new_bits): - if o_bit != '0': - self.assertTrue(n_bit != '0') - - def setup_test(self): - # Create and populate the tables. - self.session.create(self.table1_uri, self.create_config) - self.session.create(self.table2_uri, self.create_config) - c1 = self.session.open_cursor(self.table1_uri) - c2 = self.session.open_cursor(self.table2_uri) - # Only add a few entries. - self.pr("Write: " + str(self.few) + " initial data items") - for i in range(1, self.few): - val = str(i) + self.value_base - c1[i] = val - c2[i] = val - self.session.checkpoint() - - # Take the initial full backup for incremental. We don't actually need to - # take the backup, we only need to open and close the backup cursor to have - # the library keep track of the bitmaps. - config = 'incremental=(enabled,granularity=4k,this_id="ID1")' - bkup_c = self.session.open_cursor('backup:', None, config) - # Uncomment these lines if actually taking the full backup is helpful for debugging. - # os.mkdir(self.dir) - # self.take_full_backup(self.dir, bkup_c) - bkup_c.close() - - # Add a lot more data to both tables to generate a filled-in block mod bitmap. - last_i = self.few - self.pr("Write: " + str(self.nentries) + " additional data items") - for i in range(self.few, self.nentries): - val = str(i) + self.value_base - c1[i] = val - c2[i] = val - c1.close() - c2.close() - self.session.checkpoint() - # Get the block mod bitmap from the file URI. - self.orig1_bitmap = self.parse_blkmods(self.file1_uri) - self.orig2_bitmap = self.parse_blkmods(self.file2_uri) - - - def incr_backup_and_validate(self): - # After reopening we want to open both tables, but only modify one of them for - # the first checkpoint. Then modify the other table, checkpoint, and then check the - # that the block mod bitmap remains correct for the other table. - c1 = self.session.open_cursor(self.table1_uri) - c2 = self.session.open_cursor(self.table2_uri) - last_i = self.nentries - - # Change the first table and checkpoint. Keep the second table clean. - self.pr("Update only table 1: " + str(last_i)) - val = str(last_i) + self.value_base - c1[last_i] = val - self.session.checkpoint() - new1_bitmap = self.parse_blkmods(self.file1_uri) - - # Now change the second table and checkpoint again. - self.pr("Update second table: " + str(last_i)) - c2[last_i] = val - self.session.checkpoint() - new2_bitmap = self.parse_blkmods(self.file2_uri) - - c1.close() - c2.close() - - self.compare_bitmap(self.orig1_bitmap, new1_bitmap) - self.compare_bitmap(self.orig2_bitmap, new2_bitmap) - - def test_backup29_reopen(self): - self.setup_test() - - self.pr("CLOSE and REOPEN conn") - self.reopen_conn() - self.pr("Reopened conn") - - self.incr_backup_and_validate() - - def test_backup29_sweep(self): - self.setup_test() - - self.pr("Waiting to sweep handles") - # Create another table and populate it, and checkpoint. - self.session.create(self.active_uri, self.create_config) - c = self.session.open_cursor(self.active_uri) - for i in range(1, self.few): - c[i] = str(i) + self.value_base - self.session.checkpoint() - - sleep = 0 - max = 20 - # The only files sweep won't close should be the metadata, the history store, the - # lock file, and our active file. - final_nfile = 4 - - # Keep updating and checkpointing this table until all other handles have been swept. - # The checkpoints have the side effect of sweeping the session cache, which will allow - # dhandles to be closed. - while sleep < max: - i = i + 1 - c[i] = str(i) + self.value_base - self.session.checkpoint() - sleep += 0.5 - time.sleep(0.5) - nfile = self.get_open_file_count() - if nfile == final_nfile: - break - c.close() - - # Make sure we swept everything before we ran out of time. - self.assertEqual(nfile, final_nfile) - self.pr("Sweep done") - - self.incr_backup_and_validate() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug029.py b/src/third_party/wiredtiger/test/suite/test_bug029.py deleted file mode 100644 index b2d9e1c6ac4..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_bug029.py +++ /dev/null @@ -1,112 +0,0 @@ - -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# [TEST_TAGS] -# checkpoint:recovery -# [END_TAGS] - -import wttest -import os, shutil - -# test_bug029.py -# -# Test that WT correctly propogates the most recent checkpoint time -# across restarts. We validate this by reproducing the original bug -# from WT-9457: frequent checkpoints pushed the checkpoint clock time -# into the future such that immediately after a restart a backup could -# see its checkpoint deleted out from under it. The result was fatal -# read errors when restoring the backup. - -class test_bug029(wttest.WiredTigerTestCase): - conn_config = ("cache_size=50MB") - uri = "table:test_bug029" - bigvalue = "WiredTiger" * 100 - backup_dir = "backup_dir" - - def add_data(self, uri, start, count): - cursor = self.session.open_cursor(uri, None) - for i in range(start, start + count): - cursor[i] = self.bigvalue - cursor.close() - - def test_bug029(self): - # Create and populate table - self.session.create(self.uri, "key_format=i,value_format=S") - self.add_data(self.uri, 0, 2000) - self.session.checkpoint() - - # Force the checkpoint time forward with a lot of quick checkpoints. - for i in range(100): - self.session.checkpoint("force=1") - - # Add more data and checkpoint again. This creates a bunch of pages - # in the final checkpoint that can be deleted and reused if we hit - # the bug. - self.add_data(self.uri, 2000, 2000) - self.session.checkpoint() - - # Shutdown and reopen. - self.reopen_conn() - - self.add_data(self.uri, 0, 100) - - # Open a backup cursor and force a few checkpoints. This will allow - # WT to delete older checkpoints, but as long as the backup cursor - # is open, it shouldn't delete the backup checkpoint---unless we hit - # the bug. - backup_cursor = self.session.open_cursor('backup:') - - for i in range(10): - self.session.checkpoint("force=1") - - # Write and checkpoint a bunch of data. If we erroneously deleted our - # backup checkpoint this should overwrite some of that checkpoint's - # blocks. - self.add_data(self.uri, 1000, 2000) - self.session.checkpoint() - - # Now do the backup. - os.mkdir(self.backup_dir) - while True: - ret = backup_cursor.next() - if ret != 0: - break - shutil.copy(backup_cursor.get_key(), self.backup_dir) - backup_cursor.close() - - # Open the backup and read data. If the backup snapshot was corrupted - # we will panic and die here. - backup_conn = self.wiredtiger_open(self.backup_dir, self.conn_config) - session = backup_conn.open_session() - cur1 = session.open_cursor(self.uri) - for i in range(0, 4000, 10): - self.assertEqual(cur1[i], self.bigvalue) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug030.py b/src/third_party/wiredtiger/test/suite/test_bug030.py deleted file mode 100644 index 3969d71de9d..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_bug030.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -import wttest -from helper import simulate_crash_restart -from wiredtiger import stat -from wtdataset import SimpleDataSet -from wtscenario import make_scenarios - -# test_bug030.py -# This tests for the scenario in WT-10522 where we could return early when -# appending a key's original value to its update list due to checking some -# flags that must be ignored when looking at an aborted tombstone. -class test_bug_030(wttest.WiredTigerTestCase): - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - scenarios = make_scenarios(format_values) - - def conn_config(self): - config = 'debug_mode=(update_restore_evict=true)' - return config - - def test_bug030(self): - nrows = 10 - uri = "table:test_bug030" - - if self.value_format == '8t': - valuea = 97 - valueb = 98 - else: - valuea = "abcdef" * 3 - valueb = "ghijkl" * 3 - - self.session.create(uri, 'key_format={},value_format={}'.format( - self.key_format, self.value_format)) - - # Stable insert - self.session.begin_transaction() - cursor = self.session.open_cursor(uri) - for i in range(1, nrows + 1): - cursor[i] = valuea - cursor.close() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - - self.conn.set_timestamp('oldest_timestamp={},stable_timestamp={}'.format( - self.timestamp_str(10), self.timestamp_str(20))) - - # Unstable delete - self.session.begin_transaction() - cursor = self.session.open_cursor(uri) - for i in range(1, nrows + 1): - cursor.set_key(i) - cursor.remove() - cursor.close() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) - - # Evict everything - self.session.begin_transaction() - evict_cursor = self.session.open_cursor(uri, None, 'debug=(release_evict)') - for i in range(1, nrows + 1): - evict_cursor.set_key(i) - evict_cursor.search() - evict_cursor.reset() - evict_cursor.close() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) - - # Unstable, uncommitted update - self.session.begin_transaction() - cursor = self.session.open_cursor(uri) - for i in range(1, nrows + 1): - cursor[i] = valueb - cursor.close() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) - - self.session.checkpoint() - self.conn.rollback_to_stable() - - # Another delete, committed this time - self.session.begin_transaction() - cursor = self.session.open_cursor(uri) - for i in range(1, nrows + 1): - cursor.set_key(i) - cursor.remove() - cursor.close() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(60)) - - # Finally, evict everything. At this point, our key(s) have an - # update chain with a single entry that's both aborted and restored - # from the data store, that we attempt to reconcile. - self.session.begin_transaction() - evict_cursor = self.session.open_cursor(uri, None, 'debug=(release_evict)') - for i in range(1, nrows + 1): - evict_cursor.set_key(i) - evict_cursor.search() - evict_cursor.reset() - evict_cursor.close() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(70)) diff --git a/src/third_party/wiredtiger/test/suite/test_bug032.py b/src/third_party/wiredtiger/test/suite/test_bug032.py deleted file mode 100644 index bd77d3fa2a8..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_bug032.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wttest -from wtdataset import SimpleDataSet -from wtscenario import make_scenarios - -# test_bug032.py -# This tests for the scenario discovered in WT-11845. -# Before WT-11845 fast truncate determined if a page could be fast truncated by -# looking at the pages aggregated timestamp. This would lead to keys being incorrectly -# truncated if: -# - Two transactions txn1 and txn2 have ids 10 and 20 respectively -# - txn2 is committed and txn1 is still active while a truncate transaction begins. -# txn2 is visible to the truncate while txn1 is not. -# - txn1 is then committed and a page containing updates from both transactions is written to disk. -# The aggregated timestamp uses txn id 20. -# - Truncate reads the aggregated timestamp of the page (id 20) and determines it is visible. -# The page is truncated even though it contains updates from txn1 which is not visible to the -# truncate operation. -class test_bug032(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,statistics=(all)' - - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('string_row', dict(key_format='S', value_format='S')), - ] - scenarios = make_scenarios(format_values) - - def populate(self, uri, ds, nrows, value): - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - for i in range(1, nrows): - cursor[ds.key(i)] = value + str(i) - - self.session.commit_transaction() - cursor.close() - - def test_bug032(self): - uri = 'table:bug032' - nrows = 500 - - # The key to be inserted and truncated in parallel, as described in the test description. - txn1_key_num = 32 - - # 512 byte keys. We're limiting leaf_page_max to 10KB which will give us ~20 keys per page. - value_str = 'a' * 512 - - # Create a table and populate it - ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format, config='allocation_size=512,leaf_page_max=10KB') - ds.create() - self.populate(uri, ds, nrows, value_str) - - # Remove txn1's key which was added by the populate phase. We want truncate and insert to race in - # a way they aren't aware of each other. If there's an existing version of the key then txn1's - # insert becomes a modify and truncate will see the parallel modification, rolling back instead - # of truncating. - self.session.begin_transaction() - c = self.session.open_cursor(uri) - c.set_key(ds.key(txn1_key_num)) - c.remove() - self.session.commit_transaction() - c.close() - - # Perform an in txn1 but don't commit it. - txn1_session = self.conn.open_session() - txn1_session.begin_transaction() - c1 = txn1_session.open_cursor(uri) - c1[ds.key(txn1_key_num)] = value_str - c1.reset() - - # Start and commit txn2. This has a larger txn id than txn1. - txn2_session = self.conn.open_session() - txn2_session.begin_transaction() - c2 = txn2_session.open_cursor(uri) - c2[ds.key(txn1_key_num + 1)] = value_str - txn2_session.commit_transaction() - c2.close() - - # Start the truncate transaction. Here the snapshot will see txn2 but not txn1. - truncate_session = self.conn.open_session() - truncate_session.begin_transaction() - - # Commit txn1. - txn1_session.commit_transaction() - c1.reset() - c1.close() - - # Evict the page txn1 containing modifications from both txn1 and txn2. - evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") - evict_cursor[ds.key(txn1_key_num)] - self.assertEqual(evict_cursor.reset(), 0) - evict_cursor.close() - - # Truncate everything. This will attempt to fast truncate the page we evicted to disk. - # The page cannot be fast truncated as txn1's key is not visible to the truncate. - truncate_session.truncate(uri, None, None, None) - truncate_session.commit_transaction() - - # Search for our key inserted by txn1. This was not truncated by the truncate operation as - # txn1 wasn't visible. - validate_cursor = self.session.open_cursor(ds.uri) - validate_cursor.set_key(ds.key(txn1_key_num)) - self.assertEqual(validate_cursor.search(), 0) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bulk01.py b/src/third_party/wiredtiger/test/suite/test_bulk01.py index 8317151e5ec..76893efd9ce 100755 --- a/src/third_party/wiredtiger/test/suite/test_bulk01.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk01.py @@ -33,12 +33,10 @@ import wiredtiger, wttest from wtdataset import simple_key, simple_value from wtscenario import make_scenarios -from wiredtiger import stat # Smoke test bulk-load. class test_bulk_load(wttest.WiredTigerTestCase): name = 'test_bulk' - conn_config = "statistics=(fast)" types = [ ('file', dict(type='file:')), @@ -56,27 +54,14 @@ class test_bulk_load(wttest.WiredTigerTestCase): ] scenarios = make_scenarios(types, keyfmt, valfmt) - def get_stat(self, stat): - stat_cursor = self.session.open_cursor('statistics:') - val = stat_cursor[stat][2] - stat_cursor.close() - return val - # Test a simple bulk-load def test_bulk_load(self): uri = self.type + self.name self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) - - self.assertEqual(self.get_stat(stat.conn.cursor_bulk_count), 0) cursor = self.session.open_cursor(uri, None, "bulk") - self.assertEqual(self.get_stat(stat.conn.cursor_bulk_count), 1) - for i in range(1, 1000): cursor[simple_key(cursor, i)] = simple_value(cursor, i) - cursor.close() - - self.assertEqual(self.get_stat(stat.conn.cursor_bulk_count), 0) # Test a bulk-load triggers variable-length column-store RLE correctly. def test_bulk_load_var_rle(self): diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py index 76ded0c810f..e9411acfe0d 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py @@ -63,7 +63,7 @@ class test_checkpoint04(wttest.WiredTigerTestCase): def test_checkpoint_stats(self): nrows = 100 - ntables = 50 + ntables = 10 multiplier = 1 # Run the loop and increase the value size with each iteration until @@ -117,10 +117,7 @@ class test_checkpoint04(wttest.WiredTigerTestCase): time_total = self.get_stat(stat.conn.txn_checkpoint_time_total) self.pr('txn_checkpoint_time_total ' + str(time_total)) - # Account for When the connection re-opens on an existing datable as we perform a - # checkpoint during the open stage. - expected_ckpts = 3 if multiplier > 1 else 2 - self.assertEqual(num_ckpt, expected_ckpts) + self.assertEqual(num_ckpt, 2 * multiplier) self.assertEqual(running, 0) self.assertEqual(prep_running, 0) # Assert if this loop continues for more than 100 iterations. @@ -130,11 +127,8 @@ class test_checkpoint04(wttest.WiredTigerTestCase): # Run the loop again if any of the below condition fails and exit if the test passes. if prep_min < time_min and prep_max < time_max and prep_recent < time_recent and prep_total < time_total: break - - multiplier += 1 - # Reopen the connection to reset statistics. - # We don't want stats from earlier runs to interfere with later runs. - self.reopen_conn() + else: + multiplier += 1 if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint30.py b/src/third_party/wiredtiger/test/suite/test_checkpoint30.py deleted file mode 100644 index 066b51a80bc..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint30.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import threading, time -import wttest -import wiredtiger -from wtdataset import SimpleDataSet -from wtscenario import make_scenarios - -# test_checkpoint30.py -# -# Test reading a cursor when the aggregate time window is visible to the snapshot -# but not all deleted keys on-disk version are not visible. -class test_checkpoint(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,statistics=(all)' - - format_values = [ - ('column', dict(key_format='r', value_format='S', extraconfig='')), - ('string_row', dict(key_format='S', value_format='S', extraconfig='')), - ] - scenarios = make_scenarios(format_values) - - def large_updates(self, uri, ds, nrows, value, ts): - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - for i in range(1, nrows + 1): - cursor[ds.key(i)] = value - if i % 101 == 0: - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) - self.session.begin_transaction() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) - cursor.close() - - def large_removes(self, uri, ds, start_row, nrows, ts): - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - for i in range(start_row, nrows + 1): - cursor.set_key(ds.key(i)) - cursor.remove() - if i % 101 == 0: - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) - self.session.begin_transaction() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) - cursor.close() - - def evict(self, ds, lo, hi, value, ts): - evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) - # Evict every 10th key. FUTURE: when that's possible, evict each page exactly once. - for k in range(lo, hi, 10): - v = evict_cursor[ds.key(k)] - self.assertEqual(v, value) - self.assertEqual(evict_cursor.reset(), 0) - self.session.rollback_transaction() - - def check(self, session_local, ds, nrows, value, ts): - cursor = session_local.open_cursor(ds.uri) - count = 0 - for k, v in cursor: - self.assertEqual(v, value) - count += 1 - self.assertEqual(count, nrows) - cursor.close() - - def test_checkpoint(self): - uri = 'table:checkpoint30' - nrows = 100 - - # Create a table. - ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format=self.value_format, - config=self.extraconfig) - ds.populate() - - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - - # Pin oldest and stable timestamps to 5. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5) + - ',stable_timestamp=' + self.timestamp_str(5)) - - # Write some data at time 10. - self.large_updates(uri, ds, nrows, value_a, 10) - self.check(self.session, ds, nrows, value_a, 10) - - # Remove the key-1 at time 20 and keep this transaction open until a reader is started. - session2 = self.conn.open_session() - cursor = session2.open_cursor(uri) - session2.begin_transaction() - cursor.set_key(ds.key(1)) - cursor.remove() - - # Remove the remaining keys with another transaction and commit. - self.large_removes(uri, ds, 2, nrows, 20) - self.check(self.session, ds, 1, value_a, 20) - - # Start the reader transaction. According to the snapshot visibility, except key-1 - # rest of the keys are removed ignoring the read timestamp. - session3 = self.conn.open_session() - session3.begin_transaction() - - # Commit the key-1 remove transaction. - session2.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) - cursor.close() - - # Reader transaction cannot see the remove on key-1. - self.check(session3, ds, 1, value_a, 25) - - # Evict the data. - self.evict(ds, 1, nrows + 1, value_a, 10) - - # Reader transaction cannot see the remove on key-1 after eviction. - self.check(session3, ds, 1, value_a, 25) - - # Checkpoint. - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(25)) - self.session.checkpoint() - - # Reader transaction cannot see the remove on key-1 after the checkpoint. - self.check(session3, ds, 1, value_a, 25) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint32.py b/src/third_party/wiredtiger/test/suite/test_checkpoint32.py deleted file mode 100644 index a8cd84bb5da..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint32.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import threading, time -import wttest -import wiredtiger -from wtdataset import SimpleDataSet -from wtscenario import make_scenarios -from wiredtiger import stat - -# test_checkpoint32.py -# -# Test that skipping in-memory reconciled deleted pages as part of the tree walk. -class test_checkpoint32(wttest.WiredTigerTestCase): - - format_values = [ - # FLCS doesn't support skipping pages based on aggregated time. - ('column', dict(key_format='r', value_format='S', extraconfig='')), - ('string_row', dict(key_format='S', value_format='S', extraconfig='')), - ] - - scenarios = make_scenarios(format_values) - - def conn_config(self): - return 'statistics=(all)' - - def check(self, ds, nrows, value): - cursor = self.session.open_cursor(ds.uri) - count = 0 - for k, v in cursor: - self.assertEqual(v, value) - count += 1 - self.assertEqual(count, nrows) - cursor.close() - - def test_checkpoint(self): - uri = 'table:checkpoint32' - nrows = 1000 - - # Create a table. - ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format=self.value_format, - config=self.extraconfig) - ds.populate() - - value_a = "aaaaa" * 100 - - # Write some initial data. - cursor = self.session.open_cursor(ds.uri, None, None) - for i in range(1, nrows + 1): - self.session.begin_transaction() - cursor[ds.key(i)] = value_a - self.session.commit_transaction() - - # Create a reader transaction that will not be able to see what happens next. - # We don't need to do anything with this; it just needs to exist. - session2 = self.conn.open_session() - session2.begin_transaction() - - # Now remove all data. - for i in range(1, nrows + 1): - self.session.begin_transaction() - cursor.set_key(ds.key(i)) - self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction() - - # Checkpoint. - self.session.checkpoint() - - # Get the existing in-memory delete page skip statistic value. - stat_cursor = self.session.open_cursor('statistics:', None, None) - prev_cur_inmem_del_page_skip = stat_cursor[stat.conn.cursor_tree_walk_inmem_del_page_skip][2] - stat_cursor.close() - - # Now read the removed data. - self.check(ds, 0, value_a) - - # Get the new in-memory delete page skip statistic value. - stat_cursor = self.session.open_cursor('statistics:', None, None) - cur_inmem_del_page_skip = stat_cursor[stat.conn.cursor_tree_walk_inmem_del_page_skip][2] - stat_cursor.close() - - self.assertGreater(cur_inmem_del_page_skip, prev_cur_inmem_del_page_skip) - - # Tidy up. - session2.rollback_transaction() - session2.close() - cursor.close() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py index 7ee28a240cc..89a7faddfc0 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py @@ -161,15 +161,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - - # Wait for checkpoint to start and acquire its snapshot before committing. - ckpt_snapshot = 0 - while not ckpt_snapshot: - time.sleep(1) - stat_cursor = self.session.open_cursor('statistics:', None, None) - ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] - stat_cursor.close() - + # Sleep for sometime so that checkpoint starts before committing last transaction. + time.sleep(2) session1.commit_transaction() finally: @@ -222,15 +215,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - - # Wait for checkpoint to start and acquire its snapshot before committing. - ckpt_snapshot = 0 - while not ckpt_snapshot: - time.sleep(1) - stat_cursor = self.session.open_cursor('statistics:', None, None) - ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] - stat_cursor.close() - + # Sleep for sometime so that checkpoint starts before committing last transaction. + time.sleep(2) session1.commit_transaction() finally: @@ -286,15 +272,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - - # Wait for checkpoint to start and acquire its snapshot before committing. - ckpt_snapshot = 0 - while not ckpt_snapshot: - time.sleep(1) - stat_cursor = self.session.open_cursor('statistics:', None, None) - ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] - stat_cursor.close() - + # Sleep for sometime so that checkpoint starts before committing last transaction. + time.sleep(2) session2.commit_transaction() finally: diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py index 583808915b6..e8ff0067f61 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py @@ -147,15 +147,8 @@ class test_checkpoint_snapshot05(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - - # Wait for checkpoint to start and acquire its snapshot before committing. - ckpt_snapshot = 0 - while not ckpt_snapshot: - time.sleep(1) - stat_cursor = self.session.open_cursor('statistics:', None, None) - ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] - stat_cursor.close() - + # Sleep for sometime so that checkpoint starts before committing last transaction. + time.sleep(2) session1.commit_transaction() self.evict(self.uri, ds, self.nrows) finally: diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index a339ea3d305..3b64532c036 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest, sys +import wiredtiger, wttest from wtdataset import SimpleDataSet, ComplexDataSet, simple_key, simple_value from wtscenario import make_scenarios @@ -42,7 +42,6 @@ class test_cursor_random(wttest.WiredTigerTestCase): ('not-sample', dict(config='next_random=true')) ] scenarios = make_scenarios(types, config) - expected_warning_msg = 'Eviction took more than 1 minute' # Check that opening a random cursor on a row-store returns not-supported # for methods other than next, reconfigure and reset, and next returns @@ -111,12 +110,6 @@ class test_cursor_random(wttest.WiredTigerTestCase): list.append(cursor.get_key()) self.assertGreater(len(set(list)), 80) - # Ignore the eviction generation drain warning as it is possible for eviction to - # take longer to evict pages due to overflow items on the page. - self.conn.close() - if (sys.platform.startswith('darwin')): - self.ignoreStdoutPatternIfExists(self.expected_warning_msg) - def test_cursor_random_multiple_insert_records_small(self): self.cursor_random_multiple_insert_records(2000) def test_cursor_random_multiple_insert_records_large(self): @@ -144,12 +137,6 @@ class test_cursor_random(wttest.WiredTigerTestCase): list.append(cursor.get_key()) self.assertGreater(len(set(list)), 80) - # Ignore the eviction generation drain warning as it is possible for eviction to - # take longer to evict pages due to overflow items on the page. - self.conn.close() - if (sys.platform.startswith('darwin')): - self.ignoreStdoutPatternIfExists(self.expected_warning_msg) - def test_cursor_random_multiple_page_records_reopen_small(self): self.cursor_random_multiple_page_records(2000, True) def test_cursor_random_multiple_page_records_reopen_large(self): @@ -182,12 +169,6 @@ class test_cursor_random(wttest.WiredTigerTestCase): for i in range(1,10): self.assertEqual(cursor.next(), 0) - # Ignore the eviction generation drain warning as it is possible for eviction to - # take longer to evict pages due to overflow items on the page. - self.conn.close() - if (sys.platform.startswith('darwin')): - self.ignoreStdoutPatternIfExists(self.expected_warning_msg) - # Check that next_random fails in the presence of a set of values, all of # which are deleted. def test_cursor_random_deleted_all(self): @@ -205,12 +186,6 @@ class test_cursor_random(wttest.WiredTigerTestCase): for i in range(1,10): self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) - # Ignore the eviction generation drain warning as it is possible for eviction to - # take longer to evict pages due to overflow items on the page. - self.conn.close() - if (sys.platform.startswith('darwin')): - self.ignoreStdoutPatternIfExists(self.expected_warning_msg) - # Check that opening a random cursor on column-store returns not-supported. class test_cursor_random_column(wttest.WiredTigerTestCase): type_values = [ diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random03.py b/src/third_party/wiredtiger/test/suite/test_cursor_random03.py deleted file mode 100644 index 4d8696f63ea..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random03.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wttest -from wtdataset import SimpleDataSet - -# test_cursor_random03.py -# This python test reproduces the WT-12225 bug where the same stream of numbers are generated -# from the random cursor. In the presence of where all the values are the insert list, open up -# two cursors close to each other in time, and both cursors will return back the exact same stream -# of records again. The test makes sure that records returned in both cursors return at least -# one record that is different, ensuring that the stream of numbers returned from __wt_random -# are not in the same pattern. -class test_cursor_random03(wttest.WiredTigerTestCase): - def test_cursor_random_bug(self): - uri = 'table:random' - - # Do not change the chosen number of records. The records is to make sure that - # the skip insert list random estimate is a power of 2 to act as a mask. - # Also set the leaf-page-max value, otherwise the page might split. - ds = SimpleDataSet(self, uri, 2135, config='leaf_page_max=100MB') - ds.populate() - - for _ in range(0, 5000): - random_keys = [] - - cursor = self.session.open_cursor(uri, None, 'next_random=true') - - # Perform 100 nexts and append to our random keys. - for _ in range(0, 100): - self.assertEqual(cursor.next(), 0) - current = cursor.get_key() - random_keys.append(current) - cursor.close() - - # The random cursor initial seed is time based, reset the initial random seed again - # to make sure that we have not generated with the same random numbers - cursor = self.session.open_cursor(uri, None, 'next_random=true') - found_different = False - for i in range(0, 100): - self.assertEqual(cursor.next(), 0) - current = cursor.get_key() - # Exit early once we found a key that is different. - if (random_keys[i] != current): - found_different = True - break - cursor.close() - - # We expect that the values between two recently opened cursors return different stream - # of records. - self.assertTrue(found_different) - diff --git a/src/third_party/wiredtiger/test/suite/test_hs07.py b/src/third_party/wiredtiger/test/suite/test_hs07.py index 27430cbf8ff..2137a37ba7f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs07.py +++ b/src/third_party/wiredtiger/test/suite/test_hs07.py @@ -225,7 +225,5 @@ class test_hs07(wttest.WiredTigerTestCase): # Check that the new updates are only seen after the update timestamp self.check(bigvalue, uri, nrows, 300) - self.ignoreStdoutPatternIfExists('Eviction took more than 1 minute') - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs11.py b/src/third_party/wiredtiger/test/suite/test_hs11.py index 417d37cca29..647a7bfece5 100755 --- a/src/third_party/wiredtiger/test/suite/test_hs11.py +++ b/src/third_party/wiredtiger/test/suite/test_hs11.py @@ -44,20 +44,8 @@ class test_hs11(wttest.WiredTigerTestCase): ('deletion', dict(update_type='deletion')), ('update', dict(update_type='update')) ] - long_running_txn_values = [ - ('long-running', dict(long_run_txn=True)), - ('no-long-running', dict(long_run_txn=False)) - ] - last_update_type_values = [ - ('modify', dict(modify=True)), - ('no-modify', dict(modify=False)) - ] - nrows = [ - ('small-nrows', dict(nrows=100)), - ('large-nrows', dict(nrows=10000)) - ] - scenarios = make_scenarios(format_values, update_type_values,long_running_txn_values, last_update_type_values, nrows) - timestamps = 5 + scenarios = make_scenarios(format_values, update_type_values) + nrows = 10000 def create_key(self, i): if self.key_format == 'S': @@ -70,18 +58,6 @@ class test_hs11(wttest.WiredTigerTestCase): stat_cursor.close() return val - def evict_cursor(self, uri, nrows): - s = self.conn.open_session() - s.begin_transaction() - # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. - evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") - for i in range(1, nrows + 1): - evict_cursor.set_key(self.create_key(i)) - evict_cursor.search() - evict_cursor.reset() - s.rollback_transaction() - evict_cursor.close() - def test_non_ts_updates_clears_hs(self): uri = 'table:test_hs11' create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) @@ -93,14 +69,13 @@ class test_hs11(wttest.WiredTigerTestCase): else: value1 = 'a' * 500 value2 = 'b' * 500 - mod_value = 'm' + 'a' * 499 # FIXME-WT-9063 revisit the use of self.retry() throughout this file. # Apply a series of updates from timestamps 1-4. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for ts in range(1, self.timestamps): + for ts in range(1, 5): for i in range(1, self.nrows): for retry in self.retry(): with retry.transaction(commit_timestamp = ts): @@ -108,24 +83,8 @@ class test_hs11(wttest.WiredTigerTestCase): # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() - self.evict_cursor(uri, self.nrows) - - # Apply a modify update at timestamp 5. - if self.modify and self.value_format != '8t': - for i in range(1, self.nrows): - for retry in self.retry(): - with retry.transaction(commit_timestamp = 5): - cursor.set_key(self.create_key(i)) - cursor.modify([wiredtiger.Modify("m", 0, 1)]) - self.timestamps += 1 - - # Start a long running transaction at timestamp 5. - if self.long_run_txn: - session2 = self.conn.open_session() - session2.begin_transaction('read_timestamp=5') - # Apply an update without timestamp. If we have a long running transaction this update - # should not be globally visible until that transaction has ended. + # Apply an update without timestamp. for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': @@ -136,11 +95,6 @@ class test_hs11(wttest.WiredTigerTestCase): # Reconcile and remove the obsolete entries. self.session.checkpoint() - if self.long_run_txn: - session2.rollback_transaction() - - # At this point any updates with no timestamp should be globally visible. - self.evict_cursor(uri, self.nrows) # Now apply an update at timestamp 10. for i in range(1, self.nrows): @@ -148,10 +102,8 @@ class test_hs11(wttest.WiredTigerTestCase): with retry.transaction(commit_timestamp = 10): cursor[self.create_key(i)] = value2 - self.session.checkpoint() - # Ensure that we blew away history store content. - for ts in range(1, self.timestamps): + for ts in range(1, 5): for retry in self.retry(): with retry.transaction(read_timestamp = ts, rollback = True): for i in range(1, self.nrows): @@ -166,10 +118,7 @@ class test_hs11(wttest.WiredTigerTestCase): else: self.assertEqual(cursor[self.create_key(i)], value2) else: - if ts == 5 and self.modify and self.value_format != '8t': - self.assertEqual(cursor[self.create_key(i)], mod_value) - else: - self.assertEqual(cursor[self.create_key(i)], value1) + self.assertEqual(cursor[self.create_key(i)], value1) if self.update_type == 'deletion': hs_truncate = self.get_stat(stat.conn.cache_hs_key_truncate_onpage_removal) @@ -186,12 +135,11 @@ class test_hs11(wttest.WiredTigerTestCase): else: value1 = 'a' * 500 value2 = 'b' * 500 - mod_value = 'm' + 'a' * 499 # Apply a series of updates from timestamps 1-4. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for ts in range(1, self.timestamps): + for ts in range(1, 5): for i in range(1, self.nrows): for retry in self.retry(): with retry.transaction(commit_timestamp = ts): @@ -200,15 +148,6 @@ class test_hs11(wttest.WiredTigerTestCase): # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() - # Apply a modify update at timestamp 5. - if self.modify and self.value_format != '8t': - for i in range(1, self.nrows): - for retry in self.retry(): - with retry.transaction(commit_timestamp = 5): - cursor.set_key(self.create_key(i)) - cursor.modify([wiredtiger.Modify("m", 0, 1)]) - self.timestamps += 1 - # Remove the key with timestamp 10. for i in range(1, self.nrows): if i % 2 == 0: @@ -239,10 +178,7 @@ class test_hs11(wttest.WiredTigerTestCase): else: self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: - if self.modify and self.value_format != '8t': - self.assertEqual(cursor[self.create_key(i)], mod_value) - else: - self.assertEqual(cursor[self.create_key(i)], value1) + self.assertEqual(cursor[self.create_key(i)], value1) hs_truncate = self.get_stat(stat.conn.cache_hs_key_truncate_onpage_removal) self.assertEqual(hs_truncate, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_hs20.py b/src/third_party/wiredtiger/test/suite/test_hs20.py index e2b9d878517..071d33de9bd 100755 --- a/src/third_party/wiredtiger/test/suite/test_hs20.py +++ b/src/third_party/wiredtiger/test/suite/test_hs20.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest, sys +import wiredtiger, wttest from wtscenario import make_scenarios # test_hs20.py @@ -103,8 +103,3 @@ class test_hs20(wttest.WiredTigerTestCase): for retry in self.retry(): with retry.transaction(read_timestamp = 3, rollback = True): self.assertEqual(cursor[self.make_key(i)], value1 + "B") - - if (sys.platform.startswith('darwin')): - # Ignore the eviction generation drain warning as it is possible for eviction to take - # longer to evict pages due to overflow items on the page. - self.ignoreStdoutPatternIfExists('Eviction took more than 1 minute') diff --git a/src/third_party/wiredtiger/test/suite/test_hs32.py b/src/third_party/wiredtiger/test/suite/test_hs32.py deleted file mode 100755 index b90bbaa15db..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_hs32.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios -from wiredtiger import stat - -# test_hs32.py -# Ensure that updates without timestamps clear the history store records. -class test_hs32(wttest.WiredTigerTestCase): - conn_config = 'cache_size=500MB,statistics=(all)' - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column-fix', dict(key_format='r', value_format='8t')), - ('integer-row', dict(key_format='i', value_format='S')), - ('string-row', dict(key_format='S', value_format='S')), - ] - update_type_values = [ - ('deletion', dict(update_type='deletion')), - ('update', dict(update_type='update')) - ] - long_running_txn_values = [ - ('no-long-run-txn', dict(long_run_txn=False)), - ('long-run-txn', dict(long_run_txn=True)) - ] - scenarios = make_scenarios(format_values, update_type_values,long_running_txn_values) - nrows = 100 - - def create_key(self, i): - if self.key_format == 'S': - return str(i) - return i - - def get_stat(self, stat): - stat_cursor = self.session.open_cursor('statistics:') - val = stat_cursor[stat][2] - stat_cursor.close() - return val - - def evict_cursor(self, uri, nrows): - s = self.conn.open_session() - s.begin_transaction() - # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. - evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") - for i in range(1, nrows + 1): - evict_cursor.set_key(self.create_key(i)) - evict_cursor.search() - evict_cursor.reset() - s.rollback_transaction() - evict_cursor.close() - - def test_non_ts_updates_tombstone_clears_hs(self): - uri = 'table:test_hs32' - create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) - self.session.create(uri, create_params) - - if self.value_format == '8t': - value1 = 97 - value2 = 98 - else: - value1 = 'a' * 500 - value2 = 'b' * 500 - - # Apply a series of updates from timestamps 1-4. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - cursor = self.session.open_cursor(uri) - for ts in range(1, 5): - for i in range(1, self.nrows): - for retry in self.retry(): - with retry.transaction(commit_timestamp = ts): - cursor[self.create_key(i)] = value1 - - # Reconcile and flush versions 1-3 to the history store. - self.session.checkpoint() - self.evict_cursor(uri, self.nrows) - - if self.long_run_txn: - # Apply a another update at timestamp 5. - for i in range(1, self.nrows): - for retry in self.retry(): - with retry.transaction(commit_timestamp = 5): - cursor[self.create_key(i)] = value1 - - # Start a long running transaction to make tombstone not globally visible. - session2 = self.conn.open_session() - session2.begin_transaction('read_timestamp=5') - - # Apply an update/delete without timestamp. - for i in range(1, self.nrows): - self.session.begin_transaction() - if i % 2 == 0: - if self.update_type == 'deletion': - cursor.set_key(self.create_key(i)) - cursor.remove() - else: - cursor[self.create_key(i)] = value2 - self.session.commit_transaction() - - if self.long_run_txn: - # Reconcile and remove the obsolete entries. - self.session.checkpoint() - self.evict_cursor(uri, self.nrows) - - # Rollback the long running transaction. - session2.rollback_transaction() - - # Now apply an update at timestamp 10. - for i in range(1, self.nrows): - for retry in self.retry(): - with retry.transaction(commit_timestamp = 10): - cursor[self.create_key(i)] = value2 - - self.session.checkpoint() - - # Ensure that we blew away history store content. - for ts in range(1, 5): - for retry in self.retry(): - with retry.transaction(read_timestamp = ts, rollback = True): - for i in range(1, self.nrows): - if i % 2 == 0: - if self.update_type == 'deletion': - cursor.set_key(self.create_key(i)) - if self.value_format == '8t': - self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_value(), 0) - else: - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) - else: - self.assertEqual(cursor[self.create_key(i)], value2) - else: - self.assertEqual(cursor[self.create_key(i)], value1) - - if self.update_type == 'deletion': - cache_hs_key_truncate = self.get_stat(stat.conn.cache_hs_key_truncate) - self.assertGreater(cache_hs_key_truncate, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare20.py b/src/third_party/wiredtiger/test/suite/test_prepare20.py index 6f219ea2c83..0b541d40730 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare20.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare20.py @@ -27,14 +27,14 @@ # OTHER DEALINGS IN THE SOFTWARE. -# test_prepare20.py +# test_prepare19.py # Check that we can use an application-level log to replay unstable transactions. -import wttest +import wiredtiger, wttest from wtscenario import make_scenarios from helper import simulate_crash_restart -class test_prepare20(wttest.WiredTigerTestCase): +class test_prepare19(wttest.WiredTigerTestCase): # Do write the logs immediately, but don't waste time fsyncing them. conn_config = 'log=(enabled),transaction_sync=(enabled=true,method=none)' @@ -131,7 +131,7 @@ class test_prepare20(wttest.WiredTigerTestCase): # First pass: find prepared txns. Ignore (thus abort) any that didn't even prepare. txns = {} self.lcursor.reset() - for _, op, k, oldv, newv in self.lcursor: + for lsn, op, k, oldv, newv in self.lcursor: if op == self.BEGIN: # "key" is the txnid txns[k] = False @@ -143,7 +143,7 @@ class test_prepare20(wttest.WiredTigerTestCase): writing = False committime = None durabletime = None - for _, op, k, oldv, newv in self.lcursor: + for lsn, op, k, oldv, newv in self.lcursor: if op == self.BEGIN: # "key" is the txnid if txns[k]: @@ -198,9 +198,9 @@ class test_prepare20(wttest.WiredTigerTestCase): # Now the test. - def test_prepare20(self): - data_uri = 'file:prepare20data' - log_uri = 'file:prepare20log' + def test_prepare19(self): + data_uri = 'file:prepare19data' + log_uri = 'file:prepare19log' # Create one table for data and another to be an application-level log. # The log's format is application-lsn -> operation, key, oldvalue, newvalue diff --git a/src/third_party/wiredtiger/test/suite/test_prepare21.py b/src/third_party/wiredtiger/test/suite/test_prepare21.py deleted file mode 100644 index ec61092d04c..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_prepare21.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import threading, time -from helper import simulate_crash_restart -from test_rollback_to_stable01 import test_rollback_to_stable_base -from wiredtiger import stat -from wtdataset import SimpleDataSet -from wtscenario import make_scenarios -from wtthread import checkpoint_thread - -# test_prepare21.py -# Test prepare rollback doesn't crash because of triggering out of order fix. -class test_prepare21(test_rollback_to_stable_base): - - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - scenarios = make_scenarios(format_values) - - def conn_config(self): - config = 'cache_size=10MB,statistics=(all),timing_stress_for_test=[history_store_checkpoint_delay]' - return config - - def evict_cursor(self, uri, nrows): - # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - self.session.begin_transaction("ignore_prepare=true") - for i in range (1, nrows + 1): - evict_cursor.set_key(i) - evict_cursor.search() - evict_cursor.reset() - evict_cursor.close() - self.session.rollback_transaction() - - def test_prepare_rollback(self): - nrows = 10 - - # Create a table. - uri = "table:prepare21" - ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format) - ds.populate() - - if self.value_format == '8t': - value_a = 97 - value_b = 98 - value_c = 99 - value_d = 100 - else: - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - - # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + - ',stable_timestamp=' + self.timestamp_str(10)) - - self.large_updates(uri, value_a, ds, nrows, False, 20) - self.large_updates(uri, value_b, ds, nrows, False, 30) - self.large_removes(uri, ds, nrows, False, 40) - - prepare_session = self.conn.open_session() - prepare_session.begin_transaction() - cursor = prepare_session.open_cursor(uri) - for i in range (1, nrows + 1): - cursor[i] = value_c - cursor.close() - prepare_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) - - # Verify data is visible and correct. - self.check(value_a, uri, nrows, None, 20) - self.check(value_b, uri, nrows, None, 30) - - self.evict_cursor(uri, nrows) - - # Pin stable to timestamp 40. - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) - - # Rollback the prepared update - prepare_session.rollback_transaction() - self.large_updates(uri, value_d, ds, nrows, False, 60) - - done = threading.Event() - ckpt = checkpoint_thread(self.conn, done) - try: - ckpt.start() - - # Wait for checkpoint to start before committing last transaction. - ckpt_started = 0 - while not ckpt_started: - stat_cursor = self.session.open_cursor('statistics:', None, None) - ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] - stat_cursor.close() - - self.evict_cursor(uri, nrows) - finally: - done.set() - ckpt.join() - - # Verify data is visible and correct. - self.check(value_a, uri, nrows, None, 20) - self.check(value_b, uri, nrows, None, 30) - self.check(value_d, uri, nrows, None, 60) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare22.py b/src/third_party/wiredtiger/test/suite/test_prepare22.py deleted file mode 100644 index f35f4a50052..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_prepare22.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios - -# test_prepare22.py -# Test prepare with rollback to stable without failed eviction. -class test_prepare22(wttest.WiredTigerTestCase): - - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - delete = [ - ('delete', dict(delete=True)), - ('non-delete', dict(delete=False)), - ] - - scenarios = make_scenarios(format_values, delete) - - def test_prepare22(self): - uri = "table:test_prepare22" - self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) - - if self.value_format == '8t': - value_a = 97 - value_b = 98 - value_c = 99 - else: - value_a = "a" - value_b = "b" - value_c = "c" - - # Pin oldest timestamp to 1 - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - - # Do the first update - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - cursor[1] = value_a - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - - # Do the second update - self.session.begin_transaction() - cursor[1] = value_b - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) - - if self.delete: - self.session.begin_transaction() - cursor.set_key(1) - cursor.remove() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) - - # Do a prepared update - self.session.begin_transaction() - cursor[1] = value_c - self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) - - # Evict the page - session2 = self.conn.open_session() - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(20)) - self.assertEquals(evict_cursor[1], value_b) - evict_cursor.reset() - evict_cursor.close() - session2.rollback_transaction() - - # Ensure the history store is checkpointed - session2.checkpoint() - - # Rollback the prepared transaction - self.session.rollback_transaction() - - # Set stable timestamp to 30 - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) - - # Call rollback to stable - self.conn.rollback_to_stable() - - # Evict the page again - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - session2.begin_transaction('read_timestamp=' + self.timestamp_str(20)) - self.assertEquals(evict_cursor[1], value_b) - evict_cursor.reset() - evict_cursor.close() - session2.rollback_transaction() - - # Verify we can still read back value a - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(10)) - self.assertEquals(cursor[1], value_a) - self.session.rollback_transaction() - - # Verify we can still read back value b - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) - self.assertEquals(cursor[1], value_b) - self.session.rollback_transaction() - - # Verify we can still read back the deletion - if self.delete: - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) - if self.value_format == '8t': - self.assertEquals(cursor[1], 0) - else: - cursor.set_key(1) - self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) - self.session.rollback_transaction() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare23.py b/src/third_party/wiredtiger/test/suite/test_prepare23.py deleted file mode 100644 index f7496a5d228..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_prepare23.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios - -# test_prepare23.py -# Test prepare rollback with rollback to stable and failed eviction. -class test_prepare23(wttest.WiredTigerTestCase): - conn_config = 'timing_stress_for_test=[failpoint_eviction_fail_after_reconciliation]' - - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - delete = [ - ('delete', dict(delete=True)), - ('non-delete', dict(delete=False)), - ] - - scenarios = make_scenarios(format_values, delete) - - def test_prepare23(self): - uri = "table:test_prepare23" - self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) - - if self.value_format == '8t': - value_a = 97 - value_b = 98 - value_c = 99 - else: - value_a = "a" - value_b = "b" - value_c = "c" - - # Pin oldest timestamp to 1 - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - - cursor = self.session.open_cursor(uri) - session2 = self.conn.open_session() - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - ts = 0 - for i in range (1, 1001): - # Do the first update - self.session.begin_transaction() - cursor[i] = value_a - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 10)) - - # Do the second update - self.session.begin_transaction() - cursor[i] = value_b - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 20)) - - if self.delete: - self.session.begin_transaction() - cursor.set_key(i) - cursor.remove() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 30)) - - # Do a prepared update - self.session.begin_transaction() - cursor[i] = value_c - self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 40)) - cursor.reset() - - # Evict the page - session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 20)) - self.assertEquals(evict_cursor[i], value_b) - evict_cursor.reset() - session2.rollback_transaction() - - # Rollback the prepared transaction - self.session.rollback_transaction() - - # Set stable timestamp to 30 * i - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(ts + 30)) - - # Call rollback to stable - self.conn.rollback_to_stable() - - # Verify we can still read back value a - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 10)) - self.assertEquals(cursor[i], value_a) - self.session.rollback_transaction() - - # Verify we can still read back value b - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 20)) - self.assertEquals(cursor[i], value_b) - self.session.rollback_transaction() - - # Verify we can still read back the deletion - if self.delete: - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 30)) - if self.value_format == '8t': - self.assertEquals(cursor[i], 0) - else: - cursor.set_key(i) - self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) - self.session.rollback_transaction() - - ts += 40 - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare24.py b/src/third_party/wiredtiger/test/suite/test_prepare24.py deleted file mode 100644 index af367af87e5..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_prepare24.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios - -# test_prepare24.py -# Test prepare commit after eviction failure. -class test_prepare24(wttest.WiredTigerTestCase): - conn_config = 'timing_stress_for_test=[failpoint_eviction_fail_after_reconciliation]' - - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - delete = [ - ('delete', dict(delete=True)), - ('non-delete', dict(delete=False)), - ] - - scenarios = make_scenarios(format_values, delete) - - def test_prepare24(self): - uri = "table:test_prepare24" - self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) - - if self.value_format == '8t': - value_a = 97 - value_b = 98 - else: - value_a = "a" - value_b = "b" - - # Pin oldest timestamp to 1 - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - - cursor = self.session.open_cursor(uri) - session2 = self.conn.open_session() - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - ts = 0 - for i in range (1, 1001): - # Insert a value - self.session.begin_transaction() - cursor[i] = value_a - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 10)) - - if self.delete: - self.session.begin_transaction() - cursor.set_key(i) - cursor.remove() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 20)) - - # Do a prepared update - self.session.begin_transaction() - cursor[i] = value_b - self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 30)) - cursor.reset() - - # Evict the page - session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) - self.assertEquals(evict_cursor[i], value_a) - evict_cursor.reset() - session2.rollback_transaction() - - # Commit the prepared transaction - self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(ts + 30)) - self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(ts + 40)) - self.session.commit_transaction() - - # Evict the page again - session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) - self.assertEquals(evict_cursor[i], value_a) - evict_cursor.reset() - session2.rollback_transaction() - - # Verify we can still read back value a - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 10)) - self.assertEquals(cursor[i], value_a) - self.session.rollback_transaction() - - # Verify we can still read back the deletion - if self.delete: - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 20)) - if self.value_format == '8t': - self.assertEquals(cursor[i], 0) - else: - cursor.set_key(i) - self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) - self.session.rollback_transaction() - - # Verify we can still read back the prepared update - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 30)) - self.assertEquals(cursor[i], value_b) - self.session.rollback_transaction() - - ts += 40 - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare25.py b/src/third_party/wiredtiger/test/suite/test_prepare25.py deleted file mode 100644 index e512c3c7fef..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_prepare25.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios - -# test_prepare25.py -# Test prepare rollback and then prepare commit with failed eviction. -class test_prepare25(wttest.WiredTigerTestCase): - conn_config = 'timing_stress_for_test=[failpoint_eviction_fail_after_reconciliation]' - - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - delete = [ - ('delete', dict(delete=True)), - ('non-delete', dict(delete=False)), - ] - - scenarios = make_scenarios(format_values, delete) - - def test_prepare25(self): - uri = "table:test_prepare25" - self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) - - if self.value_format == '8t': - value_a = 97 - value_b = 98 - value_c = 99 - else: - value_a = "a" - value_b = "b" - value_c = "c" - - # Pin oldest timestamp to 1 - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - - cursor = self.session.open_cursor(uri) - session2 = self.conn.open_session() - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - ts = 0 - for i in range (1, 1001): - # Insert an update - self.session.begin_transaction() - cursor[i] = value_a - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 10)) - - if self.delete: - self.session.begin_transaction() - cursor.set_key(i) - cursor.remove() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 20)) - - # Do a prepared update - self.session.begin_transaction() - cursor[i] = value_b - self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 30)) - cursor.reset() - - # Evict the page - session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) - self.assertEquals(evict_cursor[i], value_a) - evict_cursor.reset() - session2.rollback_transaction() - - # Rollback the prepared transaction - self.session.rollback_transaction() - - # Do another prepared update - self.session.begin_transaction() - cursor[i] = value_c - self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 40)) - - # Commit the prepared update - self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(ts + 40)) - self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(ts + 50)) - self.session.commit_transaction() - cursor.reset() - - # Evict the page again - session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) - self.assertEquals(evict_cursor[i], value_a) - evict_cursor.reset() - session2.rollback_transaction() - - # Verify we can still read back value a - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 10)) - self.assertEquals(cursor[i], value_a) - self.session.rollback_transaction() - - # Verify we can still read back the deletion - if self.delete: - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 20)) - if self.value_format == '8t': - self.assertEquals(cursor[i], 0) - else: - cursor.set_key(i) - self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) - self.session.rollback_transaction() - - # Verify we can still read back value c - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 40)) - self.assertEquals(cursor[i], value_c) - self.session.rollback_transaction() - - ts += 50 - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare26.py b/src/third_party/wiredtiger/test/suite/test_prepare26.py deleted file mode 100644 index 52e08cedff6..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_prepare26.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios - -# test_prepare26.py -# Test prepare rollback and then delete the key. -class test_prepare26(wttest.WiredTigerTestCase): - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - scenarios = make_scenarios(format_values) - - def test_prepare26(self): - uri = "table:test_prepare26" - self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) - - if self.value_format == '8t': - value_a = 97 - value_b = 98 - value_c = 99 - else: - value_a = "a" - value_b = "b" - value_c = "c" - - # Pin oldest timestamp to 1 - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - - # Insert a value - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - cursor[1] = value_a - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - - # Do a prepared update - self.session.begin_transaction() - cursor[1] = value_c - self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) - - # Evict the page - session2 = self.conn.open_session() - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(10)) - self.assertEquals(evict_cursor[1], value_a) - evict_cursor.reset() - evict_cursor.close() - session2.rollback_transaction() - - # Rollback the prepared transaction - self.session.rollback_transaction() - - # Delete the key - self.session.begin_transaction() - cursor.set_key(1) - cursor.remove() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) - - # Set oldest timestamp to 30 - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30)) - - # Evict the page again - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - session2.begin_transaction() - evict_cursor.set_key(1) - if self.value_format == '8t': - self.assertEquals(evict_cursor[1], 0) - else: - evict_cursor.set_key(1) - self.assertEquals(evict_cursor.search(), wiredtiger.WT_NOTFOUND) - evict_cursor.reset() - evict_cursor.close() - session2.rollback_transaction() - - # Do another update - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - cursor[1] = value_b - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) - - # Do another update - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - cursor[1] = value_c - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) - - # Evict the page again - evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') - session2.begin_transaction('read_timestamp=' + self.timestamp_str(50)) - self.assertEquals(evict_cursor[1], value_c) - evict_cursor.reset() - evict_cursor.close() - session2.rollback_transaction() - - # Verify we read nothing at the oldest - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) - if self.value_format == '8t': - self.assertEquals(cursor[1], 0) - else: - cursor.set_key(1) - self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) - self.session.rollback_transaction() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare28.py b/src/third_party/wiredtiger/test/suite/test_prepare28.py deleted file mode 100644 index 55946188089..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_prepare28.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -from time import sleep -import wttest, threading - -# Prior to a bugfix in WiredTiger it was possible to read a partial transaction if the config -# ignore prepare was provided. This test demonstrates that case. -class test_prepare28(wttest.WiredTigerTestCase): - conn_config= 'timing_stress_for_test=[prepare_resolution_2]' - uri = 'table:test_prepare28' - numrows = 1 - value1 = 'aaaaa' - value2 = 'bbbbb' - value3 = 'ccccc' - - def test_ignore_prepare(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - cursor = self.session.open_cursor(self.uri) - # Prepare a value at timestamp 4 - self.session.begin_transaction() - cursor[1] = self.value1 - cursor[1] = self.value2 - cursor[1] = self.value3 - self.session.prepare_transaction('prepare_timestamp=4') - # Create a thread. - ooo_thread = threading.Thread(target=self.read_update) - # Start the thread - ooo_thread.start() - self.session.commit_transaction('commit_timestamp=6,durable_timestamp=6') - - def read_update(self): - sleep(0.1) - session = self.setUpSessionOpen(self.conn) - cursor = session.open_cursor(self.uri) - session.begin_transaction('ignore_prepare=true') - cursor.set_key(1) - session.breakpoint() - # Read here - ret = cursor.search() - # Assert it didn't find anything, i.e. not WT_NOTFOUND - assert(ret == -31803) - session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py index 8713ac2c32b..430032cc344 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py @@ -151,7 +151,5 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): nkeys = 4000 self.prepare_updates(uri, ds, nrows, nsessions, nkeys) - self.ignoreStdoutPatternIfExists('Eviction took more than 1 minute') - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_recovery01.py b/src/third_party/wiredtiger/test/suite/test_recovery01.py deleted file mode 100644 index 4e8dd46bd1e..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_recovery01.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wttest -from helper import simulate_crash_restart -from wtdataset import SimpleDataSet -from wiredtiger import stat -from wtscenario import make_scenarios - -# test_recovery01.py -# Test WiredTiger logs time spent during recovery and shutdown. -class test_recovery01(wttest.WiredTigerTestCase): - - format_values = [ - ('column', dict(key_format='r', value_format='S')), - ('column_fix', dict(key_format='r', value_format='8t')), - ('row_integer', dict(key_format='i', value_format='S')), - ] - - restart_values = [ - ('crash', dict(crash=True)), - ('shutdown', dict(crash=False)) - ] - - scenarios = make_scenarios(format_values, restart_values) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # BACKPORT remove for CI - #self.ignoreStdoutPattern('WT_VERB_RECOVERY_PROGRESS') - - def conn_config(self): - # BACKPORT modify for CI - #config = 'cache_size=50MB,statistics=(all),log=(enabled=true),verbose=(recovery_progress)' - config = 'cache_size=50MB,statistics=(all),log=(enabled=true)' - return config - - def large_updates(self, uri, value, ds, nrows, commit_ts): - # Update a large number of records. - session = self.session - cursor = session.open_cursor(uri) - for i in range(1, nrows+1): - session.begin_transaction() - cursor[ds.key(i)] = value - if commit_ts == 0: - session.commit_transaction() - else: - session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) - cursor.close() - - def check(self, check_value, uri, nrows, read_ts): - session = self.session - if read_ts == 0: - session.begin_transaction() - else: - session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) - cursor = session.open_cursor(uri) - count = 0 - for k, v in cursor: - self.assertEqual(v, check_value) - count += 1 - session.commit_transaction() - self.assertEqual(count, nrows) - - def test_recovery(self): - nrows = 1000 - - # Create two tables. One logged and another one non-logged. - uri_1 = "table:recovery01_1" - ds_1 = SimpleDataSet( - self, uri_1, 0, key_format=self.key_format, value_format=self.value_format, - config='log=(enabled=true)') - ds_1.populate() - - uri_2 = "table:recovery01_2" - ds_2 = SimpleDataSet( - self, uri_2, 0, key_format=self.key_format, value_format=self.value_format, - config='log=(enabled=false)') - ds_2.populate() - - if self.value_format == '8t': - valuea = 97 - valueb = 98 - else: - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 - - # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + - ',stable_timestamp=' + self.timestamp_str(1)) - - self.large_updates(uri_1, valuea, ds_1, nrows, 0) - self.check(valuea, uri_1, nrows, 0) - - self.large_updates(uri_2, valuea, ds_2, nrows, 10) - self.check(valuea, uri_2, nrows, 10) - - self.large_updates(uri_1, valueb, ds_1, nrows, 0) - self.check(valueb, uri_1, nrows, 0) - - self.large_updates(uri_2, valueb, ds_2, nrows, 20) - self.check(valueb, uri_2, nrows, 20) - - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) - self.session.checkpoint() - - if self.crash: - simulate_crash_restart(self, ".", "RESTART") - else: - self.reopen_conn() - - self.check(valueb, uri_1, nrows, 0) - self.check(valuea, uri_2, nrows, 10) - self.check(valuea, uri_2, nrows, 20) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py index a65f7c5ff79..6bc9cda70a4 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py @@ -55,12 +55,7 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - evict = [ - ('no_evict', dict(evict=False)), - ('evict', dict(evict=True)) - ] - - scenarios = make_scenarios(format_values, in_memory_values, prepare_values, evict) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=500MB,statistics=(all)' @@ -117,11 +112,6 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40) self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50) - - # Evict the pages to disk - if self.evict: - self.evict_cursor(uri, nrows, value_modS) - self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) self.large_updates(uri, value_c, ds, nrows, self.prepare, 70) self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 80) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py index c9eb37918c5..676bf934b6e 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py @@ -52,12 +52,7 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - evict = [ - ('no_evict', dict(evict=False)), - ('evict', dict(evict=True)) - ] - - scenarios = make_scenarios(format_values, in_memory_values, prepare_values, evict) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -134,21 +129,5 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): else: self.assertGreaterEqual(upd_aborted + hs_removed + keys_removed, nrows * 4) - # Reinsert the same updates with the same timestamps and flush to disk. - # If the updates have not been correctly removed by RTS WiredTiger will - # see the key already exists in the history store and abort. - self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) - self.large_updates(uri, value_b, ds, nrows, self.prepare, 30) - self.large_updates(uri, value_c, ds, nrows, self.prepare, 40) - self.large_updates(uri, value_d, ds, nrows, self.prepare, 50) - - # Do a checkpoint before shutdown - if not self.in_memory: - self.session.checkpoint() - - # Evict the pages to disk - if self.evict: - self.evict_cursor(uri, nrows, value_d) - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable39.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable39.py deleted file mode 100644 index 90bf63935da..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable39.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -import threading, time -from helper import simulate_crash_restart -from test_rollback_to_stable01 import test_rollback_to_stable_base -from wiredtiger import stat -from wtdataset import SimpleDataSet -from wtscenario import make_scenarios -from wtthread import checkpoint_thread - -# test_rollback_to_stable39.py -# Test to delay checkpoint and perform eviction in parallel to ensure eviction moves the content from data store to history store -# and then checkpoint history store to see the same content in data store and history store. Later use the checkpoint to restore -# the database which will trigger eviction to insert the same record from data store to history store. -class test_rollback_to_stable39(test_rollback_to_stable_base): - restart_config = False - - format_values = [ - ('column', dict(key_format='r', value_format='S', prepare_extraconfig='')), - ('column_fix', dict(key_format='r', value_format='8t', - prepare_extraconfig=',allocation_size=512,leaf_page_max=512')), - ('row_integer', dict(key_format='i', value_format='S', prepare_extraconfig='')), - ] - - prepare_values = [ - ('no_prepare', dict(prepare=False)), - ('prepare', dict(prepare=True)) - ] - - scenarios = make_scenarios(format_values, prepare_values) - - def conn_config(self): - config = 'cache_size=25MB,statistics=(all),statistics_log=(json,on_close,wait=1)' - if self.restart_config: - config += ',timing_stress_for_test=[checkpoint_slow]' - else: - config += ',timing_stress_for_test=[history_store_checkpoint_delay]' - return config - - def test_rollback_to_stable(self): - nrows = 1000 - - # Create a table. - uri = "table:rollback_to_stable39" - ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format=self.value_format) - ds.populate() - - if self.value_format == '8t': - value_a = 97 - value_b = 98 - value_c = 99 - else: - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - - # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + - ',stable_timestamp=' + self.timestamp_str(10)) - - # Perform several updates. - self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) - # Verify data is visible and correct. - self.check(value_a, uri, nrows, None, 21 if self.prepare else 20) - - self.large_removes(uri, ds, nrows, self.prepare, 30) - # Verify no data is visible. - self.check(value_a, uri, 0, nrows, 31 if self.prepare else 30) - - # Pin stable to timestamp 40 if prepare otherwise 30. - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40 if self.prepare else 30)) - - # Create a checkpoint thread - done = threading.Event() - ckpt = checkpoint_thread(self.conn, done) - try: - ckpt.start() - - # Wait for checkpoint to start before committing. - ckpt_started = 0 - while not ckpt_started: - stat_cursor = self.session.open_cursor('statistics:', None, None) - ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] - stat_cursor.close() - time.sleep(1) - - # Perform several updates in parallel with checkpoint. - # Rollbacks may occur when checkpoint is running, so retry as needed. - self.retry_rollback('update ds, e', None, - lambda: self.large_updates(uri, value_b, ds, nrows, self.prepare, 50)) - self.evict_cursor(uri, nrows, value_b) - finally: - done.set() - ckpt.join() - - # Simulate a crash by copying to a new directory(RESTART). - self.restart_config = True - simulate_crash_restart(self, ".", "RESTART") - - # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri, nrows, None, 21 if self.prepare else 20) - self.check(value_a, uri, 0, nrows, 40) - - stat_cursor = self.session.open_cursor('statistics:', None, None) - hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] - hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] - keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] - keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] - upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] - stat_cursor.close() - - self.assertEqual(keys_removed, 0) - self.assertEqual(keys_restored, 0) - self.assertEqual(upd_aborted, 0) - self.assertEqual(hs_removed, 0) - self.assertEqual(hs_sweep, 0) - - # Perform several updates. - self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) - - # Verify data is visible and correct. - self.check(value_c, uri, nrows, None, 61 if self.prepare else 60) - - # Create a checkpoint thread - done = threading.Event() - ckpt = checkpoint_thread(self.conn, done) - try: - ckpt.start() - - # Wait for checkpoint to start before committing. - ckpt_started = 0 - while not ckpt_started: - stat_cursor = self.session.open_cursor('statistics:', None, None) - ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] - stat_cursor.close() - time.sleep(1) - - self.evict_cursor(uri, nrows, value_c) - finally: - done.set() - ckpt.join() - - -if __name__ == '__main__': - wttest.run() - diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable40.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable40.py deleted file mode 100755 index 07df57c2966..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable40.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import fnmatch, os, shutil, time -from helper import copy_wiredtiger_home, simulate_crash_restart -from test_rollback_to_stable01 import test_rollback_to_stable_base -from wiredtiger import stat -from wtdataset import SimpleDataSet -from wtscenario import make_scenarios - -# test_rollback_to_stable40.py -# Test the rollback to stable operation performs as expected following a server crash -# and recovery. Verify that the on-disk value is replaced by the correct value from -# the history store. -class test_rollback_to_stable40(test_rollback_to_stable_base): - session_config = 'isolation=snapshot' - - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), - ] - - scenarios = make_scenarios(key_format_values) - - def conn_config(self): - config = 'cache_size=1MB,statistics=(all),log=(enabled=true)' - return config - - def test_rollback_to_stable(self): - nrows = 3 - - # Create a table without logging. - uri = "table:rollback_to_stable40" - ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') - ds.populate() - - # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + - ',stable_timestamp=' + self.timestamp_str(10)) - - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - - # Insert 3 keys with same updates. - cursor = self.session.open_cursor(uri) - self.session.begin_transaction() - cursor[1] = value_a - cursor[2] = value_a - cursor[3] = value_a - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) - - # Update the first and last key with another value with a large timestamp. - self.session.begin_transaction() - cursor[1] = value_d - cursor[3] = value_d - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1000)) - - # Update the middle key with lot of updates to generate more history. - for i in range(21, 499): - self.session.begin_transaction() - cursor[2] = value_b + str(i) - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(i)) - - # With this checkpoint, all the updates in the history store are persisted to disk. - self.session.checkpoint() - - self.session.begin_transaction() - cursor[2] = value_c - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(500)) - - # Pin oldest and stable to timestamp 500. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(500) + - ',stable_timestamp=' + self.timestamp_str(500)) - - # Evict the globally visible update to write to the disk, this will reset the time window. - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - self.session.begin_transaction("ignore_prepare=true") - evict_cursor.set_key(2) - self.assertEqual(evict_cursor[2], value_c) - evict_cursor.reset() - evict_cursor.close() - self.session.rollback_transaction() - - self.session.begin_transaction() - cursor[2] = value_d - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(501)) - - # 1. This checkpoint will move the globally visible update to the first of the key range. - # 2. The existing updates in the history store are having with a larger timestamp are - # obsolete, so they are not explicitly removed. - # 3. Any of the history store updates that are already evicted will not rewrite by the - # checkpoint. - self.session.checkpoint() - - # Verify data is visible and correct. - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1000)) - for i in range (1, nrows + 1): - cursor.set_key(ds.key(i)) - self.assertEqual(cursor.search(), 0) - self.assertEquals(cursor.get_value(), value_d) - self.session.rollback_transaction() - cursor.close() - - # Simulate a server crash and restart. - simulate_crash_restart(self, ".", "RESTART") - - # Verify data is visible and correct. - cursor = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1000)) - for i in range (1, nrows + 1): - cursor.set_key(ds.key(i)) - self.assertEqual(cursor.search(), 0) - if i % 2 == 0: - self.assertEquals(cursor.get_value(), value_c) - else: - self.assertEquals(cursor.get_value(), value_a) - self.session.rollback_transaction() - - stat_cursor = self.session.open_cursor('statistics:', None, None) - calls = stat_cursor[stat.conn.txn_rts][2] - hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] - keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] - keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] - pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] - upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] - stat_cursor.close() - - self.assertEqual(calls, 0) - self.assertEqual(keys_removed, 0) - self.assertEqual(keys_restored, 0) - self.assertGreaterEqual(upd_aborted, 0) - self.assertGreater(pages_visited, 0) - self.assertGreaterEqual(hs_removed, 3) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_scrub_eviction_prepare.py b/src/third_party/wiredtiger/test/suite/test_scrub_eviction_prepare.py deleted file mode 100644 index 92eabcc4a37..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_scrub_eviction_prepare.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wiredtiger import stat, WiredTigerError - -# test_scrub_eviction_prepare.py -# -# Test to do the following steps. -# 1. Prepare an update with one key (key-2) -# 2. Commit an update with another key(key-1) -# 3, Make sure both the keys are on the same page. -# 4. Set the key to full update and evict the page with release_evict -# 5. Read the page back in memory -# 6. Checkpoint -# 7. Repeat steps 5,6 and validate that the page read back into memory should -# not be reconciled everytime with the help of btree stat. -class test_scrub_eviction_prepare(wttest.WiredTigerTestCase): - - def conn_config(self): - config = 'cache_size=100MB,statistics=(all),statistics_log=(json,on_close,wait=1)' - return config - - def get_stats(self, uri): - stat_cursor = self.session.open_cursor('statistics:' + uri) - btree_ckpt_pages_rec = stat_cursor[stat.dsrc.btree_checkpoint_pages_reconciled][2] - stat_cursor.close() - return btree_ckpt_pages_rec - - def read_key(self, uri): - cur2 = self.session.open_cursor(uri) - cur2.set_key(2) - self.assertEqual(cur2.search(), 0) - cur2.close() - - def test_scrub_eviction_prepare(self): - uri = 'table:test_scrub_eviction_prepare' - - # Create a table. - self.session.create(uri, 'key_format=i,value_format=S') - session2 = self.conn.open_session() - session3 = self.conn.open_session() - cursor2 = session2.open_cursor(uri) - - # Insert a key 2 and commit the transaction. - session2.begin_transaction() - cursor2[2] = '20' - session2.commit_transaction() - - # Insert a key 1 and prepare the transaction. - session3.begin_transaction() - cursor2[1] = '10' - session3.prepare_transaction('prepare_timestamp=10') - - # Leaving a cursor open after updates can affect the release_evict cursor's - # behavior. It might not guarantee eviction because the open cursor pins the - # page in the cache. Therefore, always close the cursor explicitly before using. - cursor2.close() - - # Set the key to 2(to avoid prepare conflict if the key is set to 1) in the evict - # cursor and evict the page which has both the keys 1 and 2. - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - evict_cursor.set_key(2) - self.assertEqual(evict_cursor.search(), 0) - self.assertEqual(evict_cursor.reset(), 0) - evict_cursor.close() - - self.session.checkpoint() - self.assertEqual(1, self.get_stats(uri)) - - # Read the key 2 to avoid prepared conflict, this will bring back the page - # that has both the keys 1 & 2 into the memory. - self.read_key(uri) - self.session.checkpoint() - # The page with prepared update should not be reconciled again. - self.assertEqual(1, self.get_stats(uri)) - - # Read the key 2 to avoid prepared conflict, this will bring back the page - # that has both the keys 1 & 2 into the memory. - self.read_key(uri) - self.session.checkpoint() - # The page with prepared update should not be reconciled again. - self.assertEqual(1, self.get_stats(uri)) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near05.py b/src/third_party/wiredtiger/test/suite/test_search_near05.py deleted file mode 100644 index f54485817c1..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_search_near05.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_search_near05.py -# Search_near with a key past the end. - -import wttest -from wtscenario import make_scenarios - -class test_search_near05(wttest.WiredTigerTestCase): - uri = 'file:test_search_near05' - - key_format_values = [ - ('fix', dict(key_format='r', value_format='8t')), - ('var', dict(key_format='r', value_format='I')), - ('row', dict(key_format='Q', value_format='I')), - ] - - ops = [ - ('update', dict(delete=False)), - ('delete', dict(delete=True)), - ] - - scenarios = make_scenarios(key_format_values, ops) - - def evict(self, value): - evict_cursor = self.session.open_cursor(self.uri, None, "debug=(release_evict)") - self.session.begin_transaction() - for i in range(1, 1001): - v = evict_cursor[i] - self.assertEqual(v, value) - self.assertEqual(evict_cursor.reset(), 0) - self.session.rollback_transaction() - - def test_implicit_record_cursor_insert_next(self): - self.session.create(self.uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format)) - cursor = self.session.open_cursor(self.uri) - value1 = 1 - value2 = 2 - for i in range(1, 1001): - cursor[i] = value1 - - # Do a checkpoint to write everything to the disk image - self.session.checkpoint() - # Evict the data - self.evict(value1) - - # Update or delete the last key - if self.delete: - self.session.begin_transaction() - cursor.set_key(1000) - cursor.remove() - self.session.commit_transaction() - else: - cursor[1000] = value2 - - self.session.begin_transaction() - cursor.set_key(1100) - cursor.search_near() - - if self.delete: - if self.value_format == "8t": - self.assertEqual(cursor.get_key(), 1000) - self.assertEqual(cursor.get_value(), 0) - else: - self.assertEqual(cursor.get_key(), 999) - self.assertEqual(cursor.get_value(), value1) - else: - self.assertEqual(cursor.get_key(), 1000) - self.assertEqual(cursor.get_value(), value2) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near06.py b/src/third_party/wiredtiger/test/suite/test_search_near06.py deleted file mode 100644 index 81f9f853c76..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_search_near06.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_search_near06.py -# Search_near with a key past the end much like search_near_05, but -# this time use timestamps to ensure the update isn't visible. - -import wttest -from wtscenario import make_scenarios - -class test_search_near06(wttest.WiredTigerTestCase): - uri = 'file:test_search_near06' - - key_format_values = [ - ('fix', dict(key_format='r', value_format='8t')), - ('var', dict(key_format='r', value_format='I')), - ('row', dict(key_format='Q', value_format='I')), - ] - - ops = [ - ('update', dict(delete=False)), - ('delete', dict(delete=True)), - ] - - scenarios = make_scenarios(key_format_values, ops) - - def evict(self, value): - evict_cursor = self.session.open_cursor(self.uri, None, "debug=(release_evict)") - self.session.begin_transaction() - for i in range(1, 1001): - v = evict_cursor[i] - self.assertEqual(v, value) - self.assertEqual(evict_cursor.reset(), 0) - self.session.rollback_transaction() - - def test_implicit_record_cursor_insert_next(self): - self.session.create(self.uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format)) - cursor = self.session.open_cursor(self.uri) - value1 = 1 - value2 = 2 - for i in range(1, 1001): - cursor[i] = value1 - - # Do a checkpoint to write everything to the disk image - self.session.checkpoint() - # Evict the data - self.evict(value1) - - # Update or delete the last key - if self.delete: - self.session.begin_transaction() - cursor.set_key(1000) - cursor.remove() - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - else: - self.session.begin_transaction() - cursor[1000] = value2 - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) - cursor.set_key(1100) - cursor.search_near() - - self.assertEqual(cursor.get_key(), 1000) - self.assertEqual(cursor.get_value(), value1) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_stat08.py b/src/third_party/wiredtiger/test/suite/test_stat08.py index 0b833c1934c..7e91932d1d7 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat08.py +++ b/src/third_party/wiredtiger/test/suite/test_stat08.py @@ -33,29 +33,14 @@ import wiredtiger, wttest # Session statistics for bytes read into the cache. class test_stat08(wttest.WiredTigerTestCase): - nentries = 100000 - # Leave the cache size on the default setting to avoid filling up the cache - # too much and triggering unnecessary rollbacks. But make the value fairly - # large to make obvious change to the statistics. - conn_config = 'statistics=(all)' - entry_value = "abcde" * 400 + nentries = 350000 + conn_config = 'cache_size=10MB,statistics=(all)' + entry_value = "abcde" * 40 BYTES_READ = wiredtiger.stat.session.bytes_read READ_TIME = wiredtiger.stat.session.read_time session_stats = { BYTES_READ : "session: bytes read into cache", \ READ_TIME : "session: page read from disk to cache time (usecs)"} - def get_stat(self, stat): - statc = self.session.open_cursor('statistics:session', None, None) - val = statc[stat][2] - statc.close() - return val - - def get_cstat(self, stat): - statc = self.session.open_cursor('statistics:', None, None) - val = statc[stat][2] - statc.close() - return val - def check_stats(self, cur, k): # # Some Windows machines lack the time granularity to detect microseconds. @@ -74,33 +59,13 @@ class test_stat08(wttest.WiredTigerTestCase): self.assertTrue(value > 0) def test_session_stats(self): - # We want to configure for pages to be explicitly evicted when we are done with them so - # that we can correctly verify the statistic measuring bytes read from cache. - self.session = self.conn.open_session("debug=(release_evict_page=true)") - self.session.create("table:test_stat08", "key_format=i,value_format=S") + self.session = self.conn.open_session() + self.session.create("table:test_stat08", + "key_format=i,value_format=S") cursor = self.session.open_cursor('table:test_stat08', None, None) - self.session.begin_transaction() - txn_dirty = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) - cache_dirty = self.get_cstat(wiredtiger.stat.conn.cache_bytes_dirty) - self.assertEqual(txn_dirty, 0) - self.assertLessEqual(txn_dirty, cache_dirty) # Write the entries. - for i in range(1, self.nentries): - txn_dirty_before = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) + for i in range(0, self.nentries): cursor[i] = self.entry_value - txn_dirty_after = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) - self.assertLess(txn_dirty_before, txn_dirty_after) - # Since we're using an explicit transaction, we need to resolve somewhat frequently. - # So check the statistics and restart the transaction every 200 operations. - if i % 200 == 0: - cache_dirty_txn = self.get_cstat(wiredtiger.stat.conn.cache_bytes_dirty) - # Make sure the txn's dirty bytes doesn't exceed the cache. - self.assertLessEqual(txn_dirty_after, cache_dirty_txn) - self.session.rollback_transaction() - self.session.begin_transaction() - txn_dirty = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) - self.assertEqual(txn_dirty, 0) - self.session.commit_transaction() cursor.reset() # Read the entries. diff --git a/src/third_party/wiredtiger/test/suite/test_sweep04.py b/src/third_party/wiredtiger/test/suite/test_sweep04.py deleted file mode 100755 index a741639f82c..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_sweep04.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_sweep04.py -# Test lots of tables with more steadily created and dropped. -# A core group of tables is used most often and hangs around. -# Test that the total number of dhandles, while increasing, -# starts to level off, that is, the sweeps are keeping up. -# Then test that if we only access the core tables for a while, -# the total number of dhandles comes back down to a small number. - -import time -from suite_random import suite_random -from wiredtiger import stat -import wttest - -# Given a set of values corresponding to successive times, -# we have an implied set of points in two dimensions. -# Compute the average for the values and the slope for the -# least squares regression of the line. -# -# We'd use numpy, but we aren't assured that it is always installed -# for our python. -def average_slope(y): - n = len(y) - if n == 0: - return [0, 0] # there's no average or slope - elif n == 1: - return [y[0], 0] # there's no slope - - average = sum(y) / n - x = range(1, n + 1) # The implied x axis, steadily increasing integers. - - # Here's a formula for least squares slope: - # https://www.mathsisfun.com/data/least-squares-regression.html - top = n * sum(x[i] * y[i] for i in range(n)) - sum(x) * sum(y) - bottom = n * sum(x[i]**2 for i in range(n)) - sum(x)**2 - slope = top / bottom - return [average, slope] - -@wttest.longtest("lots of files") -class test_sweep04(wttest.WiredTigerTestCase): - tablebase = 'test_sweep04' - uri = 'table:' + tablebase - - # Configuration values for the run. If any of these values are changed, - # the simulation will run, but the acceptance criteria may fail at the end. - core_tables = 10 # Number of core tables that always exist. - transient_tables = 100 # Number of transient tables at any time. - transient_examined = 10 # Number of transient tables opened at a time. - ratio_examined = 0.01 # The chance that transient tables are examined. - transient_table_max = 10000 # Defines the length of the run. - numkv = 1 # Number of k/v pairs. Shouldn't matter for this test. - nsessions = 100 # Number of sessions in our pool. - - conn_config = 'file_manager=(close_handle_minimum=0,' + \ - 'close_idle_time=3,close_scan_interval=1),' + \ - 'statistics=(fast),operation_tracking=(enabled=false),' - - create_params = 'key_format=i,value_format=i' - - # Create a uri for one of the core tables - def core_uri(self, i): - return '%s-c.%d' % (self.uri, i) - - # Create a uri for one of the transient tables - def transient_uri(self, i): - return '%s-t.%d' % (self.uri, i) - - def create_table(self, uri): - self.session.create(uri, self.create_params) - c = self.session.open_cursor(uri, None) - for k in range(self.numkv): - c[k+1] = 1 - c.close() - - # Access a set of tables in some minimal way that ensures its - # dhandle is at least momentarily in use. "uri_maker" is a - # function that is used to create the uri. - def examine(self, session, uri_maker, start, count): - for i in range(0, count): - c = session.open_cursor(uri_maker(start + i)) - self.assertEquals(c[1], 1) - c.close() - - def test_big_run(self): - # populate - r = suite_random() - - # Create the set of core tables - for i in range(0, self.core_tables): - self.create_table(self.core_uri(i)) - - created_transient = 0 # Running total of transients created. - available_transient = 0 # The next available transient number. - - # Create the initial batch of transient tables. - while created_transient < self.transient_tables: - self.create_table(self.transient_uri(created_transient)) - created_transient += 1 - - # Open all the session we'll use in advance. - sessions = [] - for i in range(self.nsessions): - sessions.append(self.conn.open_session()) - - # We keep the dhandle counts for each time we get stats. - dhandle_counts = [] - - # The big loop: For half the run, we are stressing by accessing both core tables - # and transient tables, and creating/dropping transient tables. The second half - # of the run, we stop creating/dropping tables and only access core tables - # just to see if all outstanding dhandles are swept. - maxloop = self.transient_table_max * 2 - lasttime = time.time() - for loopcount in range(0, maxloop): - stressing = (created_transient < self.transient_table_max) - if loopcount % 100 == 0: - self.pr('{}/{} stressing={}'.format(loopcount, maxloop, stressing)) - - # Make sure at least 3 seconds elapses between each 100 times through - # the loop, to give the various sweeps time to operate. - thistime = time.time() - delta = thistime - lasttime - if delta < 3.0: - time.sleep(3.0 - delta) - lasttime = thistime - - if stressing: - self.session.drop(self.transient_uri(available_transient), "force") - available_transient += 1 - self.create_table(self.transient_uri(created_transient)) - created_transient += 1 - - rand_session = sessions[r.rand_range(0, self.nsessions)] - - # In the stress part of the run, some small fraction (given by ratio_examined) will - # look at the transient tables. Looking at these rarely makes them candidates for - # closing by the connection sweep. - big = 1000000 # Any large number works. - if stressing and r.rand32() % big > big * self.ratio_examined: - # Access "count" transient tables, starting at table "tnum". - count = self.transient_examined - tnum = r.rand_range(available_transient, available_transient + self.transient_tables - count) - self.examine(rand_session, self.transient_uri, tnum, count) - else: - # Access a single core table, numbered "tnum". - tnum = r.rand_range(0, self.core_tables) - self.examine(rand_session, self.core_uri, tnum, 1) - - if loopcount % 100 == 99: - # Gather statistics about the number of dhandles, to be checked at - # the end of the run. - stat_cursor = self.session.open_cursor('statistics:', None, None) - - # Enable for detailed output. - if False: - close = stat_cursor[stat.conn.dh_sweep_close][2] - remove = stat_cursor[stat.conn.dh_sweep_remove][2] - sweep = stat_cursor[stat.conn.dh_sweeps][2] - sclose = stat_cursor[stat.conn.dh_session_handles][2] - ssweep = stat_cursor[stat.conn.dh_session_sweeps][2] - tod = stat_cursor[stat.conn.dh_sweep_tod][2] - ref = stat_cursor[stat.conn.dh_sweep_ref][2] - self.pr(('DHANDLE STATS: close={}, remove={}, sweep={}, session_handles={}, '+ - 'session_sweeps={}, sweep_tod={}, sweep_ref={}').format( - close, remove, sweep, sclose, ssweep, tod, ref)) - - dhandles = stat_cursor[stat.conn.dh_conn_handle_count][2] - files_open = stat_cursor[stat.conn.file_open][2] - self.pr(' dhandle_count={}'.format(dhandles)) - self.pr(' file_open={}'.format(files_open)) - dhandle_counts.append(dhandles) - stat_cursor.close() - - # This (extremely verbose) debugging is disabled, as it writes to stdout, - # and that causes the test framework fail the test. - if False: - if loopcount % 1000 == 999: - self.conn.debug_info('handles=true') - self.pr(' average,slope={}'.format(str(average_slope(dhandle_counts)))) - - # Reset any sessions we've used. This may be necessary to trigger some session sweeps. - rand_session.reset() - self.session.reset() - - # The run is finished, process and check the dhandle counts we've collected. - # - # Half the run is dhandle growth, and the second half should see a decline. - # If everything is working right in the first half of the run, we should start - # to see the number of dhandles start to reach an asymptote. So we check the slope of - # the dhandle line in the second quarter to see that is the case. In the second - # half of the run, where we aren't accessing the transient tables, those references - # should free up, and we should see an asymptote at the end much closer to the - # number of files. - half = len(dhandle_counts)//2 - qtr = len(dhandle_counts)//4 - tenth = len(dhandle_counts)//10 - - (q1_avg, q1_slope) = average_slope(dhandle_counts[0:qtr]) - (q2_avg, q2_slope) = average_slope(dhandle_counts[qtr:half]) - (q3_avg, q3_slope) = average_slope(dhandle_counts[half:-qtr]) - (q4_avg, q4_slope) = average_slope(dhandle_counts[-qtr:]) - (end_run_avg, end_run_slope) = average_slope(dhandle_counts[-tenth:]) - - self.pr('1st qtr: average={},slope={}'.format(q1_avg, q1_slope)) - self.pr('2nd qtr: average={},slope={}'.format(q2_avg, q2_slope)) - self.pr('3rd qtr: average={},slope={}'.format(q3_avg, q3_slope)) - self.pr('4th qtr: average={},slope={}'.format(q4_avg, q4_slope)) - - self.pr('end run: average={},slope={}'.format(end_run_avg, end_run_slope)) - - # Note, we don't check the first half average, it's likely to be big, but its size - # depends on many factors. The important thing is that the slope has flattened out. - # Even with variations due to sweep timing, the slope shouldn't be greater than - # 15.0 (dhandles per 100 times through the loop). - self.assertLess(abs(q2_slope), q1_slope) - self.assertLess(abs(q2_slope), 15.0) - - # At the end of the run, we expect a pretty flat slope and a pretty small number - # of dhandles. A slope of 5.0 (dhandles per 100 times though the loop) is rather - # flat and still leaves room for some variation. - self.assertLess(abs(end_run_slope), 5.0) - self.assertLess(end_run_avg, self.core_tables + self.transient_tables + 20) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py index c109596361f..8b638788736 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -71,7 +71,8 @@ class test_tiered05(wttest.WiredTigerTestCase): 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ - 'name=%s)' % self.ss_name + 'name=%s,' % self.ss_name + \ + 'object_target_size=20M)' # Test calling the flush_tier API with a tiered manager. Should get an error. def test_tiered(self): diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index d876b305c75..c4e931c7a3f 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -58,7 +58,7 @@ class test_tiered06(wttest.WiredTigerTestCase): config = '' # S3 store is built as an optional loadable extension, not all test environments build S3. if self.ss_name == 's3_store': - #config = '=(config=\"(verbose=[api:1,version,tiered:1])\")' + #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' @@ -104,14 +104,22 @@ class test_tiered06(wttest.WiredTigerTestCase): self.get_fs_config(prefix)) # The object doesn't exist yet. - self.assertFalse(fs.fs_exist(session, 'foobar')) + if self.ss_name == 's3_store': + with self.expectedStderrPattern('.*HTTP response code: 404.*'): + self.assertFalse(fs.fs_exist(session, 'foobar')) + else: + self.assertFalse(fs.fs_exist(session, 'foobar')) # We cannot use the file system to create files, it is readonly. # So use python I/O to build up the file. f = open('foobar', 'wb') # The object still doesn't exist yet. - self.assertFalse(fs.fs_exist(session, 'foobar')) + if self.ss_name == 's3_store': + with self.expectedStderrPattern('.*HTTP response code: 404.*'): + self.assertFalse(fs.fs_exist(session, 'foobar')) + else: + self.assertFalse(fs.fs_exist(session, 'foobar')) outbytes = ('MORE THAN ENOUGH DATA\n'*100000).encode() f.write(outbytes) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered14.py b/src/third_party/wiredtiger/test/suite/test_tiered14.py index c041bb11635..61e3a54cf91 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered14.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered14.py @@ -28,7 +28,7 @@ from helper_tiered import generate_s3_prefix, get_auth_token, get_bucket1_name from wtscenario import make_scenarios -import os, random, wttest +import os, random, wtscenario, wttest from wtdataset import TrackedSimpleDataSet, TrackedComplexDataSet # test_tiered14.py @@ -69,7 +69,7 @@ class test_tiered14(wttest.WiredTigerTestCase): num_ops = 20, ss_name = 's3_store')), ] - scenarios = make_scenarios(multiplier, keyfmt, dataset, storage_sources) + scenarios = wtscenario.make_scenarios(multiplier, keyfmt, dataset, storage_sources) def conn_config(self): if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): @@ -86,7 +86,7 @@ class test_tiered14(wttest.WiredTigerTestCase): config = '' # S3 store is built as an optional loadable extension, not all test environments build S3. if self.ss_name == 's3_store': - #config = '=(config=\"(verbose=[api:1,version,tiered:-3])\")' + #config = '=(config=\"(verbose=1)\")' extlist.skip_if_missing = True #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' diff --git a/src/third_party/wiredtiger/test/suite/test_txn13.py b/src/third_party/wiredtiger/test/suite/test_txn13.py index 6c7529fc250..6b2ed77e4ac 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn13.py +++ b/src/third_party/wiredtiger/test/suite/test_txn13.py @@ -91,7 +91,6 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): else: self.session.commit_transaction() - self.ignoreStdoutPatternIfExists('Eviction took more than 1 minute') self.assertTrue(gotException == self.expect_err) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py index 37804eb327b..af0ae6e5f11 100755 --- a/src/third_party/wiredtiger/test/suite/test_verify.py +++ b/src/third_party/wiredtiger/test/suite/test_verify.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, re, struct +import os, struct from suite_subprocess import suite_subprocess import wiredtiger, wttest @@ -63,15 +63,6 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(i, self.nentries) cursor.close() - def count_file_contains(self, filename, content): - count = 0 - with open(filename) as f: - for line in f: - if content in line: - count += 1 - f.close() - return count - def open_and_position(self, tablename, pct): """ Open the file for the table, position it at a 4K page @@ -86,30 +77,13 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): filename = tablename + ".wt" filesize = os.path.getsize(filename) - position = int((filesize * pct) // 100) + position = (filesize * pct) // 100 self.pr('damaging file at: ' + str(position)) fp = open(filename, "r+b") fp.seek(position) return fp - def open_and_offset(self, tablename, offset): - """ - Open the file for the table, position it at the given offset. - As a side effect, the connection is closed. - """ - # we close the connection to guarantee everything is - # flushed and closed from the WT point of view. - if self.conn != None: - self.conn.close() - self.conn = None - filename = tablename + ".wt" - - self.pr('damaging file at: ' + str(offset)) - fp = open(filename, "r+b") - fp.seek(offset) - return fp - def test_verify_process_empty(self): """ Test verify in a 'wt' process, using an empty table @@ -163,67 +137,8 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.conn = self.setUpConnectionOpen(".") self.session = self.setUpSessionOpen(self.conn) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.verify('table:' + self.tablename, "read_corrupt"), + lambda: self.session.verify('table:' + self.tablename, None), "/WT_SESSION.verify/") - self.assertEqual(self.count_file_contains("stderr.txt", - "calculated block checksum doesn't match expected checksum"), 1) - - def test_verify_api_read_corrupt_pages(self): - """ - Test verify via API, on a table that is purposely corrupted in - multiple places. A verify operation with read_corrupt on should - result in multiple checksum errors being logged. - """ - params = 'key_format=S,value_format=S' - self.session.create('table:' + self.tablename, params) - self.populate(self.tablename) - with self.open_and_position(self.tablename, 25) as f: - for i in range(0, 100): - f.write(b'\x01\xff\x80') - with self.open_and_position(self.tablename, 50) as f: - for i in range(0, 100): - f.write(b'\x01\xff\x80') - with self.open_and_position(self.tablename, 75) as f: - for i in range(0, 100): - f.write(b'\x01\xff\x80') - - def test_verify_api_corrupt_first_page(self): - """ - Test that verify works when the first child of an internal node is corrupted. A verify - operation with read_corrupt on should result in a checksum errors being logged. - """ - params = 'key_format=S,value_format=S' - self.session.create('table:' + self.tablename, params) - self.populate(self.tablename) - - # wt verify -d dump_address performs a depth-first traversal of the BTree. So the first - # leaf page it prints is the first child of its parent. Grab the offset of this one so we - # can corrupt it. - self.runWt(['verify', '-d', 'dump_address', 'table:' + self.tablename], outfilename='dump.out') - - # Grab the offset position of the first page. - offset = 0 - lines = open('dump.out').readlines() - for line in lines: - m = re.search('(\d+)-(\d+).*row-store leaf', line) - if m: - offset = int((int(m.group(2)) - int(m.group(1)))/2) - break - - # Open the file and corrupt the first page. - with self.open_and_offset(self.tablename, offset) as f: - for i in range(0, 100): - f.write(b'\x01\xff\x80') - - # open_and_position closed the session/connection, reopen them now. - self.conn = self.setUpConnectionOpen(".") - self.session = self.setUpSessionOpen(self.conn) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.verify('table:' + self.tablename, "read_corrupt"), - "/WT_SESSION.verify/") - - self.assertEqual(self.count_file_contains("stderr.txt", - "calculated block checksum doesn't match expected checksum"), 1) def test_verify_process_75pct_null(self): """ @@ -236,11 +151,9 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): with self.open_and_position(self.tablename, 75) as f: for i in range(0, 4096): f.write(struct.pack('B', 0)) - self.runWt(["verify", "-c", "table:" + self.tablename], + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) self.check_non_empty_file("verifyerr.out") - self.assertEqual(self.count_file_contains("verifyerr.out", - "calculated block checksum doesn't match expected checksum"), 1) def test_verify_process_25pct_junk(self): """ @@ -253,43 +166,10 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): with self.open_and_position(self.tablename, 25) as f: for i in range(0, 100): f.write(b'\x01\xff\x80') - self.runWt(["verify", "-c", "table:" + self.tablename], - errfilename="verifyerr.out", failure=True) - self.check_non_empty_file("verifyerr.out") - self.assertEqual(self.count_file_contains("verifyerr.out", - "calculated block checksum doesn't match expected checksum"), 1) - - def test_verify_process_read_corrupt_pages(self): - """ - Test verify in a 'wt' process on a table that is purposely corrupted - in multiple places. A verify operation with read_corrupt on should - result in multiple checksum errors being logged. - """ - params = 'key_format=S,value_format=S' - self.session.create('table:' + self.tablename, params) - self.populate(self.tablename) - with self.open_and_position(self.tablename, 25) as f: - for i in range(0, 100): - f.write(b'\x01\xff\x80') - with self.open_and_position(self.tablename, 75) as f: - for i in range(0, 100): - f.write(b'\x01\xff\x80') - with self.open_and_position(self.tablename, 80) as f: - for i in range(0, 100): - f.write(b'\x01\xff\x80') - self.runWt(["verify", "-c", "table:" + self.tablename], + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) - self.check_non_empty_file("verifyerr.out") - # It is expected that more than one checksum error is logged given - # that we have corrupted the table in multiple locations, but we may - # not necessarily detect all three corruptions - e.g. we won't detect - # a corruption if we overwrite free space or overwrite a page that is - # a child of another page that we overwrite. - self.assertGreater(self.count_file_contains("verifyerr.out", - "calculated block checksum doesn't match expected checksum"), 1) - def test_verify_process_truncated(self): """ Test verify in a 'wt' process on a table that is purposely damaged, diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index a751231f334..edd22e346c7 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -42,7 +42,7 @@ except ImportError: import unittest from contextlib import contextmanager -import errno, glob, os, re, shutil, sys, threading, time, traceback, types +import errno, glob, os, re, shutil, sys, time, traceback import wiredtiger, wtscenario, wthooks def shortenWithEllipsis(s, maxlen): @@ -180,38 +180,6 @@ class ExtensionList(list): ext = '' if extarg == None else '=' + extarg self.append(dirname + '/' + name + ext) -# Custom result class that will prefix the pid in text output (including if it's a child). -# Only enabled when we are in verbose mode so we don't check that here. -class PidAwareTextTestResult(unittest.TextTestResult): - _thread_prefix = threading.local() - - def __init__(self, stream, descriptions, verbosity): - super(PidAwareTextTestResult, self).__init__(stream, descriptions, verbosity) - self._thread_prefix.value = "[pid:{}]: ".format(os.getpid()) - - def tags(self, new_tags, gone_tags): - # We attach the PID to the thread so we only need the new_tags. - for tag in new_tags: - if tag.startswith("pid:"): - pid = tag[len("pid:"):] - self._thread_prefix.value = "[pid:{}/{}]: ".format(os.getpid(), pid) - - def startTest(self, test): - self.stream.write(self._thread_prefix.value) - super(PidAwareTextTestResult, self).startTest(test) - - def getDescription(self, test): - return str(test.shortDescription()) - - def printErrorList(self, flavour, errors): - for test, err in errors: - self.stream.writeln(self.separator1) - self.stream.writeln("%s%s: %s" % (self._thread_prefix.value, - flavour, self.getDescription(test))) - self.stream.writeln(self.separator2) - self.stream.writeln("%s%s" % (self._thread_prefix.value, err)) - self.stream.flush() - class WiredTigerTestCase(unittest.TestCase): _globalSetup = False _printOnceSeen = {} @@ -568,11 +536,8 @@ class WiredTigerTestCase(unittest.TestCase): # In addition, check to make sure exc_info is "clean", because # the ConcurrencyTestSuite in Python2 indicates failures using that. if hasattr(self, '_outcome'): # Python 3.4+ - if hasattr(self._outcome, 'errors'): # Python 3.4 - 3.10 - result = self.defaultTestResult() # these 2 methods have no side effects - self._feedErrorsToResult(result, self._outcome.errors) - else: # Python 3.11+ - result = self._outcome.result + result = self.defaultTestResult() # these 2 methods have no side effects + self._feedErrorsToResult(result, self._outcome.errors) else: # Python 3.2 - 3.3 or 3.0 - 3.1 and 2.7 result = getattr(self, '_outcomeForDoCleanups', self._resultForDoCleanups) error = self.list2reason(result, 'errors') @@ -621,9 +586,9 @@ class WiredTigerTestCase(unittest.TestCase): elapsed = time.time() - self.starttime if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: - print("[pid:{}]: {}: {:.2f} seconds".format(os.getpid(), str(self), elapsed)) + print("%s: %.2f seconds" % (str(self), elapsed)) if (not passed) and (not self.skipped): - print("[pid:{}]: ERROR in {}".format(os.getpid(), str(self))) + print("ERROR in " + str(self)) self.pr('FAIL') self.pr('preserving directory ' + self.testdir) if WiredTigerTestCase._verbose > 2: @@ -869,7 +834,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def prout(s): - os.write(WiredTigerTestCase._dupout, str.encode("[pid:{}]: {}\n".format(os.getpid(), s))) + os.write(WiredTigerTestCase._dupout, str.encode(s + '\n')) def pr(self, s): """ @@ -908,7 +873,7 @@ class WiredTigerTestCase(unittest.TestCase): def tty(message): if WiredTigerTestCase._ttyDescriptor == None: WiredTigerTestCase._ttyDescriptor = open('/dev/tty', 'w') - WiredTigerTestCase._ttyDescriptor.write("[pid:{}]: {}\n".format(os.getpid(), message)) + WiredTigerTestCase._ttyDescriptor.write(message + '\n') def ttyVerbose(self, level, message): WiredTigerTestCase.ttyVerbose(level, message) @@ -970,16 +935,6 @@ def islongtest(): def getseed(): return WiredTigerTestCase._seeds -# We have to override the ThreadsafeForwardingResult implementation of tags so it gets set immediately -# which allows us to set the pid of the process on our output stream to make debugging easier. -def immediate_tags(self, new_tags, gone_tags): - self.result.tags(new_tags, gone_tags) - -def wrap_result_for_tags(thread_safe_result, thread_number): - # We use this technique to override the method instead of extending the class as it allows for less changes. - thread_safe_result.tags = types.MethodType(immediate_tags, thread_safe_result) - return thread_safe_result - def runsuite(suite, parallel): suite_to_run = suite if parallel > 1: @@ -987,19 +942,16 @@ def runsuite(suite, parallel): if not WiredTigerTestCase._globalSetup: WiredTigerTestCase.globalSetup() WiredTigerTestCase._concurrent = True - suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel), wrap_result=wrap_result_for_tags) + suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel)) try: if WiredTigerTestCase._randomseed: WiredTigerTestCase.prout("Starting test suite with seedw={0} and seedz={1}. Rerun this test with -seed {0}.{1} to get the same randomness" .format(str(WiredTigerTestCase._seeds[0]), str(WiredTigerTestCase._seeds[1]))) - result_class = None - if WiredTigerTestCase._verbose > 1: - result_class = PidAwareTextTestResult return unittest.TextTestRunner( - verbosity=WiredTigerTestCase._verbose, resultclass=result_class).run(suite_to_run) + verbosity=WiredTigerTestCase._verbose).run(suite_to_run) except BaseException as e: # This should not happen for regular test errors, unittest should catch everything - print("[pid:{}]: ERROR: running test: {}".format(os.getpid(), e)) + print('ERROR: running test: ', e) raise e def run(name='__main__'): diff --git a/src/third_party/wiredtiger/test/test_coverage.md b/src/third_party/wiredtiger/test/test_coverage.md index a9b7c9ff9f6..b186c7ac318 100644 --- a/src/third_party/wiredtiger/test/test_coverage.md +++ b/src/third_party/wiredtiger/test/test_coverage.md @@ -15,7 +15,6 @@ |Checkpoint|History Store|[test_checkpoint03.py](../test/suite/test_checkpoint03.py) |Checkpoint|Metadata|[test_checkpoint_snapshot01.py](../test/suite/test_checkpoint_snapshot01.py) |Checkpoint|Obsolete Data|[test_checkpoint08.py](../test/suite/test_checkpoint08.py) -|Checkpoint|Recovery|[test_bug029.py](../test/suite/test_bug029.py) |Compression||[test_dictionary.py](../test/suite/test_dictionary.py) |Config Api||[test_base02.py](../test/suite/test_base02.py), [test_config02.py](../test/suite/test_config02.py) |Connection Api||[test_version.py](../test/suite/test_version.py) diff --git a/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp b/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp index a3461ccd814..59dfebc19ba 100644 --- a/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp +++ b/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp @@ -37,7 +37,7 @@ TEST_CASE("Reconciliation tracking: ovfl_discard_verbose", "[reconciliation]") SECTION("handle null page and tag") { - REQUIRE(__ut_ovfl_discard_verbose(session, nullptr, nullptr, nullptr) == EINVAL); + REQUIRE(__ut_ovfl_discard_verbose(session, nullptr, nullptr, nullptr) == 0); } } |