diff options
Diffstat (limited to 'src/third_party/wiredtiger/test/suite')
24 files changed, 2224 insertions, 59 deletions
diff --git a/src/third_party/wiredtiger/test/suite/test_backup29.py b/src/third_party/wiredtiger/test/suite/test_backup29.py new file mode 100644 index 00000000000..5cbce78e321 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup29.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, re, time +from wtscenario import make_scenarios +from wtbackup import backup_base +from wiredtiger import stat + +# test_backup29.py +# Test interaction between checkpoint and incremental backup. There was a bug in +# maintaining the incremental backup bitmaps correctly after opening an uncached dhandle. +# This test reconstructs the failure scenario and verifies correct behavior both when a +# restart and when dhandle sweep lead to opening an uncached dhandle. +class test_backup29(backup_base): + conn_config = 'file_manager=(close_handle_minimum=0,' + \ + 'close_idle_time=3,close_scan_interval=1),' + \ + 'statistics=(fast)' + create_config = 'allocation_size=512,key_format=i,value_format=S' + # Backup directory name. Uncomment if actually taking a backup. + # dir='backup.dir' + uri1 = 'test_first' + uri2 = 'test_second' + file1_uri = 'file:' + uri1 + '.wt' + file2_uri = 'file:' + uri2 + '.wt' + table1_uri = 'table:' + uri1 + table2_uri = 'table:' + uri2 + active_uri = 'table:active.wt' + + value_base = '-abcdefghijkl' + + few = 100 + nentries = 5000 + + def get_open_file_count(self): + stat_cursor = self.session.open_cursor('statistics:', None, None) + n = stat_cursor[stat.conn.file_open][2] + stat_cursor.close() + return n + + def parse_blkmods(self, uri): + meta_cursor = self.session.open_cursor('metadata:') + config = meta_cursor[uri] + meta_cursor.close() + # The search string looks like: ,blocks=feffff1f000000000000000000000000 + # Obtain just the hex string. + b = re.search(',blocks=(\w+)', config) + self.assertTrue(b is not None) + # The bitmap string after the = is in group 1. + blocks = b.group(1) + self.pr("block bitmap: " + blocks) + return blocks + + def compare_bitmap(self, orig, new): + # Compare the bitmaps from the metadata. Once a bit is set, it should never + # be cleared. But new bits could be set. So the check is only: if the original + # bitmap has a bit set then the current bitmap must be set for that bit also. + # + # First convert both bitmaps to a binary string, accounting for any possible leading + # zeroes (that would be truncated off). Then compare bit by bit. + orig_bits = bin(int('1'+orig, 16))[3:] + new_bits = bin(int('1'+new, 16))[3:] + self.pr("Original bitmap in binary: " + orig_bits) + self.pr("Reopened bitmap in binary: " + new_bits) + for o_bit, n_bit in zip(orig_bits, new_bits): + if o_bit != '0': + self.assertTrue(n_bit != '0') + + def setup_test(self): + # Create and populate the tables. + self.session.create(self.table1_uri, self.create_config) + self.session.create(self.table2_uri, self.create_config) + c1 = self.session.open_cursor(self.table1_uri) + c2 = self.session.open_cursor(self.table2_uri) + # Only add a few entries. + self.pr("Write: " + str(self.few) + " initial data items") + for i in range(1, self.few): + val = str(i) + self.value_base + c1[i] = val + c2[i] = val + self.session.checkpoint() + + # Take the initial full backup for incremental. We don't actually need to + # take the backup, we only need to open and close the backup cursor to have + # the library keep track of the bitmaps. + config = 'incremental=(enabled,granularity=4k,this_id="ID1")' + bkup_c = self.session.open_cursor('backup:', None, config) + # Uncomment these lines if actually taking the full backup is helpful for debugging. + # os.mkdir(self.dir) + # self.take_full_backup(self.dir, bkup_c) + bkup_c.close() + + # Add a lot more data to both tables to generate a filled-in block mod bitmap. + last_i = self.few + self.pr("Write: " + str(self.nentries) + " additional data items") + for i in range(self.few, self.nentries): + val = str(i) + self.value_base + c1[i] = val + c2[i] = val + c1.close() + c2.close() + self.session.checkpoint() + # Get the block mod bitmap from the file URI. + self.orig1_bitmap = self.parse_blkmods(self.file1_uri) + self.orig2_bitmap = self.parse_blkmods(self.file2_uri) + + + def incr_backup_and_validate(self): + # After reopening we want to open both tables, but only modify one of them for + # the first checkpoint. Then modify the other table, checkpoint, and then check the + # that the block mod bitmap remains correct for the other table. + c1 = self.session.open_cursor(self.table1_uri) + c2 = self.session.open_cursor(self.table2_uri) + last_i = self.nentries + + # Change the first table and checkpoint. Keep the second table clean. + self.pr("Update only table 1: " + str(last_i)) + val = str(last_i) + self.value_base + c1[last_i] = val + self.session.checkpoint() + new1_bitmap = self.parse_blkmods(self.file1_uri) + + # Now change the second table and checkpoint again. + self.pr("Update second table: " + str(last_i)) + c2[last_i] = val + self.session.checkpoint() + new2_bitmap = self.parse_blkmods(self.file2_uri) + + c1.close() + c2.close() + + self.compare_bitmap(self.orig1_bitmap, new1_bitmap) + self.compare_bitmap(self.orig2_bitmap, new2_bitmap) + + def test_backup29_reopen(self): + self.setup_test() + + self.pr("CLOSE and REOPEN conn") + self.reopen_conn() + self.pr("Reopened conn") + + self.incr_backup_and_validate() + + def test_backup29_sweep(self): + self.setup_test() + + self.pr("Waiting to sweep handles") + # Create another table and populate it, and checkpoint. + self.session.create(self.active_uri, self.create_config) + c = self.session.open_cursor(self.active_uri) + for i in range(1, self.few): + c[i] = str(i) + self.value_base + self.session.checkpoint() + + sleep = 0 + max = 20 + # The only files sweep won't close should be the metadata, the history store, the + # lock file, and our active file. + final_nfile = 4 + + # Keep updating and checkpointing this table until all other handles have been swept. + # The checkpoints have the side effect of sweeping the session cache, which will allow + # dhandles to be closed. + while sleep < max: + i = i + 1 + c[i] = str(i) + self.value_base + self.session.checkpoint() + sleep += 0.5 + time.sleep(0.5) + nfile = self.get_open_file_count() + if nfile == final_nfile: + break + c.close() + + # Make sure we swept everything before we ran out of time. + self.assertEqual(nfile, final_nfile) + self.pr("Sweep done") + + self.incr_backup_and_validate() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug029.py b/src/third_party/wiredtiger/test/suite/test_bug029.py new file mode 100644 index 00000000000..b2d9e1c6ac4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_bug029.py @@ -0,0 +1,112 @@ + +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# checkpoint:recovery +# [END_TAGS] + +import wttest +import os, shutil + +# test_bug029.py +# +# Test that WT correctly propogates the most recent checkpoint time +# across restarts. We validate this by reproducing the original bug +# from WT-9457: frequent checkpoints pushed the checkpoint clock time +# into the future such that immediately after a restart a backup could +# see its checkpoint deleted out from under it. The result was fatal +# read errors when restoring the backup. + +class test_bug029(wttest.WiredTigerTestCase): + conn_config = ("cache_size=50MB") + uri = "table:test_bug029" + bigvalue = "WiredTiger" * 100 + backup_dir = "backup_dir" + + def add_data(self, uri, start, count): + cursor = self.session.open_cursor(uri, None) + for i in range(start, start + count): + cursor[i] = self.bigvalue + cursor.close() + + def test_bug029(self): + # Create and populate table + self.session.create(self.uri, "key_format=i,value_format=S") + self.add_data(self.uri, 0, 2000) + self.session.checkpoint() + + # Force the checkpoint time forward with a lot of quick checkpoints. + for i in range(100): + self.session.checkpoint("force=1") + + # Add more data and checkpoint again. This creates a bunch of pages + # in the final checkpoint that can be deleted and reused if we hit + # the bug. + self.add_data(self.uri, 2000, 2000) + self.session.checkpoint() + + # Shutdown and reopen. + self.reopen_conn() + + self.add_data(self.uri, 0, 100) + + # Open a backup cursor and force a few checkpoints. This will allow + # WT to delete older checkpoints, but as long as the backup cursor + # is open, it shouldn't delete the backup checkpoint---unless we hit + # the bug. + backup_cursor = self.session.open_cursor('backup:') + + for i in range(10): + self.session.checkpoint("force=1") + + # Write and checkpoint a bunch of data. If we erroneously deleted our + # backup checkpoint this should overwrite some of that checkpoint's + # blocks. + self.add_data(self.uri, 1000, 2000) + self.session.checkpoint() + + # Now do the backup. + os.mkdir(self.backup_dir) + while True: + ret = backup_cursor.next() + if ret != 0: + break + shutil.copy(backup_cursor.get_key(), self.backup_dir) + backup_cursor.close() + + # Open the backup and read data. If the backup snapshot was corrupted + # we will panic and die here. + backup_conn = self.wiredtiger_open(self.backup_dir, self.conn_config) + session = backup_conn.open_session() + cur1 = session.open_cursor(self.uri) + for i in range(0, 4000, 10): + self.assertEqual(cur1[i], self.bigvalue) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug030.py b/src/third_party/wiredtiger/test/suite/test_bug030.py new file mode 100644 index 00000000000..3969d71de9d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_bug030.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +import wttest +from helper import simulate_crash_restart +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_bug030.py +# This tests for the scenario in WT-10522 where we could return early when +# appending a key's original value to its update list due to checking some +# flags that must be ignored when looking at an aborted tombstone. +class test_bug_030(wttest.WiredTigerTestCase): + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + scenarios = make_scenarios(format_values) + + def conn_config(self): + config = 'debug_mode=(update_restore_evict=true)' + return config + + def test_bug030(self): + nrows = 10 + uri = "table:test_bug030" + + if self.value_format == '8t': + valuea = 97 + valueb = 98 + else: + valuea = "abcdef" * 3 + valueb = "ghijkl" * 3 + + self.session.create(uri, 'key_format={},value_format={}'.format( + self.key_format, self.value_format)) + + # Stable insert + self.session.begin_transaction() + cursor = self.session.open_cursor(uri) + for i in range(1, nrows + 1): + cursor[i] = valuea + cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + + self.conn.set_timestamp('oldest_timestamp={},stable_timestamp={}'.format( + self.timestamp_str(10), self.timestamp_str(20))) + + # Unstable delete + self.session.begin_transaction() + cursor = self.session.open_cursor(uri) + for i in range(1, nrows + 1): + cursor.set_key(i) + cursor.remove() + cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) + + # Evict everything + self.session.begin_transaction() + evict_cursor = self.session.open_cursor(uri, None, 'debug=(release_evict)') + for i in range(1, nrows + 1): + evict_cursor.set_key(i) + evict_cursor.search() + evict_cursor.reset() + evict_cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) + + # Unstable, uncommitted update + self.session.begin_transaction() + cursor = self.session.open_cursor(uri) + for i in range(1, nrows + 1): + cursor[i] = valueb + cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) + + self.session.checkpoint() + self.conn.rollback_to_stable() + + # Another delete, committed this time + self.session.begin_transaction() + cursor = self.session.open_cursor(uri) + for i in range(1, nrows + 1): + cursor.set_key(i) + cursor.remove() + cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(60)) + + # Finally, evict everything. At this point, our key(s) have an + # update chain with a single entry that's both aborted and restored + # from the data store, that we attempt to reconcile. + self.session.begin_transaction() + evict_cursor = self.session.open_cursor(uri, None, 'debug=(release_evict)') + for i in range(1, nrows + 1): + evict_cursor.set_key(i) + evict_cursor.search() + evict_cursor.reset() + evict_cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(70)) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py index e9411acfe0d..5f7d7f9a38c 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py @@ -117,7 +117,7 @@ class test_checkpoint04(wttest.WiredTigerTestCase): time_total = self.get_stat(stat.conn.txn_checkpoint_time_total) self.pr('txn_checkpoint_time_total ' + str(time_total)) - self.assertEqual(num_ckpt, 2 * multiplier) + self.assertEqual(num_ckpt, 2) self.assertEqual(running, 0) self.assertEqual(prep_running, 0) # Assert if this loop continues for more than 100 iterations. @@ -129,6 +129,9 @@ class test_checkpoint04(wttest.WiredTigerTestCase): break else: multiplier += 1 + # Reopen the connection to reset statistics. + # We don't want stats from earlier runs to interfere with later runs. + self.reopen_conn() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py index 89a7faddfc0..7ee28a240cc 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py @@ -161,8 +161,15 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - # Sleep for sometime so that checkpoint starts before committing last transaction. - time.sleep(2) + + # Wait for checkpoint to start and acquire its snapshot before committing. + ckpt_snapshot = 0 + while not ckpt_snapshot: + time.sleep(1) + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] + stat_cursor.close() + session1.commit_transaction() finally: @@ -215,8 +222,15 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - # Sleep for sometime so that checkpoint starts before committing last transaction. - time.sleep(2) + + # Wait for checkpoint to start and acquire its snapshot before committing. + ckpt_snapshot = 0 + while not ckpt_snapshot: + time.sleep(1) + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] + stat_cursor.close() + session1.commit_transaction() finally: @@ -272,8 +286,15 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - # Sleep for sometime so that checkpoint starts before committing last transaction. - time.sleep(2) + + # Wait for checkpoint to start and acquire its snapshot before committing. + ckpt_snapshot = 0 + while not ckpt_snapshot: + time.sleep(1) + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] + stat_cursor.close() + session2.commit_transaction() finally: diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py index e8ff0067f61..583808915b6 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot05.py @@ -147,8 +147,15 @@ class test_checkpoint_snapshot05(wttest.WiredTigerTestCase): ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() - # Sleep for sometime so that checkpoint starts before committing last transaction. - time.sleep(2) + + # Wait for checkpoint to start and acquire its snapshot before committing. + ckpt_snapshot = 0 + while not ckpt_snapshot: + time.sleep(1) + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_snapshot = stat_cursor[stat.conn.txn_checkpoint_snapshot_acquired][2] + stat_cursor.close() + session1.commit_transaction() self.evict(self.uri, ds, self.nrows) finally: diff --git a/src/third_party/wiredtiger/test/suite/test_hs11.py b/src/third_party/wiredtiger/test/suite/test_hs11.py index 647a7bfece5..417d37cca29 100755 --- a/src/third_party/wiredtiger/test/suite/test_hs11.py +++ b/src/third_party/wiredtiger/test/suite/test_hs11.py @@ -44,8 +44,20 @@ class test_hs11(wttest.WiredTigerTestCase): ('deletion', dict(update_type='deletion')), ('update', dict(update_type='update')) ] - scenarios = make_scenarios(format_values, update_type_values) - nrows = 10000 + long_running_txn_values = [ + ('long-running', dict(long_run_txn=True)), + ('no-long-running', dict(long_run_txn=False)) + ] + last_update_type_values = [ + ('modify', dict(modify=True)), + ('no-modify', dict(modify=False)) + ] + nrows = [ + ('small-nrows', dict(nrows=100)), + ('large-nrows', dict(nrows=10000)) + ] + scenarios = make_scenarios(format_values, update_type_values,long_running_txn_values, last_update_type_values, nrows) + timestamps = 5 def create_key(self, i): if self.key_format == 'S': @@ -58,6 +70,18 @@ class test_hs11(wttest.WiredTigerTestCase): stat_cursor.close() return val + def evict_cursor(self, uri, nrows): + s = self.conn.open_session() + s.begin_transaction() + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") + for i in range(1, nrows + 1): + evict_cursor.set_key(self.create_key(i)) + evict_cursor.search() + evict_cursor.reset() + s.rollback_transaction() + evict_cursor.close() + def test_non_ts_updates_clears_hs(self): uri = 'table:test_hs11' create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) @@ -69,13 +93,14 @@ class test_hs11(wttest.WiredTigerTestCase): else: value1 = 'a' * 500 value2 = 'b' * 500 + mod_value = 'm' + 'a' * 499 # FIXME-WT-9063 revisit the use of self.retry() throughout this file. # Apply a series of updates from timestamps 1-4. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for ts in range(1, 5): + for ts in range(1, self.timestamps): for i in range(1, self.nrows): for retry in self.retry(): with retry.transaction(commit_timestamp = ts): @@ -83,8 +108,24 @@ class test_hs11(wttest.WiredTigerTestCase): # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() + self.evict_cursor(uri, self.nrows) + + # Apply a modify update at timestamp 5. + if self.modify and self.value_format != '8t': + for i in range(1, self.nrows): + for retry in self.retry(): + with retry.transaction(commit_timestamp = 5): + cursor.set_key(self.create_key(i)) + cursor.modify([wiredtiger.Modify("m", 0, 1)]) + self.timestamps += 1 + + # Start a long running transaction at timestamp 5. + if self.long_run_txn: + session2 = self.conn.open_session() + session2.begin_transaction('read_timestamp=5') - # Apply an update without timestamp. + # Apply an update without timestamp. If we have a long running transaction this update + # should not be globally visible until that transaction has ended. for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': @@ -95,6 +136,11 @@ class test_hs11(wttest.WiredTigerTestCase): # Reconcile and remove the obsolete entries. self.session.checkpoint() + if self.long_run_txn: + session2.rollback_transaction() + + # At this point any updates with no timestamp should be globally visible. + self.evict_cursor(uri, self.nrows) # Now apply an update at timestamp 10. for i in range(1, self.nrows): @@ -102,8 +148,10 @@ class test_hs11(wttest.WiredTigerTestCase): with retry.transaction(commit_timestamp = 10): cursor[self.create_key(i)] = value2 + self.session.checkpoint() + # Ensure that we blew away history store content. - for ts in range(1, 5): + for ts in range(1, self.timestamps): for retry in self.retry(): with retry.transaction(read_timestamp = ts, rollback = True): for i in range(1, self.nrows): @@ -118,7 +166,10 @@ class test_hs11(wttest.WiredTigerTestCase): else: self.assertEqual(cursor[self.create_key(i)], value2) else: - self.assertEqual(cursor[self.create_key(i)], value1) + if ts == 5 and self.modify and self.value_format != '8t': + self.assertEqual(cursor[self.create_key(i)], mod_value) + else: + self.assertEqual(cursor[self.create_key(i)], value1) if self.update_type == 'deletion': hs_truncate = self.get_stat(stat.conn.cache_hs_key_truncate_onpage_removal) @@ -135,11 +186,12 @@ class test_hs11(wttest.WiredTigerTestCase): else: value1 = 'a' * 500 value2 = 'b' * 500 + mod_value = 'm' + 'a' * 499 # Apply a series of updates from timestamps 1-4. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for ts in range(1, 5): + for ts in range(1, self.timestamps): for i in range(1, self.nrows): for retry in self.retry(): with retry.transaction(commit_timestamp = ts): @@ -148,6 +200,15 @@ class test_hs11(wttest.WiredTigerTestCase): # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() + # Apply a modify update at timestamp 5. + if self.modify and self.value_format != '8t': + for i in range(1, self.nrows): + for retry in self.retry(): + with retry.transaction(commit_timestamp = 5): + cursor.set_key(self.create_key(i)) + cursor.modify([wiredtiger.Modify("m", 0, 1)]) + self.timestamps += 1 + # Remove the key with timestamp 10. for i in range(1, self.nrows): if i % 2 == 0: @@ -178,7 +239,10 @@ class test_hs11(wttest.WiredTigerTestCase): else: self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: - self.assertEqual(cursor[self.create_key(i)], value1) + if self.modify and self.value_format != '8t': + self.assertEqual(cursor[self.create_key(i)], mod_value) + else: + self.assertEqual(cursor[self.create_key(i)], value1) hs_truncate = self.get_stat(stat.conn.cache_hs_key_truncate_onpage_removal) self.assertEqual(hs_truncate, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_hs20.py b/src/third_party/wiredtiger/test/suite/test_hs20.py index 071d33de9bd..e2b9d878517 100755 --- a/src/third_party/wiredtiger/test/suite/test_hs20.py +++ b/src/third_party/wiredtiger/test/suite/test_hs20.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest +import wiredtiger, wttest, sys from wtscenario import make_scenarios # test_hs20.py @@ -103,3 +103,8 @@ class test_hs20(wttest.WiredTigerTestCase): for retry in self.retry(): with retry.transaction(read_timestamp = 3, rollback = True): self.assertEqual(cursor[self.make_key(i)], value1 + "B") + + if (sys.platform.startswith('darwin')): + # Ignore the eviction generation drain warning as it is possible for eviction to take + # longer to evict pages due to overflow items on the page. + self.ignoreStdoutPatternIfExists('Eviction took more than 1 minute') diff --git a/src/third_party/wiredtiger/test/suite/test_hs32.py b/src/third_party/wiredtiger/test/suite/test_hs32.py new file mode 100755 index 00000000000..b90bbaa15db --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs32.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios +from wiredtiger import stat + +# test_hs32.py +# Ensure that updates without timestamps clear the history store records. +class test_hs32(wttest.WiredTigerTestCase): + conn_config = 'cache_size=500MB,statistics=(all)' + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('integer-row', dict(key_format='i', value_format='S')), + ('string-row', dict(key_format='S', value_format='S')), + ] + update_type_values = [ + ('deletion', dict(update_type='deletion')), + ('update', dict(update_type='update')) + ] + long_running_txn_values = [ + ('no-long-run-txn', dict(long_run_txn=False)), + ('long-run-txn', dict(long_run_txn=True)) + ] + scenarios = make_scenarios(format_values, update_type_values,long_running_txn_values) + nrows = 100 + + def create_key(self, i): + if self.key_format == 'S': + return str(i) + return i + + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + + def evict_cursor(self, uri, nrows): + s = self.conn.open_session() + s.begin_transaction() + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") + for i in range(1, nrows + 1): + evict_cursor.set_key(self.create_key(i)) + evict_cursor.search() + evict_cursor.reset() + s.rollback_transaction() + evict_cursor.close() + + def test_non_ts_updates_tombstone_clears_hs(self): + uri = 'table:test_hs32' + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, create_params) + + if self.value_format == '8t': + value1 = 97 + value2 = 98 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + + # Apply a series of updates from timestamps 1-4. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + cursor = self.session.open_cursor(uri) + for ts in range(1, 5): + for i in range(1, self.nrows): + for retry in self.retry(): + with retry.transaction(commit_timestamp = ts): + cursor[self.create_key(i)] = value1 + + # Reconcile and flush versions 1-3 to the history store. + self.session.checkpoint() + self.evict_cursor(uri, self.nrows) + + if self.long_run_txn: + # Apply a another update at timestamp 5. + for i in range(1, self.nrows): + for retry in self.retry(): + with retry.transaction(commit_timestamp = 5): + cursor[self.create_key(i)] = value1 + + # Start a long running transaction to make tombstone not globally visible. + session2 = self.conn.open_session() + session2.begin_transaction('read_timestamp=5') + + # Apply an update/delete without timestamp. + for i in range(1, self.nrows): + self.session.begin_transaction() + if i % 2 == 0: + if self.update_type == 'deletion': + cursor.set_key(self.create_key(i)) + cursor.remove() + else: + cursor[self.create_key(i)] = value2 + self.session.commit_transaction() + + if self.long_run_txn: + # Reconcile and remove the obsolete entries. + self.session.checkpoint() + self.evict_cursor(uri, self.nrows) + + # Rollback the long running transaction. + session2.rollback_transaction() + + # Now apply an update at timestamp 10. + for i in range(1, self.nrows): + for retry in self.retry(): + with retry.transaction(commit_timestamp = 10): + cursor[self.create_key(i)] = value2 + + self.session.checkpoint() + + # Ensure that we blew away history store content. + for ts in range(1, 5): + for retry in self.retry(): + with retry.transaction(read_timestamp = ts, rollback = True): + for i in range(1, self.nrows): + if i % 2 == 0: + if self.update_type == 'deletion': + cursor.set_key(self.create_key(i)) + if self.value_format == '8t': + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEqual(cursor[self.create_key(i)], value2) + else: + self.assertEqual(cursor[self.create_key(i)], value1) + + if self.update_type == 'deletion': + cache_hs_key_truncate = self.get_stat(stat.conn.cache_hs_key_truncate) + self.assertGreater(cache_hs_key_truncate, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare20.py b/src/third_party/wiredtiger/test/suite/test_prepare20.py index 0b541d40730..6f219ea2c83 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare20.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare20.py @@ -27,14 +27,14 @@ # OTHER DEALINGS IN THE SOFTWARE. -# test_prepare19.py +# test_prepare20.py # Check that we can use an application-level log to replay unstable transactions. -import wiredtiger, wttest +import wttest from wtscenario import make_scenarios from helper import simulate_crash_restart -class test_prepare19(wttest.WiredTigerTestCase): +class test_prepare20(wttest.WiredTigerTestCase): # Do write the logs immediately, but don't waste time fsyncing them. conn_config = 'log=(enabled),transaction_sync=(enabled=true,method=none)' @@ -131,7 +131,7 @@ class test_prepare19(wttest.WiredTigerTestCase): # First pass: find prepared txns. Ignore (thus abort) any that didn't even prepare. txns = {} self.lcursor.reset() - for lsn, op, k, oldv, newv in self.lcursor: + for _, op, k, oldv, newv in self.lcursor: if op == self.BEGIN: # "key" is the txnid txns[k] = False @@ -143,7 +143,7 @@ class test_prepare19(wttest.WiredTigerTestCase): writing = False committime = None durabletime = None - for lsn, op, k, oldv, newv in self.lcursor: + for _, op, k, oldv, newv in self.lcursor: if op == self.BEGIN: # "key" is the txnid if txns[k]: @@ -198,9 +198,9 @@ class test_prepare19(wttest.WiredTigerTestCase): # Now the test. - def test_prepare19(self): - data_uri = 'file:prepare19data' - log_uri = 'file:prepare19log' + def test_prepare20(self): + data_uri = 'file:prepare20data' + log_uri = 'file:prepare20log' # Create one table for data and another to be an application-level log. # The log's format is application-lsn -> operation, key, oldvalue, newvalue diff --git a/src/third_party/wiredtiger/test/suite/test_prepare21.py b/src/third_party/wiredtiger/test/suite/test_prepare21.py new file mode 100644 index 00000000000..ec61092d04c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare21.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import threading, time +from helper import simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios +from wtthread import checkpoint_thread + +# test_prepare21.py +# Test prepare rollback doesn't crash because of triggering out of order fix. +class test_prepare21(test_rollback_to_stable_base): + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + scenarios = make_scenarios(format_values) + + def conn_config(self): + config = 'cache_size=10MB,statistics=(all),timing_stress_for_test=[history_store_checkpoint_delay]' + return config + + def evict_cursor(self, uri, nrows): + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + self.session.begin_transaction("ignore_prepare=true") + for i in range (1, nrows + 1): + evict_cursor.set_key(i) + evict_cursor.search() + evict_cursor.reset() + evict_cursor.close() + self.session.rollback_transaction() + + def test_prepare_rollback(self): + nrows = 10 + + # Create a table. + uri = "table:prepare21" + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format) + ds.populate() + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + self.large_updates(uri, value_a, ds, nrows, False, 20) + self.large_updates(uri, value_b, ds, nrows, False, 30) + self.large_removes(uri, ds, nrows, False, 40) + + prepare_session = self.conn.open_session() + prepare_session.begin_transaction() + cursor = prepare_session.open_cursor(uri) + for i in range (1, nrows + 1): + cursor[i] = value_c + cursor.close() + prepare_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) + + # Verify data is visible and correct. + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + + self.evict_cursor(uri, nrows) + + # Pin stable to timestamp 40. + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) + + # Rollback the prepared update + prepare_session.rollback_transaction() + self.large_updates(uri, value_d, ds, nrows, False, 60) + + done = threading.Event() + ckpt = checkpoint_thread(self.conn, done) + try: + ckpt.start() + + # Wait for checkpoint to start before committing last transaction. + ckpt_started = 0 + while not ckpt_started: + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] + stat_cursor.close() + + self.evict_cursor(uri, nrows) + finally: + done.set() + ckpt.join() + + # Verify data is visible and correct. + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + self.check(value_d, uri, nrows, None, 60) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare22.py b/src/third_party/wiredtiger/test/suite/test_prepare22.py new file mode 100644 index 00000000000..f35f4a50052 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare22.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_prepare22.py +# Test prepare with rollback to stable without failed eviction. +class test_prepare22(wttest.WiredTigerTestCase): + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + delete = [ + ('delete', dict(delete=True)), + ('non-delete', dict(delete=False)), + ] + + scenarios = make_scenarios(format_values, delete) + + def test_prepare22(self): + uri = "table:test_prepare22" + self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "a" + value_b = "b" + value_c = "c" + + # Pin oldest timestamp to 1 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + + # Do the first update + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + + # Do the second update + self.session.begin_transaction() + cursor[1] = value_b + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) + + if self.delete: + self.session.begin_transaction() + cursor.set_key(1) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) + + # Do a prepared update + self.session.begin_transaction() + cursor[1] = value_c + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) + + # Evict the page + session2 = self.conn.open_session() + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(20)) + self.assertEquals(evict_cursor[1], value_b) + evict_cursor.reset() + evict_cursor.close() + session2.rollback_transaction() + + # Ensure the history store is checkpointed + session2.checkpoint() + + # Rollback the prepared transaction + self.session.rollback_transaction() + + # Set stable timestamp to 30 + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) + + # Call rollback to stable + self.conn.rollback_to_stable() + + # Evict the page again + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + session2.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + self.assertEquals(evict_cursor[1], value_b) + evict_cursor.reset() + evict_cursor.close() + session2.rollback_transaction() + + # Verify we can still read back value a + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(10)) + self.assertEquals(cursor[1], value_a) + self.session.rollback_transaction() + + # Verify we can still read back value b + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + self.assertEquals(cursor[1], value_b) + self.session.rollback_transaction() + + # Verify we can still read back the deletion + if self.delete: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) + if self.value_format == '8t': + self.assertEquals(cursor[1], 0) + else: + cursor.set_key(1) + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare23.py b/src/third_party/wiredtiger/test/suite/test_prepare23.py new file mode 100644 index 00000000000..f7496a5d228 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare23.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_prepare23.py +# Test prepare rollback with rollback to stable and failed eviction. +class test_prepare23(wttest.WiredTigerTestCase): + conn_config = 'timing_stress_for_test=[failpoint_eviction_fail_after_reconciliation]' + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + delete = [ + ('delete', dict(delete=True)), + ('non-delete', dict(delete=False)), + ] + + scenarios = make_scenarios(format_values, delete) + + def test_prepare23(self): + uri = "table:test_prepare23" + self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "a" + value_b = "b" + value_c = "c" + + # Pin oldest timestamp to 1 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + ts = 0 + for i in range (1, 1001): + # Do the first update + self.session.begin_transaction() + cursor[i] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 10)) + + # Do the second update + self.session.begin_transaction() + cursor[i] = value_b + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 20)) + + if self.delete: + self.session.begin_transaction() + cursor.set_key(i) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 30)) + + # Do a prepared update + self.session.begin_transaction() + cursor[i] = value_c + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 40)) + cursor.reset() + + # Evict the page + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 20)) + self.assertEquals(evict_cursor[i], value_b) + evict_cursor.reset() + session2.rollback_transaction() + + # Rollback the prepared transaction + self.session.rollback_transaction() + + # Set stable timestamp to 30 * i + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(ts + 30)) + + # Call rollback to stable + self.conn.rollback_to_stable() + + # Verify we can still read back value a + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(cursor[i], value_a) + self.session.rollback_transaction() + + # Verify we can still read back value b + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 20)) + self.assertEquals(cursor[i], value_b) + self.session.rollback_transaction() + + # Verify we can still read back the deletion + if self.delete: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 30)) + if self.value_format == '8t': + self.assertEquals(cursor[i], 0) + else: + cursor.set_key(i) + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + + ts += 40 + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare24.py b/src/third_party/wiredtiger/test/suite/test_prepare24.py new file mode 100644 index 00000000000..af367af87e5 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare24.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_prepare24.py +# Test prepare commit after eviction failure. +class test_prepare24(wttest.WiredTigerTestCase): + conn_config = 'timing_stress_for_test=[failpoint_eviction_fail_after_reconciliation]' + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + delete = [ + ('delete', dict(delete=True)), + ('non-delete', dict(delete=False)), + ] + + scenarios = make_scenarios(format_values, delete) + + def test_prepare24(self): + uri = "table:test_prepare24" + self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + else: + value_a = "a" + value_b = "b" + + # Pin oldest timestamp to 1 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + ts = 0 + for i in range (1, 1001): + # Insert a value + self.session.begin_transaction() + cursor[i] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 10)) + + if self.delete: + self.session.begin_transaction() + cursor.set_key(i) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 20)) + + # Do a prepared update + self.session.begin_transaction() + cursor[i] = value_b + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 30)) + cursor.reset() + + # Evict the page + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(evict_cursor[i], value_a) + evict_cursor.reset() + session2.rollback_transaction() + + # Commit the prepared transaction + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(ts + 30)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(ts + 40)) + self.session.commit_transaction() + + # Evict the page again + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(evict_cursor[i], value_a) + evict_cursor.reset() + session2.rollback_transaction() + + # Verify we can still read back value a + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(cursor[i], value_a) + self.session.rollback_transaction() + + # Verify we can still read back the deletion + if self.delete: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 20)) + if self.value_format == '8t': + self.assertEquals(cursor[i], 0) + else: + cursor.set_key(i) + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + + # Verify we can still read back the prepared update + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 30)) + self.assertEquals(cursor[i], value_b) + self.session.rollback_transaction() + + ts += 40 + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare25.py b/src/third_party/wiredtiger/test/suite/test_prepare25.py new file mode 100644 index 00000000000..e512c3c7fef --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare25.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_prepare25.py +# Test prepare rollback and then prepare commit with failed eviction. +class test_prepare25(wttest.WiredTigerTestCase): + conn_config = 'timing_stress_for_test=[failpoint_eviction_fail_after_reconciliation]' + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + delete = [ + ('delete', dict(delete=True)), + ('non-delete', dict(delete=False)), + ] + + scenarios = make_scenarios(format_values, delete) + + def test_prepare25(self): + uri = "table:test_prepare25" + self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "a" + value_b = "b" + value_c = "c" + + # Pin oldest timestamp to 1 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + ts = 0 + for i in range (1, 1001): + # Insert an update + self.session.begin_transaction() + cursor[i] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 10)) + + if self.delete: + self.session.begin_transaction() + cursor.set_key(i) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 20)) + + # Do a prepared update + self.session.begin_transaction() + cursor[i] = value_b + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 30)) + cursor.reset() + + # Evict the page + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(evict_cursor[i], value_a) + evict_cursor.reset() + session2.rollback_transaction() + + # Rollback the prepared transaction + self.session.rollback_transaction() + + # Do another prepared update + self.session.begin_transaction() + cursor[i] = value_c + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 40)) + + # Commit the prepared update + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(ts + 40)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(ts + 50)) + self.session.commit_transaction() + cursor.reset() + + # Evict the page again + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(evict_cursor[i], value_a) + evict_cursor.reset() + session2.rollback_transaction() + + # Verify we can still read back value a + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(cursor[i], value_a) + self.session.rollback_transaction() + + # Verify we can still read back the deletion + if self.delete: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 20)) + if self.value_format == '8t': + self.assertEquals(cursor[i], 0) + else: + cursor.set_key(i) + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + + # Verify we can still read back value c + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 40)) + self.assertEquals(cursor[i], value_c) + self.session.rollback_transaction() + + ts += 50 + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare26.py b/src/third_party/wiredtiger/test/suite/test_prepare26.py new file mode 100644 index 00000000000..52e08cedff6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare26.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_prepare26.py +# Test prepare rollback and then delete the key. +class test_prepare26(wttest.WiredTigerTestCase): + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + scenarios = make_scenarios(format_values) + + def test_prepare26(self): + uri = "table:test_prepare26" + self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "a" + value_b = "b" + value_c = "c" + + # Pin oldest timestamp to 1 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + + # Insert a value + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + + # Do a prepared update + self.session.begin_transaction() + cursor[1] = value_c + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) + + # Evict the page + session2 = self.conn.open_session() + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(10)) + self.assertEquals(evict_cursor[1], value_a) + evict_cursor.reset() + evict_cursor.close() + session2.rollback_transaction() + + # Rollback the prepared transaction + self.session.rollback_transaction() + + # Delete the key + self.session.begin_transaction() + cursor.set_key(1) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) + + # Set oldest timestamp to 30 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30)) + + # Evict the page again + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + session2.begin_transaction() + evict_cursor.set_key(1) + if self.value_format == '8t': + self.assertEquals(evict_cursor[1], 0) + else: + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), wiredtiger.WT_NOTFOUND) + evict_cursor.reset() + evict_cursor.close() + session2.rollback_transaction() + + # Do another update + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = value_b + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) + + # Do another update + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = value_c + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) + + # Evict the page again + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + session2.begin_transaction('read_timestamp=' + self.timestamp_str(50)) + self.assertEquals(evict_cursor[1], value_c) + evict_cursor.reset() + evict_cursor.close() + session2.rollback_transaction() + + # Verify we read nothing at the oldest + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) + if self.value_format == '8t': + self.assertEquals(cursor[1], 0) + else: + cursor.set_key(1) + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable39.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable39.py new file mode 100644 index 00000000000..90bf63935da --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable39.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +import threading, time +from helper import simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios +from wtthread import checkpoint_thread + +# test_rollback_to_stable39.py +# Test to delay checkpoint and perform eviction in parallel to ensure eviction moves the content from data store to history store +# and then checkpoint history store to see the same content in data store and history store. Later use the checkpoint to restore +# the database which will trigger eviction to insert the same record from data store to history store. +class test_rollback_to_stable39(test_rollback_to_stable_base): + restart_config = False + + format_values = [ + ('column', dict(key_format='r', value_format='S', prepare_extraconfig='')), + ('column_fix', dict(key_format='r', value_format='8t', + prepare_extraconfig=',allocation_size=512,leaf_page_max=512')), + ('row_integer', dict(key_format='i', value_format='S', prepare_extraconfig='')), + ] + + prepare_values = [ + ('no_prepare', dict(prepare=False)), + ('prepare', dict(prepare=True)) + ] + + scenarios = make_scenarios(format_values, prepare_values) + + def conn_config(self): + config = 'cache_size=25MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + if self.restart_config: + config += ',timing_stress_for_test=[checkpoint_slow]' + else: + config += ',timing_stress_for_test=[history_store_checkpoint_delay]' + return config + + def test_rollback_to_stable(self): + nrows = 1000 + + # Create a table. + uri = "table:rollback_to_stable39" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format=self.value_format) + ds.populate() + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + # Perform several updates. + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + # Verify data is visible and correct. + self.check(value_a, uri, nrows, None, 21 if self.prepare else 20) + + self.large_removes(uri, ds, nrows, self.prepare, 30) + # Verify no data is visible. + self.check(value_a, uri, 0, nrows, 31 if self.prepare else 30) + + # Pin stable to timestamp 40 if prepare otherwise 30. + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40 if self.prepare else 30)) + + # Create a checkpoint thread + done = threading.Event() + ckpt = checkpoint_thread(self.conn, done) + try: + ckpt.start() + + # Wait for checkpoint to start before committing. + ckpt_started = 0 + while not ckpt_started: + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] + stat_cursor.close() + time.sleep(1) + + # Perform several updates in parallel with checkpoint. + # Rollbacks may occur when checkpoint is running, so retry as needed. + self.retry_rollback('update ds, e', None, + lambda: self.large_updates(uri, value_b, ds, nrows, self.prepare, 50)) + self.evict_cursor(uri, nrows, value_b) + finally: + done.set() + ckpt.join() + + # Simulate a crash by copying to a new directory(RESTART). + self.restart_config = True + simulate_crash_restart(self, ".", "RESTART") + + # Check that the correct data is seen at and after the stable timestamp. + self.check(value_a, uri, nrows, None, 21 if self.prepare else 20) + self.check(value_a, uri, 0, nrows, 40) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] + keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] + keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + stat_cursor.close() + + self.assertEqual(keys_removed, 0) + self.assertEqual(keys_restored, 0) + self.assertEqual(upd_aborted, 0) + self.assertEqual(hs_removed, 0) + self.assertEqual(hs_sweep, 0) + + # Perform several updates. + self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) + + # Verify data is visible and correct. + self.check(value_c, uri, nrows, None, 61 if self.prepare else 60) + + # Create a checkpoint thread + done = threading.Event() + ckpt = checkpoint_thread(self.conn, done) + try: + ckpt.start() + + # Wait for checkpoint to start before committing. + ckpt_started = 0 + while not ckpt_started: + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] + stat_cursor.close() + time.sleep(1) + + self.evict_cursor(uri, nrows, value_c) + finally: + done.set() + ckpt.join() + + +if __name__ == '__main__': + wttest.run() + diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable40.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable40.py new file mode 100755 index 00000000000..07df57c2966 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable40.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, shutil, time +from helper import copy_wiredtiger_home, simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_rollback_to_stable40.py +# Test the rollback to stable operation performs as expected following a server crash +# and recovery. Verify that the on-disk value is replaced by the correct value from +# the history store. +class test_rollback_to_stable40(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + + def conn_config(self): + config = 'cache_size=1MB,statistics=(all),log=(enabled=true)' + return config + + def test_rollback_to_stable(self): + nrows = 3 + + # Create a table without logging. + uri = "table:rollback_to_stable40" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + + # Insert 3 keys with same updates. + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = value_a + cursor[2] = value_a + cursor[3] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) + + # Update the first and last key with another value with a large timestamp. + self.session.begin_transaction() + cursor[1] = value_d + cursor[3] = value_d + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1000)) + + # Update the middle key with lot of updates to generate more history. + for i in range(21, 499): + self.session.begin_transaction() + cursor[2] = value_b + str(i) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(i)) + + # With this checkpoint, all the updates in the history store are persisted to disk. + self.session.checkpoint() + + self.session.begin_transaction() + cursor[2] = value_c + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(500)) + + # Pin oldest and stable to timestamp 500. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(500) + + ',stable_timestamp=' + self.timestamp_str(500)) + + # Evict the globally visible update to write to the disk, this will reset the time window. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + self.session.begin_transaction("ignore_prepare=true") + evict_cursor.set_key(2) + self.assertEqual(evict_cursor[2], value_c) + evict_cursor.reset() + evict_cursor.close() + self.session.rollback_transaction() + + self.session.begin_transaction() + cursor[2] = value_d + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(501)) + + # 1. This checkpoint will move the globally visible update to the first of the key range. + # 2. The existing updates in the history store are having with a larger timestamp are + # obsolete, so they are not explicitly removed. + # 3. Any of the history store updates that are already evicted will not rewrite by the + # checkpoint. + self.session.checkpoint() + + # Verify data is visible and correct. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1000)) + for i in range (1, nrows + 1): + cursor.set_key(ds.key(i)) + self.assertEqual(cursor.search(), 0) + self.assertEquals(cursor.get_value(), value_d) + self.session.rollback_transaction() + cursor.close() + + # Simulate a server crash and restart. + simulate_crash_restart(self, ".", "RESTART") + + # Verify data is visible and correct. + cursor = self.session.open_cursor(uri) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1000)) + for i in range (1, nrows + 1): + cursor.set_key(ds.key(i)) + self.assertEqual(cursor.search(), 0) + if i % 2 == 0: + self.assertEquals(cursor.get_value(), value_c) + else: + self.assertEquals(cursor.get_value(), value_a) + self.session.rollback_transaction() + + stat_cursor = self.session.open_cursor('statistics:', None, None) + calls = stat_cursor[stat.conn.txn_rts][2] + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] + keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] + pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + stat_cursor.close() + + self.assertEqual(calls, 0) + self.assertEqual(keys_removed, 0) + self.assertEqual(keys_restored, 0) + self.assertGreaterEqual(upd_aborted, 0) + self.assertGreater(pages_visited, 0) + self.assertGreaterEqual(hs_removed, 3) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_stat08.py b/src/third_party/wiredtiger/test/suite/test_stat08.py index 7e91932d1d7..0b833c1934c 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat08.py +++ b/src/third_party/wiredtiger/test/suite/test_stat08.py @@ -33,14 +33,29 @@ import wiredtiger, wttest # Session statistics for bytes read into the cache. class test_stat08(wttest.WiredTigerTestCase): - nentries = 350000 - conn_config = 'cache_size=10MB,statistics=(all)' - entry_value = "abcde" * 40 + nentries = 100000 + # Leave the cache size on the default setting to avoid filling up the cache + # too much and triggering unnecessary rollbacks. But make the value fairly + # large to make obvious change to the statistics. + conn_config = 'statistics=(all)' + entry_value = "abcde" * 400 BYTES_READ = wiredtiger.stat.session.bytes_read READ_TIME = wiredtiger.stat.session.read_time session_stats = { BYTES_READ : "session: bytes read into cache", \ READ_TIME : "session: page read from disk to cache time (usecs)"} + def get_stat(self, stat): + statc = self.session.open_cursor('statistics:session', None, None) + val = statc[stat][2] + statc.close() + return val + + def get_cstat(self, stat): + statc = self.session.open_cursor('statistics:', None, None) + val = statc[stat][2] + statc.close() + return val + def check_stats(self, cur, k): # # Some Windows machines lack the time granularity to detect microseconds. @@ -59,13 +74,33 @@ class test_stat08(wttest.WiredTigerTestCase): self.assertTrue(value > 0) def test_session_stats(self): - self.session = self.conn.open_session() - self.session.create("table:test_stat08", - "key_format=i,value_format=S") + # We want to configure for pages to be explicitly evicted when we are done with them so + # that we can correctly verify the statistic measuring bytes read from cache. + self.session = self.conn.open_session("debug=(release_evict_page=true)") + self.session.create("table:test_stat08", "key_format=i,value_format=S") cursor = self.session.open_cursor('table:test_stat08', None, None) + self.session.begin_transaction() + txn_dirty = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) + cache_dirty = self.get_cstat(wiredtiger.stat.conn.cache_bytes_dirty) + self.assertEqual(txn_dirty, 0) + self.assertLessEqual(txn_dirty, cache_dirty) # Write the entries. - for i in range(0, self.nentries): + for i in range(1, self.nentries): + txn_dirty_before = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) cursor[i] = self.entry_value + txn_dirty_after = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) + self.assertLess(txn_dirty_before, txn_dirty_after) + # Since we're using an explicit transaction, we need to resolve somewhat frequently. + # So check the statistics and restart the transaction every 200 operations. + if i % 200 == 0: + cache_dirty_txn = self.get_cstat(wiredtiger.stat.conn.cache_bytes_dirty) + # Make sure the txn's dirty bytes doesn't exceed the cache. + self.assertLessEqual(txn_dirty_after, cache_dirty_txn) + self.session.rollback_transaction() + self.session.begin_transaction() + txn_dirty = self.get_stat(wiredtiger.stat.session.txn_bytes_dirty) + self.assertEqual(txn_dirty, 0) + self.session.commit_transaction() cursor.reset() # Read the entries. diff --git a/src/third_party/wiredtiger/test/suite/test_sweep04.py b/src/third_party/wiredtiger/test/suite/test_sweep04.py new file mode 100755 index 00000000000..a741639f82c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_sweep04.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_sweep04.py +# Test lots of tables with more steadily created and dropped. +# A core group of tables is used most often and hangs around. +# Test that the total number of dhandles, while increasing, +# starts to level off, that is, the sweeps are keeping up. +# Then test that if we only access the core tables for a while, +# the total number of dhandles comes back down to a small number. + +import time +from suite_random import suite_random +from wiredtiger import stat +import wttest + +# Given a set of values corresponding to successive times, +# we have an implied set of points in two dimensions. +# Compute the average for the values and the slope for the +# least squares regression of the line. +# +# We'd use numpy, but we aren't assured that it is always installed +# for our python. +def average_slope(y): + n = len(y) + if n == 0: + return [0, 0] # there's no average or slope + elif n == 1: + return [y[0], 0] # there's no slope + + average = sum(y) / n + x = range(1, n + 1) # The implied x axis, steadily increasing integers. + + # Here's a formula for least squares slope: + # https://www.mathsisfun.com/data/least-squares-regression.html + top = n * sum(x[i] * y[i] for i in range(n)) - sum(x) * sum(y) + bottom = n * sum(x[i]**2 for i in range(n)) - sum(x)**2 + slope = top / bottom + return [average, slope] + +@wttest.longtest("lots of files") +class test_sweep04(wttest.WiredTigerTestCase): + tablebase = 'test_sweep04' + uri = 'table:' + tablebase + + # Configuration values for the run. If any of these values are changed, + # the simulation will run, but the acceptance criteria may fail at the end. + core_tables = 10 # Number of core tables that always exist. + transient_tables = 100 # Number of transient tables at any time. + transient_examined = 10 # Number of transient tables opened at a time. + ratio_examined = 0.01 # The chance that transient tables are examined. + transient_table_max = 10000 # Defines the length of the run. + numkv = 1 # Number of k/v pairs. Shouldn't matter for this test. + nsessions = 100 # Number of sessions in our pool. + + conn_config = 'file_manager=(close_handle_minimum=0,' + \ + 'close_idle_time=3,close_scan_interval=1),' + \ + 'statistics=(fast),operation_tracking=(enabled=false),' + + create_params = 'key_format=i,value_format=i' + + # Create a uri for one of the core tables + def core_uri(self, i): + return '%s-c.%d' % (self.uri, i) + + # Create a uri for one of the transient tables + def transient_uri(self, i): + return '%s-t.%d' % (self.uri, i) + + def create_table(self, uri): + self.session.create(uri, self.create_params) + c = self.session.open_cursor(uri, None) + for k in range(self.numkv): + c[k+1] = 1 + c.close() + + # Access a set of tables in some minimal way that ensures its + # dhandle is at least momentarily in use. "uri_maker" is a + # function that is used to create the uri. + def examine(self, session, uri_maker, start, count): + for i in range(0, count): + c = session.open_cursor(uri_maker(start + i)) + self.assertEquals(c[1], 1) + c.close() + + def test_big_run(self): + # populate + r = suite_random() + + # Create the set of core tables + for i in range(0, self.core_tables): + self.create_table(self.core_uri(i)) + + created_transient = 0 # Running total of transients created. + available_transient = 0 # The next available transient number. + + # Create the initial batch of transient tables. + while created_transient < self.transient_tables: + self.create_table(self.transient_uri(created_transient)) + created_transient += 1 + + # Open all the session we'll use in advance. + sessions = [] + for i in range(self.nsessions): + sessions.append(self.conn.open_session()) + + # We keep the dhandle counts for each time we get stats. + dhandle_counts = [] + + # The big loop: For half the run, we are stressing by accessing both core tables + # and transient tables, and creating/dropping transient tables. The second half + # of the run, we stop creating/dropping tables and only access core tables + # just to see if all outstanding dhandles are swept. + maxloop = self.transient_table_max * 2 + lasttime = time.time() + for loopcount in range(0, maxloop): + stressing = (created_transient < self.transient_table_max) + if loopcount % 100 == 0: + self.pr('{}/{} stressing={}'.format(loopcount, maxloop, stressing)) + + # Make sure at least 3 seconds elapses between each 100 times through + # the loop, to give the various sweeps time to operate. + thistime = time.time() + delta = thistime - lasttime + if delta < 3.0: + time.sleep(3.0 - delta) + lasttime = thistime + + if stressing: + self.session.drop(self.transient_uri(available_transient), "force") + available_transient += 1 + self.create_table(self.transient_uri(created_transient)) + created_transient += 1 + + rand_session = sessions[r.rand_range(0, self.nsessions)] + + # In the stress part of the run, some small fraction (given by ratio_examined) will + # look at the transient tables. Looking at these rarely makes them candidates for + # closing by the connection sweep. + big = 1000000 # Any large number works. + if stressing and r.rand32() % big > big * self.ratio_examined: + # Access "count" transient tables, starting at table "tnum". + count = self.transient_examined + tnum = r.rand_range(available_transient, available_transient + self.transient_tables - count) + self.examine(rand_session, self.transient_uri, tnum, count) + else: + # Access a single core table, numbered "tnum". + tnum = r.rand_range(0, self.core_tables) + self.examine(rand_session, self.core_uri, tnum, 1) + + if loopcount % 100 == 99: + # Gather statistics about the number of dhandles, to be checked at + # the end of the run. + stat_cursor = self.session.open_cursor('statistics:', None, None) + + # Enable for detailed output. + if False: + close = stat_cursor[stat.conn.dh_sweep_close][2] + remove = stat_cursor[stat.conn.dh_sweep_remove][2] + sweep = stat_cursor[stat.conn.dh_sweeps][2] + sclose = stat_cursor[stat.conn.dh_session_handles][2] + ssweep = stat_cursor[stat.conn.dh_session_sweeps][2] + tod = stat_cursor[stat.conn.dh_sweep_tod][2] + ref = stat_cursor[stat.conn.dh_sweep_ref][2] + self.pr(('DHANDLE STATS: close={}, remove={}, sweep={}, session_handles={}, '+ + 'session_sweeps={}, sweep_tod={}, sweep_ref={}').format( + close, remove, sweep, sclose, ssweep, tod, ref)) + + dhandles = stat_cursor[stat.conn.dh_conn_handle_count][2] + files_open = stat_cursor[stat.conn.file_open][2] + self.pr(' dhandle_count={}'.format(dhandles)) + self.pr(' file_open={}'.format(files_open)) + dhandle_counts.append(dhandles) + stat_cursor.close() + + # This (extremely verbose) debugging is disabled, as it writes to stdout, + # and that causes the test framework fail the test. + if False: + if loopcount % 1000 == 999: + self.conn.debug_info('handles=true') + self.pr(' average,slope={}'.format(str(average_slope(dhandle_counts)))) + + # Reset any sessions we've used. This may be necessary to trigger some session sweeps. + rand_session.reset() + self.session.reset() + + # The run is finished, process and check the dhandle counts we've collected. + # + # Half the run is dhandle growth, and the second half should see a decline. + # If everything is working right in the first half of the run, we should start + # to see the number of dhandles start to reach an asymptote. So we check the slope of + # the dhandle line in the second quarter to see that is the case. In the second + # half of the run, where we aren't accessing the transient tables, those references + # should free up, and we should see an asymptote at the end much closer to the + # number of files. + half = len(dhandle_counts)//2 + qtr = len(dhandle_counts)//4 + tenth = len(dhandle_counts)//10 + + (q1_avg, q1_slope) = average_slope(dhandle_counts[0:qtr]) + (q2_avg, q2_slope) = average_slope(dhandle_counts[qtr:half]) + (q3_avg, q3_slope) = average_slope(dhandle_counts[half:-qtr]) + (q4_avg, q4_slope) = average_slope(dhandle_counts[-qtr:]) + (end_run_avg, end_run_slope) = average_slope(dhandle_counts[-tenth:]) + + self.pr('1st qtr: average={},slope={}'.format(q1_avg, q1_slope)) + self.pr('2nd qtr: average={},slope={}'.format(q2_avg, q2_slope)) + self.pr('3rd qtr: average={},slope={}'.format(q3_avg, q3_slope)) + self.pr('4th qtr: average={},slope={}'.format(q4_avg, q4_slope)) + + self.pr('end run: average={},slope={}'.format(end_run_avg, end_run_slope)) + + # Note, we don't check the first half average, it's likely to be big, but its size + # depends on many factors. The important thing is that the slope has flattened out. + # Even with variations due to sweep timing, the slope shouldn't be greater than + # 15.0 (dhandles per 100 times through the loop). + self.assertLess(abs(q2_slope), q1_slope) + self.assertLess(abs(q2_slope), 15.0) + + # At the end of the run, we expect a pretty flat slope and a pretty small number + # of dhandles. A slope of 5.0 (dhandles per 100 times though the loop) is rather + # flat and still leaves room for some variation. + self.assertLess(abs(end_run_slope), 5.0) + self.assertLess(end_run_avg, self.core_tables + self.transient_tables + 20) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py index 8b638788736..c109596361f 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -71,8 +71,7 @@ class test_tiered05(wttest.WiredTigerTestCase): 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ - 'name=%s,' % self.ss_name + \ - 'object_target_size=20M)' + 'name=%s)' % self.ss_name # Test calling the flush_tier API with a tiered manager. Should get an error. def test_tiered(self): diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index c4e931c7a3f..d876b305c75 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -58,7 +58,7 @@ class test_tiered06(wttest.WiredTigerTestCase): config = '' # S3 store is built as an optional loadable extension, not all test environments build S3. if self.ss_name == 's3_store': - #config = '=(config=\"(verbose=1)\")' + #config = '=(config=\"(verbose=[api:1,version,tiered:1])\")' extlist.skip_if_missing = True #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' @@ -104,22 +104,14 @@ class test_tiered06(wttest.WiredTigerTestCase): self.get_fs_config(prefix)) # The object doesn't exist yet. - if self.ss_name == 's3_store': - with self.expectedStderrPattern('.*HTTP response code: 404.*'): - self.assertFalse(fs.fs_exist(session, 'foobar')) - else: - self.assertFalse(fs.fs_exist(session, 'foobar')) + self.assertFalse(fs.fs_exist(session, 'foobar')) # We cannot use the file system to create files, it is readonly. # So use python I/O to build up the file. f = open('foobar', 'wb') # The object still doesn't exist yet. - if self.ss_name == 's3_store': - with self.expectedStderrPattern('.*HTTP response code: 404.*'): - self.assertFalse(fs.fs_exist(session, 'foobar')) - else: - self.assertFalse(fs.fs_exist(session, 'foobar')) + self.assertFalse(fs.fs_exist(session, 'foobar')) outbytes = ('MORE THAN ENOUGH DATA\n'*100000).encode() f.write(outbytes) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered14.py b/src/third_party/wiredtiger/test/suite/test_tiered14.py index 61e3a54cf91..c041bb11635 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered14.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered14.py @@ -28,7 +28,7 @@ from helper_tiered import generate_s3_prefix, get_auth_token, get_bucket1_name from wtscenario import make_scenarios -import os, random, wtscenario, wttest +import os, random, wttest from wtdataset import TrackedSimpleDataSet, TrackedComplexDataSet # test_tiered14.py @@ -69,7 +69,7 @@ class test_tiered14(wttest.WiredTigerTestCase): num_ops = 20, ss_name = 's3_store')), ] - scenarios = wtscenario.make_scenarios(multiplier, keyfmt, dataset, storage_sources) + scenarios = make_scenarios(multiplier, keyfmt, dataset, storage_sources) def conn_config(self): if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): @@ -86,7 +86,7 @@ class test_tiered14(wttest.WiredTigerTestCase): config = '' # S3 store is built as an optional loadable extension, not all test environments build S3. if self.ss_name == 's3_store': - #config = '=(config=\"(verbose=1)\")' + #config = '=(config=\"(verbose=[api:1,version,tiered:-3])\")' extlist.skip_if_missing = True #if self.ss_name == 'dir_store': #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index edd22e346c7..a751231f334 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -42,7 +42,7 @@ except ImportError: import unittest from contextlib import contextmanager -import errno, glob, os, re, shutil, sys, time, traceback +import errno, glob, os, re, shutil, sys, threading, time, traceback, types import wiredtiger, wtscenario, wthooks def shortenWithEllipsis(s, maxlen): @@ -180,6 +180,38 @@ class ExtensionList(list): ext = '' if extarg == None else '=' + extarg self.append(dirname + '/' + name + ext) +# Custom result class that will prefix the pid in text output (including if it's a child). +# Only enabled when we are in verbose mode so we don't check that here. +class PidAwareTextTestResult(unittest.TextTestResult): + _thread_prefix = threading.local() + + def __init__(self, stream, descriptions, verbosity): + super(PidAwareTextTestResult, self).__init__(stream, descriptions, verbosity) + self._thread_prefix.value = "[pid:{}]: ".format(os.getpid()) + + def tags(self, new_tags, gone_tags): + # We attach the PID to the thread so we only need the new_tags. + for tag in new_tags: + if tag.startswith("pid:"): + pid = tag[len("pid:"):] + self._thread_prefix.value = "[pid:{}/{}]: ".format(os.getpid(), pid) + + def startTest(self, test): + self.stream.write(self._thread_prefix.value) + super(PidAwareTextTestResult, self).startTest(test) + + def getDescription(self, test): + return str(test.shortDescription()) + + def printErrorList(self, flavour, errors): + for test, err in errors: + self.stream.writeln(self.separator1) + self.stream.writeln("%s%s: %s" % (self._thread_prefix.value, + flavour, self.getDescription(test))) + self.stream.writeln(self.separator2) + self.stream.writeln("%s%s" % (self._thread_prefix.value, err)) + self.stream.flush() + class WiredTigerTestCase(unittest.TestCase): _globalSetup = False _printOnceSeen = {} @@ -536,8 +568,11 @@ class WiredTigerTestCase(unittest.TestCase): # In addition, check to make sure exc_info is "clean", because # the ConcurrencyTestSuite in Python2 indicates failures using that. if hasattr(self, '_outcome'): # Python 3.4+ - result = self.defaultTestResult() # these 2 methods have no side effects - self._feedErrorsToResult(result, self._outcome.errors) + if hasattr(self._outcome, 'errors'): # Python 3.4 - 3.10 + result = self.defaultTestResult() # these 2 methods have no side effects + self._feedErrorsToResult(result, self._outcome.errors) + else: # Python 3.11+ + result = self._outcome.result else: # Python 3.2 - 3.3 or 3.0 - 3.1 and 2.7 result = getattr(self, '_outcomeForDoCleanups', self._resultForDoCleanups) error = self.list2reason(result, 'errors') @@ -586,9 +621,9 @@ class WiredTigerTestCase(unittest.TestCase): elapsed = time.time() - self.starttime if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: - print("%s: %.2f seconds" % (str(self), elapsed)) + print("[pid:{}]: {}: {:.2f} seconds".format(os.getpid(), str(self), elapsed)) if (not passed) and (not self.skipped): - print("ERROR in " + str(self)) + print("[pid:{}]: ERROR in {}".format(os.getpid(), str(self))) self.pr('FAIL') self.pr('preserving directory ' + self.testdir) if WiredTigerTestCase._verbose > 2: @@ -834,7 +869,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def prout(s): - os.write(WiredTigerTestCase._dupout, str.encode(s + '\n')) + os.write(WiredTigerTestCase._dupout, str.encode("[pid:{}]: {}\n".format(os.getpid(), s))) def pr(self, s): """ @@ -873,7 +908,7 @@ class WiredTigerTestCase(unittest.TestCase): def tty(message): if WiredTigerTestCase._ttyDescriptor == None: WiredTigerTestCase._ttyDescriptor = open('/dev/tty', 'w') - WiredTigerTestCase._ttyDescriptor.write(message + '\n') + WiredTigerTestCase._ttyDescriptor.write("[pid:{}]: {}\n".format(os.getpid(), message)) def ttyVerbose(self, level, message): WiredTigerTestCase.ttyVerbose(level, message) @@ -935,6 +970,16 @@ def islongtest(): def getseed(): return WiredTigerTestCase._seeds +# We have to override the ThreadsafeForwardingResult implementation of tags so it gets set immediately +# which allows us to set the pid of the process on our output stream to make debugging easier. +def immediate_tags(self, new_tags, gone_tags): + self.result.tags(new_tags, gone_tags) + +def wrap_result_for_tags(thread_safe_result, thread_number): + # We use this technique to override the method instead of extending the class as it allows for less changes. + thread_safe_result.tags = types.MethodType(immediate_tags, thread_safe_result) + return thread_safe_result + def runsuite(suite, parallel): suite_to_run = suite if parallel > 1: @@ -942,16 +987,19 @@ def runsuite(suite, parallel): if not WiredTigerTestCase._globalSetup: WiredTigerTestCase.globalSetup() WiredTigerTestCase._concurrent = True - suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel)) + suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel), wrap_result=wrap_result_for_tags) try: if WiredTigerTestCase._randomseed: WiredTigerTestCase.prout("Starting test suite with seedw={0} and seedz={1}. Rerun this test with -seed {0}.{1} to get the same randomness" .format(str(WiredTigerTestCase._seeds[0]), str(WiredTigerTestCase._seeds[1]))) + result_class = None + if WiredTigerTestCase._verbose > 1: + result_class = PidAwareTextTestResult return unittest.TextTestRunner( - verbosity=WiredTigerTestCase._verbose).run(suite_to_run) + verbosity=WiredTigerTestCase._verbose, resultclass=result_class).run(suite_to_run) except BaseException as e: # This should not happen for regular test errors, unittest should catch everything - print('ERROR: running test: ', e) + print("[pid:{}]: ERROR: running test: {}".format(os.getpid(), e)) raise e def run(name='__main__'): |