diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
9 files changed, 313 insertions, 22 deletions
diff --git a/src/third_party/wiredtiger/test/mciproject.yml b/src/third_party/wiredtiger/test/mciproject.yml index 6456475aa00..50a910d9e58 100644 --- a/src/third_party/wiredtiger/test/mciproject.yml +++ b/src/third_party/wiredtiger/test/mciproject.yml @@ -157,20 +157,6 @@ buildvariants: - name: unit-test - name: fops -- name: solaris - display_name: Solaris - run_on: - - solaris - expansions: - make_command: PATH=/opt/mongodbtoolchain/bin:$PATH gmake - test_env_vars: LD_LIBRARY_PATH=`pwd`/.libs - smp_command: -j $(kstat cpu | sort -u | grep -c "^module") - configure_env_vars: PATH=/opt/mongodbtoolchain/bin:$PATH CFLAGS="-m64" - tasks: - - name: compile - - name: unit-test - - name: fops - - name: windows-64 display_name: Windows 64-bit run_on: diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index febe6530534..b53383e5730 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -47,9 +47,9 @@ static bool inmem; #define RECORDS_FILE "records-%" PRIu32 #define ENV_CONFIG_DEF \ - "create,log=(file_max=10M,archive=false,enabled)" + "create,log=(file_max=10M,enabled)" #define ENV_CONFIG_TXNSYNC \ - "create,log=(file_max=10M,archive=false,enabled)," \ + "create,log=(file_max=10M,enabled)," \ "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(recover=on)" #define MAX_VAL 4096 diff --git a/src/third_party/wiredtiger/test/suite/test_bug018.py b/src/third_party/wiredtiger/test/suite/test_bug018.py new file mode 100644 index 00000000000..7d20ebcaacb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_bug018.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import copy_wiredtiger_home +import os +import wiredtiger, wttest + +# test_bug018.py +# JIRA WT-3590: if writing table data fails during close then tables +# that were updated within the same transaction could get out of sync with +# each other. +class test_bug018(wttest.WiredTigerTestCase): + '''Test closing/reopening/recovering tables when writes fail''' + + conn_config = 'log=(enabled)' + + def setUp(self): + # This test uses Linux-specific code so skip on any other system. + if os.name != 'posix' or os.uname()[0] != 'Linux': + self.skipTest('Linux-specific test skipped on ' + os.name) + super(test_bug018, self).setUp() + + def create_table(self, uri): + self.session.create(uri, 'key_format=S,value_format=S') + return self.session.open_cursor(uri) + + def test_bug018(self): + '''Test closing multiple tables''' + basename = 'bug018.' + baseuri = 'file:' + basename + c1 = self.create_table(baseuri + '01.wt') + c2 = self.create_table(baseuri + '02.wt') + + self.session.begin_transaction() + c1['key'] = 'value' + c2['key'] = 'value' + self.session.commit_transaction() + + # Simulate a write failure by closing the file descriptor for the second + # table out from underneath WiredTiger. We do this right before + # closing the connection so that the write error happens during close + # when writing out the final data. Allow table 1 to succeed and force + # an erorr writing out table 2. + # + # This is Linux-specific code to figure out the file descriptor. + for f in os.listdir('/proc/self/fd'): + try: + if os.readlink('/proc/self/fd/' + f).endswith(basename + '02.wt'): + os.close(int(f)) + except OSError: + pass + + # Expect an error and messages, so turn off stderr checking. + with self.expectedStderrPattern(''): + try: + self.close_conn() + except wiredtiger.WiredTigerError: + self.conn = None + + # Make a backup for forensics in case something goes wrong. + backup_dir = 'BACKUP' + copy_wiredtiger_home('.', backup_dir, True) + + # After reopening and running recovery both tables should be in + # sync even though table 1 was successfully written and table 2 + # had an error on close. + self.open_conn() + c1 = self.session.open_cursor(baseuri + '01.wt') + c2 = self.session.open_cursor(baseuri + '02.wt') + self.assertEqual(list(c1), list(c2)) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 388485db29b..d280642942a 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -108,12 +108,15 @@ class test_inmem01(wttest.WiredTigerTestCase): cursor.reset() # Spin inserting to give eviction a chance to reclaim space + sleeps = 0 inserted = False for i in range(1, 1000): try: cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() + sleeps = sleeps + 1 + self.assertLess(sleeps, 60 * 5) sleep(1) continue inserted = True diff --git a/src/third_party/wiredtiger/test/suite/test_las.py b/src/third_party/wiredtiger/test/suite/test_las.py new file mode 100644 index 00000000000..d0bd1d108fa --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_las.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# test_las.py +# Smoke tests to ensure lookaside tables are working. +class test_las(wttest.WiredTigerTestCase): + # Force a small cache. + def conn_config(self): + return 'cache_size=1GB' + + @wttest.longtest('lookaside table smoke test') + def test_las(self): + # Create a small table. + uri = "table:test_las" + nrows = 100 + ds = SimpleDataSet(self, uri, nrows, key_format="S") + ds.populate() + + # Take a snapshot. + self.session.snapshot("name=xxx") + + # Insert a large number of records, we'll hang if the lookaside table + # isn't doing its thing. + c = self.session.open_cursor(uri) + bigvalue = "abcde" * 100 + for i in range(1, 1000000): + c.set_key(ds.key(nrows + i)) + c.set_value(bigvalue) + self.assertEquals(c.insert(), 0) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index 01626057b9e..76a325743e9 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -169,7 +169,6 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): try: session = backup_conn.open_session() finally: - session.checkpoint("force") self.check(backup_conn.open_session(), None, committed) # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 7aaff221ba4..7099bc972aa 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -134,12 +134,12 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): session = backup_conn.open_session() finally: self.check(session, None, committed) - # Force a checkpoint because we don't record the recovery - # checkpoint as available for archiving. - session.checkpoint("force") # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) + if count == 0: + first_logs = \ + fnmatch.filter(os.listdir(self.backup_dir), "*Log*") backup_conn.close() count += 1 # @@ -149,6 +149,11 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): # cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") for o in orig_logs: + # Creating the backup was effectively an unclean shutdown so + # even after sleeping, we should never archive log files + # because a checkpoint has not run. Later opens and runs of + # recovery will detect a clean shutdown and allow archiving. + self.assertEqual(True, o in first_logs) if self.archive == 'true': self.assertEqual(False, o in cur_logs) else: diff --git a/src/third_party/wiredtiger/test/suite/test_txn09.py b/src/third_party/wiredtiger/test/suite/test_txn09.py index 768d714e248..b8a3d7f38ae 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn09.py +++ b/src/third_party/wiredtiger/test/suite/test_txn09.py @@ -26,8 +26,8 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -# test_txn02.py -# Transactions: commits and rollbacks +# test_txn09.py +# Transactions: recovery toggling logging # import fnmatch, os, shutil, time diff --git a/src/third_party/wiredtiger/test/suite/test_txn16.py b/src/third_party/wiredtiger/test/suite/test_txn16.py new file mode 100644 index 00000000000..929da2291c7 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn16.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn16.py +# Recovery: Test that toggling between logging and not logging does not +# continue to generate more log files. +# + +import fnmatch, os, shutil, time +from suite_subprocess import suite_subprocess +import wttest + +class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): + t1 = 'table:test_txn16_1' + t2 = 'table:test_txn16_2' + t3 = 'table:test_txn16_3' + nentries = 1000 + create_params = 'key_format=i,value_format=i' + # Set the log file size small so we generate checkpoints + # with LSNs in different files. + conn_config = 'config_base=false,' + \ + 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(method=dsync,enabled)' + conn_on = 'config_base=false,' + \ + 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(method=dsync,enabled)' + conn_off = 'config_base=false,log=(enabled=false)' + + def populate_table(self, uri): + self.session.create(uri, self.create_params) + c = self.session.open_cursor(uri, None, None) + # Populate with an occasional checkpoint to generate + # some varying LSNs. + for i in range(self.nentries): + c[i] = i + 1 + if i % 900 == 0: + self.session.checkpoint() + c.close() + + def copy_dir(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + # close the original connection. + self.close_conn() + + def run_toggle(self, homedir): + loop = 0 + # Record original log files. There should never be overlap + # with these even after they're removed. + orig_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + while loop < 3: + # Reopen with logging on to run recovery first time + on_conn = self.wiredtiger_open(homedir, self.conn_on) + on_conn.close() + if loop > 0: + # Get current log files. + cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + scur = set(cur_logs) + sorig = set(orig_logs) + # There should never be overlap with the log files that + # were there originally. Mostly this checks that after + # opening with logging disabled and then re-enabled, we + # don't see log file 1. + self.assertEqual(scur.isdisjoint(sorig), True) + if loop > 1: + # We should be creating the same log files each time. + for l in cur_logs: + self.assertEqual(l in last_logs, True) + for l in last_logs: + self.assertEqual(l in cur_logs, True) + last_logs = cur_logs + loop += 1 + # Remove all log files before opening without logging. + cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + for l in cur_logs: + path=homedir + "/" + l + os.remove(path) + off_conn = self.wiredtiger_open(homedir, self.conn_off) + off_conn.close() + + def test_recovery(self): + ''' Check log file creation when toggling. ''' + + # Here's the strategy: + # - With logging populate 4 tables. Checkpoint + # them at different times. + # - Copy to a new directory to simulate a crash. + # - Close the original connection. + # On both a "copy" to simulate a crash and the original (3x): + # - Record log files existing. + # - Reopen with logging to run recovery. Close connection. + # - Record log files existing. + # - Remove all log files. + # - Open connection with logging disabled. + # - Record log files existing. Verify we don't keep adding. + # + self.populate_table(self.t1) + self.populate_table(self.t2) + self.populate_table(self.t3) + self.copy_dir(".", "RESTART") + self.run_toggle(".") + self.run_toggle("RESTART") + +if __name__ == '__main__': + wttest.run() |