diff options
Diffstat (limited to 'bench/wtperf')
28 files changed, 423 insertions, 155 deletions
diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index 5b14a4cdf68..e4eee66e4cb 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -215,6 +215,7 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) return (EINVAL); } workp = &wtperf->workload[wtperf->workload_cnt++]; + workp->table_index = INT32_MAX; while ((ret = scan->next(scan, &k, &v)) == 0) { if (STRING_MATCH("count", k.str, k.len)) { @@ -233,12 +234,28 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) goto err; continue; } + if (STRING_MATCH("pause", k.str, k.len)) { + if ((workp->pause = v.val) < 0) + goto err; + continue; + } if (STRING_MATCH("read", k.str, k.len) || STRING_MATCH("reads", k.str, k.len)) { if ((workp->read = v.val) < 0) goto err; continue; } + if (STRING_MATCH("read_range", k.str, k.len)) { + if ((workp->read_range = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("table", k.str, k.len)) { + if (v.val <= 0) + goto err; + workp->table_index = (int32_t)v.val - 1; + continue; + } if (STRING_MATCH("throttle", k.str, k.len)) { workp->throttle = (uint64_t)v.val; continue; @@ -421,14 +438,13 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) return (EINVAL); } strp = (char **)valueloc; - newlen = v->len + 1; if (*strp == NULL) begin = newstr = dstrdup(v->str); else { - newlen += strlen(*strp) + 1; - newstr = dcalloc(newlen, sizeof(char)); - snprintf(newstr, newlen, - "%s,%*s", *strp, (int)v->len, v->str); + newlen = strlen(*strp) + v->len + strlen(",") + 1; + newstr = dmalloc(newlen); + testutil_check(__wt_snprintf(newstr, newlen, + "%s,%.*s", *strp, (int)v->len, v->str)); /* Free the old value now we've copied it. */ free(*strp); begin = &newstr[(newlen - 1) - v->len]; @@ -622,17 +638,9 @@ config_opt_str(WTPERF *wtperf, const char *optstr) return (ret); } - /* - * Append the current line to our copy of the config. The config is - * stored in the order it is processed, so added options will be after - * any parsed from the original config. We allocate len + 1 to allow for - * a null byte to be added. - */ - config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); - config_line->string = dstrdup(optstr); - TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); - while (ret == 0) { + size_t pos; + if ((ret = scan->next(scan, &k, &v)) != 0) { /* Any parse error has already been reported. */ if (ret == WT_NOTFOUND) @@ -640,6 +648,46 @@ config_opt_str(WTPERF *wtperf, const char *optstr) break; } ret = config_opt(wtperf, &k, &v); + + /* + * Append the key-value pair to our copy of the config. + * The config is stored in the order it is processed, so added + * options will be after any parsed from the original config. + */ + config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); + /* + * If key or value is a string, consider extra space for the + * quotes. Add 2 to the required space for '=' and the ending + * null character in "key=value". + */ + config_line->string = dcalloc( + k.len + (k.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + + v.len + (v.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + 2, 1); + pos = 0; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, k.str, k.len); + pos += k.len; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '='; + pos++; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, v.str, v.len); + pos += v.len; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '\0'; + TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); } if ((t_ret = scan->close(scan)) != 0) { lprintf(wtperf, ret, 0, "Error in config_scan_end"); @@ -663,7 +711,7 @@ config_opt_name_value(WTPERF *wtperf, const char *name, const char *value) /* name="value" */ len = strlen(name) + strlen(value) + 4; optstr = dmalloc(len); - snprintf(optstr, len, "%s=\"%s\"", name, value); + testutil_check(__wt_snprintf(optstr, len, "%s=\"%s\"", name, value)); ret = config_opt_str(wtperf, optstr); free(optstr); return (ret); @@ -728,16 +776,33 @@ config_sanity(WTPERF *wtperf) opts->value_sz_min = opts->value_sz; } - if (opts->readonly && wtperf->workload != NULL) + if (wtperf->workload != NULL) for (i = 0, workp = wtperf->workload; - i < wtperf->workload_cnt; ++i, ++workp) - if (workp->insert != 0 || workp->update != 0 || - workp->truncate != 0) { + i < wtperf->workload_cnt; ++i, ++workp) { + if (opts->readonly && + (workp->insert != 0 || workp->update != 0 || + workp->truncate != 0)) { fprintf(stderr, "Invalid workload: insert, update or " "truncate specified with readonly\n"); return (EINVAL); } + if (workp->insert != 0 && + workp->table_index != INT32_MAX) { + fprintf(stderr, + "Invalid workload: Cannot insert into " + "specific table only\n"); + return (EINVAL); + } + if (workp->table_index != INT32_MAX && + workp->table_index >= (int32_t)opts->table_count) { + fprintf(stderr, + "Workload table index %" PRId32 + " is larger than table count %" PRId32, + workp->table_index, opts->table_count); + return (EINVAL); + } + } return (0); } @@ -754,8 +819,11 @@ config_consolidate(CONFIG_OPTS *opts) /* * This loop iterates over the config queue and for each entry checks if - * a later queue entry has the same key. If there's a match, the current - * queue entry is removed and we continue. + * a later queue entry has the same key. If there's a match, and key is + * "conn_config" or "table_config", the later queue entry is replaced + * with a concatenated entry of the two queue entries, the current queue + * entry is removed. For any other key, if there is a match, the current + * queue entry is removed. */ conf_line = TAILQ_FIRST(&opts->config_head); while (conf_line != NULL) { @@ -771,6 +839,34 @@ config_consolidate(CONFIG_OPTS *opts) if (strncmp(conf_line->string, test_line->string, (size_t)((string_key - conf_line->string) + 1)) == 0) { + if ((strncmp("conn_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + + 1)) == 0) || + (strncmp("table_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + + 1)) == 0)) { + char *concat_str, *val_pointer; + + /* + * To concatenate the two config + * strings, copy the first string to a + * new one, replace the ending '"' with + * a ',' and then concatenate the second + * string's value after its starting '"' + */ + val_pointer = + strchr(test_line->string, '=') + 2; + concat_str = + dmalloc(strlen(conf_line->string) + + strlen(val_pointer) + 1); + strcpy(concat_str, conf_line->string); + concat_str[strlen(concat_str) - 1] = + ','; + strcat(concat_str, val_pointer); + free(test_line->string); + test_line->string = concat_str; + } + TAILQ_REMOVE(&opts->config_head, conf_line, q); free(conf_line->string); free(conf_line); diff --git a/bench/wtperf/idle_table_cycle.c b/bench/wtperf/idle_table_cycle.c index 13fa55e86f5..4387860cfb2 100644 --- a/bench/wtperf/idle_table_cycle.c +++ b/bench/wtperf/idle_table_cycle.c @@ -80,8 +80,8 @@ cycle_idle_tables(void *arg) } for (cycle_count = 0; wtperf->idle_cycle_run; ++cycle_count) { - snprintf(uri, sizeof(uri), - "%s_cycle%07d", wtperf->uris[0], cycle_count); + testutil_check(__wt_snprintf(uri, sizeof(uri), + "%s_cycle%07d", wtperf->uris[0], cycle_count)); /* Don't busy cycle in this loop. */ __wt_sleep(1, 0); @@ -120,6 +120,7 @@ cycle_idle_tables(void *arg) return (NULL); start = stop; +#if 1 /* * Drop the table. Keep retrying on EBUSY failure - it is an * expected return when checkpoints are happening. @@ -136,6 +137,7 @@ cycle_idle_tables(void *arg) } if (check_timing(wtperf, "drop", start, &stop) != 0) return (NULL); +#endif } return (NULL); diff --git a/bench/wtperf/misc.c b/bench/wtperf/misc.c index 24b3323a49a..0874794e01e 100644 --- a/bench/wtperf/misc.c +++ b/bench/wtperf/misc.c @@ -46,8 +46,8 @@ setup_log_file(WTPERF *wtperf) len = strlen(wtperf->monitor_dir) + strlen(opts->table_name) + strlen(".stat") + 2; fname = dmalloc(len); - snprintf(fname, len, - "%s/%s.stat", wtperf->monitor_dir, opts->table_name); + testutil_check(__wt_snprintf(fname, len, + "%s/%s.stat", wtperf->monitor_dir, opts->table_name)); if ((wtperf->logf = fopen(fname, "w")) == NULL) { ret = errno; fprintf(stderr, "%s: %s\n", fname, strerror(ret)); diff --git a/bench/wtperf/runners/500m-btree-50r50u.wtperf b/bench/wtperf/runners/500m-btree-50r50u.wtperf index 536127f0dd8..4d2a70f1107 100644 --- a/bench/wtperf/runners/500m-btree-50r50u.wtperf +++ b/bench/wtperf/runners/500m-btree-50r50u.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" sess_config="isolation=snapshot" diff --git a/bench/wtperf/runners/500m-btree-80r20u.wtperf b/bench/wtperf/runners/500m-btree-80r20u.wtperf index d6218c44af0..6645df835df 100644 --- a/bench/wtperf/runners/500m-btree-80r20u.wtperf +++ b/bench/wtperf/runners/500m-btree-80r20u.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" # close_conn as false allows this test to close/finish faster, but if running diff --git a/bench/wtperf/runners/500m-btree-populate.wtperf b/bench/wtperf/runners/500m-btree-populate.wtperf index f9aed094aa1..ab7b17ca683 100644 --- a/bench/wtperf/runners/500m-btree-populate.wtperf +++ b/bench/wtperf/runners/500m-btree-populate.wtperf @@ -9,7 +9,7 @@ # # This generates about 80 Gb of uncompressed data. But it should compress # well and be small on disk. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" compact=true compression="snappy" sess_config="isolation=snapshot" diff --git a/bench/wtperf/runners/500m-btree-rdonly.wtperf b/bench/wtperf/runners/500m-btree-rdonly.wtperf index 2c9540ff589..e8958d20e2c 100644 --- a/bench/wtperf/runners/500m-btree-rdonly.wtperf +++ b/bench/wtperf/runners/500m-btree-rdonly.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" sess_config="isolation=snapshot" diff --git a/bench/wtperf/runners/checkpoint-stress.wtperf b/bench/wtperf/runners/checkpoint-stress.wtperf index bbd3a3ba5ed..5daa276e622 100644 --- a/bench/wtperf/runners/checkpoint-stress.wtperf +++ b/bench/wtperf/runners/checkpoint-stress.wtperf @@ -1,6 +1,6 @@ # A stress configuration to create long running checkpoints while doing a lot # of updates. -conn_config="cache_size=16GB,eviction=(threads_max=4),log=(enabled=false)" +conn_config="cache_size=16GB,eviction=(threads_max=8),log=(enabled=false)" table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" # Enough data to fill the cache. 150 million 1k records results in two ~11GB # tables diff --git a/bench/wtperf/runners/evict-btree-1.wtperf b/bench/wtperf/runners/evict-btree-1.wtperf index 24da4dd7902..741101d083f 100644 --- a/bench/wtperf/runners/evict-btree-1.wtperf +++ b/bench/wtperf/runners/evict-btree-1.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M" +conn_config="cache_size=50M,eviction=(threads_max=1)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-btree-readonly.wtperf b/bench/wtperf/runners/evict-btree-readonly.wtperf index 25599fadd8d..972bc371f2d 100644 --- a/bench/wtperf/runners/evict-btree-readonly.wtperf +++ b/bench/wtperf/runners/evict-btree-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4),mmap=false" +conn_config="cache_size=50M,eviction=(threads_max=8),mmap=false" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-btree-stress-multi.wtperf b/bench/wtperf/runners/evict-btree-stress-multi.wtperf index a5a29f66fa0..5a2cad6d78e 100644 --- a/bench/wtperf/runners/evict-btree-stress-multi.wtperf +++ b/bench/wtperf/runners/evict-btree-stress-multi.wtperf @@ -1,4 +1,4 @@ -conn_config="cache_size=1G,eviction=(threads_max=4),session_max=2000" +conn_config="cache_size=1G,eviction=(threads_max=8),session_max=2000" table_config="type=file" table_count=100 close_conn=false diff --git a/bench/wtperf/runners/evict-btree-stress.wtperf b/bench/wtperf/runners/evict-btree-stress.wtperf index 740fb88c050..96e3f01b325 100644 --- a/bench/wtperf/runners/evict-btree-stress.wtperf +++ b/bench/wtperf/runners/evict-btree-stress.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4)" +conn_config="cache_size=50M,eviction=(threads_max=8)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-btree.wtperf b/bench/wtperf/runners/evict-btree.wtperf index e7d967e5c63..3810e6a8294 100644 --- a/bench/wtperf/runners/evict-btree.wtperf +++ b/bench/wtperf/runners/evict-btree.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4)" +conn_config="cache_size=50M,eviction=(threads_max=8)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-lsm-1.wtperf b/bench/wtperf/runners/evict-lsm-1.wtperf index ad885d98eb7..641a85dc889 100644 --- a/bench/wtperf/runners/evict-lsm-1.wtperf +++ b/bench/wtperf/runners/evict-lsm-1.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +conn_config="cache_size=50M,eviction=(threads_max=1),lsm_manager=(worker_thread_max=6)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/bench/wtperf/runners/evict-lsm-readonly.wtperf b/bench/wtperf/runners/evict-lsm-readonly.wtperf index 661b8e21924..470dca695dd 100644 --- a/bench/wtperf/runners/evict-lsm-readonly.wtperf +++ b/bench/wtperf/runners/evict-lsm-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=8)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/bench/wtperf/runners/evict-lsm.wtperf b/bench/wtperf/runners/evict-lsm.wtperf index b872d429046..a0f2a78d013 100644 --- a/bench/wtperf/runners/evict-lsm.wtperf +++ b/bench/wtperf/runners/evict-lsm.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=8)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/bench/wtperf/runners/log.wtperf b/bench/wtperf/runners/log.wtperf index 6cf50dfb5a5..4379ba22373 100644 --- a/bench/wtperf/runners/log.wtperf +++ b/bench/wtperf/runners/log.wtperf @@ -16,7 +16,7 @@ # - Config + "-C "checkpoint=(wait=0)": no checkpoints # - Config + "-C "log=(enabled,prealloc=false,file_max=1M)": no pre-allocation # -conn_config="cache_size=5G,log=(enabled=true),checkpoint=(log_size=500M),eviction=(threads_max=4)" +conn_config="cache_size=5G,log=(enabled=true),checkpoint=(log_size=500M),eviction=(threads_max=8)" table_config="type=file" icount=1000000 report_interval=5 diff --git a/bench/wtperf/runners/many-table-stress.wtperf b/bench/wtperf/runners/many-table-stress.wtperf new file mode 100644 index 00000000000..6cf1d5d2696 --- /dev/null +++ b/bench/wtperf/runners/many-table-stress.wtperf @@ -0,0 +1,19 @@ +# Create a set of tables with uneven distribution of data +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=20,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +table_config="type=file" +table_count=2000 +icount=0 +random_range=1000000000 +pareto=10 +range_partition=true +report_interval=5 + +run_ops=1000000 +populate_threads=0 +icount=0 +threads=((count=60,inserts=1)) + +# Warn if a latency over 1 second is seen +max_latency=1000 +sample_interval=5 +sample_rate=1 diff --git a/bench/wtperf/runners/mongodb-secondary-apply.wtperf b/bench/wtperf/runners/mongodb-secondary-apply.wtperf index f9e41184f95..58bd1a76b97 100644 --- a/bench/wtperf/runners/mongodb-secondary-apply.wtperf +++ b/bench/wtperf/runners/mongodb-secondary-apply.wtperf @@ -1,5 +1,5 @@ # Simulate the MongoDB oplog apply threads on a secondary. -conn_config="cache_size=10GB,session_max=1000,eviction=(threads_min=4,threads_max=4),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" +conn_config="cache_size=10GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" table_config="allocation_size=4k,memory_page_max=5MB,prefix_compression=false,split_pct=75,leaf_page_max=32k,internal_page_max=16k,type=file" # Spread the workload out over several tables. table_count=4 diff --git a/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf b/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf index d7b27f8fda4..f07e6c80b39 100644 --- a/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf +++ b/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf @@ -2,7 +2,7 @@ # up by dividing the workload across a lot of threads. This needs to be # tuned to the particular machine so the workload is close to capacity in the # steady state, but not overwhelming. -conn_config="cache_size=20GB,session_max=1000,eviction=(threads_min=4,threads_max=4),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" +conn_config="cache_size=20GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file" # Divide original icount by database_count. table_count=8 diff --git a/bench/wtperf/runners/multi-btree-stress.wtperf b/bench/wtperf/runners/multi-btree-stress.wtperf index b10b08f6035..bee1f431043 100644 --- a/bench/wtperf/runners/multi-btree-stress.wtperf +++ b/bench/wtperf/runners/multi-btree-stress.wtperf @@ -1,7 +1,7 @@ # wtperf options file: multi-database configuration attempting to # trigger slow operations by overloading CPU and disk. # References Jira WT-2131 -conn_config="cache_size=2GB,eviction=(threads_min=2,threads_max=2),log=(enabled=false),direct_io=(data,checkpoint),buffer_alignment=4096,checkpoint_sync=true,checkpoint=(wait=60)" +conn_config="cache_size=2GB,eviction=(threads_min=2,threads_max=8),log=(enabled=false),direct_io=(data,checkpoint),buffer_alignment=4096,checkpoint_sync=true,checkpoint=(wait=60)" table_config="allocation_size=4k,prefix_compression=false,split_pct=75,leaf_page_max=4k,internal_page_max=16k,leaf_item_max=1433,internal_item_max=3100,type=file" # Divide original icount by database_count. database_count=5 diff --git a/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf b/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf index ddd9c055eac..1fdba049779 100644 --- a/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf +++ b/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf @@ -1,5 +1,5 @@ # Create a set of tables with uneven distribution of data -conn_config="cache_size=1G,eviction=(threads_max=4),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" table_config="type=file" table_count=100 icount=0 diff --git a/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf b/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf index 380350c88c8..dfb3306a7a5 100644 --- a/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf +++ b/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf @@ -1,5 +1,5 @@ # Read from a set of tables with uneven distribution of data -conn_config="cache_size=1G,eviction=(threads_max=4),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" table_config="type=file" table_count=100 icount=0 diff --git a/bench/wtperf/stress/btree-split-stress.wtperf b/bench/wtperf/stress/btree-split-stress.wtperf index deb8c70d12f..eb6ca1cfddc 100644 --- a/bench/wtperf/stress/btree-split-stress.wtperf +++ b/bench/wtperf/stress/btree-split-stress.wtperf @@ -1,4 +1,4 @@ -conn_config="cache_size=2GB,statistics=[fast,clear],statistics_log=(wait=10),eviction=(threads_max=4,threads_min=4)" +conn_config="cache_size=2GB,statistics=[fast,clear],statistics_log=(wait=10),eviction=(threads_max=8,threads_min=4)" table_config="type=file,leaf_page_max=8k,internal_page_max=8k,memory_page_max=2MB,split_deepen_min_child=250" icount=200000 report_interval=5 @@ -6,5 +6,4 @@ run_time=300 reopen_connection=false populate_threads=2 value_sz=256 -read_range=100 -threads=((count=4,inserts=1,throttle=100000),(count=8,reads=1)) +threads=((count=4,inserts=1,throttle=100000),(count=8,reads=1,read_range=100)) diff --git a/bench/wtperf/track.c b/bench/wtperf/track.c index 822bdaa4b4a..86a26120a6a 100644 --- a/bench/wtperf/track.c +++ b/bench/wtperf/track.c @@ -288,8 +288,8 @@ latency_print_single(WTPERF *wtperf, TRACK *total, const char *name) uint64_t cumops; char path[1024]; - snprintf(path, sizeof(path), - "%s/latency.%s", wtperf->monitor_dir, name); + testutil_check(__wt_snprintf(path, sizeof(path), + "%s/latency.%s", wtperf->monitor_dir, name)); if ((fp = fopen(path, "w")) == NULL) { lprintf(wtperf, errno, 0, "%s", path); return; diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 8c7f0053388..6d79eebe8b2 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -32,9 +32,6 @@ #define DEFAULT_HOME "WT_TEST" #define DEFAULT_MONITOR_DIR "WT_TEST" -static const char * const debug_cconfig = ""; -static const char * const debug_tconfig = ""; - static void *checkpoint_worker(void *); static int drop_all_tables(WTPERF *); static int execute_populate(WTPERF *); @@ -432,19 +429,17 @@ err: wtperf->error = wtperf->stop = true; * search do them. Ensuring the keys we see are always in order. */ static int -do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) +do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor, int64_t read_range) { - CONFIG_OPTS *opts; - size_t range; uint64_t next_val, prev_val; + int64_t range; char *range_key_buf; char buf[512]; int ret; - opts = wtperf->opts; ret = 0; - if (opts->read_range == 0) + if (read_range == 0) return (0); memset(&buf[0], 0, 512 * sizeof(char)); @@ -454,7 +449,7 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) testutil_check(cursor->get_key(cursor, &range_key_buf)); extract_key(range_key_buf, &next_val); - for (range = 0; range < opts->read_range; ++range) { + for (range = 0; range < read_range; ++range) { prev_val = next_val; ret = cursor->next(cursor); /* We are done if we reach the end. */ @@ -475,12 +470,56 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) return (0); } +/* pre_load_data -- + * Pull everything into cache before starting the workload phase. + */ +static int +pre_load_data(WTPERF *wtperf) +{ + CONFIG_OPTS *opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + char *key; + int ret; + size_t i; + + opts = wtperf->opts; + conn = wtperf->conn; + + if ((ret = conn->open_session( + conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); + goto err; + } + for (i = 0; i < opts->table_count; i++) { + if ((ret = session->open_cursor(session, + wtperf->uris[i], NULL, NULL, &cursor)) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: %s", + wtperf->uris[i]); + goto err; + } + while (cursor->next(cursor) == 0) + if ((ret = cursor->get_key(cursor, &key)) != 0) + goto err; + if ((ret = cursor->close(cursor)) != 0) + goto err; + } + if ((ret = session->close(session, NULL)) != 0) + goto err; + if (ret != 0) +err: lprintf(wtperf, ret, 0, "Pre-workload traverse error"); + return (ret); +} + static void * worker(void *arg) { struct timespec start, stop; CONFIG_OPTS *opts; TRACK *trk; + WORKLOAD *workload; WTPERF *wtperf; WTPERF_THREAD *thread; WT_CONNECTION *conn; @@ -495,13 +534,14 @@ worker(void *arg) char buf[512]; thread = (WTPERF_THREAD *)arg; + workload = thread->workload; wtperf = thread->wtperf; opts = wtperf->opts; conn = wtperf->conn; cursors = NULL; - log_table_cursor = NULL; /* -Wconditional-initialized */ + cursor = log_table_cursor = NULL; /* -Wconditional-initialized */ ops = 0; - ops_per_txn = thread->workload->ops_per_txn; + ops_per_txn = workload->ops_per_txn; session = NULL; trk = NULL; @@ -510,9 +550,9 @@ worker(void *arg) lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); goto err; } - cursors = dcalloc(opts->table_count, sizeof(WT_CURSOR *)); for (i = 0; i < opts->table_count_idle; i++) { - snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i); + testutil_check(__wt_snprintf( + buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); if ((ret = session->open_cursor( session, buf, NULL, NULL, &tmp_cursor)) != 0) { lprintf(wtperf, ret, 0, @@ -525,14 +565,34 @@ worker(void *arg) goto err; } } - for (i = 0; i < opts->table_count; i++) { + if (workload->table_index != INT32_MAX) { if ((ret = session->open_cursor(session, - wtperf->uris[i], NULL, NULL, &cursors[i])) != 0) { + wtperf->uris[workload->table_index], + NULL, NULL, &cursor)) != 0) { lprintf(wtperf, ret, 0, "worker: WT_SESSION.open_cursor: %s", - wtperf->uris[i]); + wtperf->uris[workload->table_index]); + goto err; + } + if ((ret = session->open_cursor(session, + wtperf->uris[workload->table_index], + NULL, "next_random=true", &thread->rand_cursor)) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: random %s", + wtperf->uris[workload->table_index]); goto err; } + } else { + cursors = dcalloc(opts->table_count, sizeof(WT_CURSOR *)); + for (i = 0; i < opts->table_count; i++) { + if ((ret = session->open_cursor(session, + wtperf->uris[i], NULL, NULL, &cursors[i])) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: %s", + wtperf->uris[i]); + goto err; + } + } } if (opts->log_like_table && (ret = session->open_cursor(session, wtperf->log_table_uri, NULL, NULL, &log_table_cursor)) != 0) { @@ -543,19 +603,19 @@ worker(void *arg) } /* Setup the timer for throttling. */ - if (thread->workload->throttle != 0) + if (workload->throttle != 0) setup_throttle(thread); /* Setup for truncate */ - if (thread->workload->truncate != 0) + if (workload->truncate != 0) if ((ret = setup_truncate(wtperf, thread, session)) != 0) goto err; key_buf = thread->key_buf; value_buf = thread->value_buf; - op = thread->workload->ops; - op_end = op + sizeof(thread->workload->ops); + op = workload->ops; + op_end = op + sizeof(workload->ops); if ((ops_per_txn != 0 || opts->log_like_table) && (ret = session->begin_transaction(session, NULL)) != 0) { @@ -564,6 +624,8 @@ worker(void *arg) } while (!wtperf->stop) { + if (workload->pause != 0) + (void)sleep((unsigned int)workload->pause); /* * Generate the next key and setup operation specific * statistics tracking objects. @@ -603,10 +665,12 @@ worker(void *arg) generate_key(opts, key_buf, next_val); - /* - * Spread the data out around the multiple databases. - */ - cursor = cursors[map_key_to_table(wtperf->opts, next_val)]; + if (workload->table_index == INT32_MAX) + /* + * Spread the data out around the multiple databases. + */ + cursor = cursors[ + map_key_to_table(wtperf->opts, next_val)]; /* * Skip the first time we do an operation, when trk->ops @@ -642,7 +706,8 @@ worker(void *arg) * for several operations, confirming that the * next key is in the correct order. */ - ret = do_range_reads(wtperf, cursor); + ret = do_range_reads(wtperf, + cursor, workload->read_range); } if (ret == 0 || ret == WT_NOTFOUND) @@ -689,7 +754,7 @@ worker(void *arg) */ strncpy(value_buf, value, opts->value_sz_max - 1); - if (thread->workload->update_delta != 0) + if (workload->update_delta != 0) update_value_delta(thread); if (value_buf[0] == 'a') value_buf[0] = 'b'; @@ -806,7 +871,7 @@ op_err: if (ret == WT_ROLLBACK && ops_per_txn != 0) { /* Schedule the next operation */ if (++op == op_end) - op = thread->workload->ops; + op = workload->ops; /* * Decrement throttle ops and check if we should sleep @@ -843,7 +908,7 @@ run_mix_schedule_op(WORKLOAD *workp, int op, int64_t op_cnt) uint8_t *p, *end; /* Jump around the array to roughly spread out the operations. */ - jump = 100 / op_cnt; + jump = (int)(100 / op_cnt); /* * Find a read operation and replace it with another operation. This @@ -884,17 +949,6 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) opts = wtperf->opts; - /* Confirm reads, inserts, truncates and updates cannot all be zero. */ - if (workp->insert == 0 && workp->read == 0 && - workp->truncate == 0 && workp->update == 0) { - lprintf(wtperf, EINVAL, 0, "no operations scheduled"); - return (EINVAL); - } - - /* - * Handle truncate first - it's a special case that can't be used in - * a mixed workload. - */ if (workp->truncate != 0) { if (workp->insert != 0 || workp->read != 0 || workp->update != 0) { @@ -906,6 +960,12 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) return (0); } + /* Confirm reads, inserts and updates cannot all be zero. */ + if (workp->insert == 0 && workp->read == 0 && workp->update == 0) { + lprintf(wtperf, EINVAL, 0, "no operations scheduled"); + return (EINVAL); + } + /* * Check for a simple case where the thread is only doing insert or * update operations (because the default operation for a @@ -1238,7 +1298,8 @@ monitor(void *arg) /* Open the logging file. */ len = strlen(wtperf->monitor_dir) + 100; path = dmalloc(len); - snprintf(path, len, "%s/monitor", wtperf->monitor_dir); + testutil_check(__wt_snprintf( + path, len, "%s/monitor", wtperf->monitor_dir)); if ((fp = fopen(path, "w")) == NULL) { lprintf(wtperf, errno, 0, "%s", path); goto err; @@ -1593,6 +1654,9 @@ close_reopen(WTPERF *wtperf) opts = wtperf->opts; + if (opts->in_memory) + return (0); + if (!opts->readonly && !opts->reopen_connection) return (0); /* @@ -1875,19 +1939,19 @@ create_uris(WTPERF *wtperf) /* If there is only one table, just use the base name. */ wtperf->uris[i] = dmalloc(len); if (opts->table_count == 1) - snprintf(wtperf->uris[i], - len, "table:%s", opts->table_name); + testutil_check(__wt_snprintf(wtperf->uris[i], + len, "table:%s", opts->table_name)); else - snprintf(wtperf->uris[i], - len, "table:%s%05d", opts->table_name, i); + testutil_check(__wt_snprintf(wtperf->uris[i], + len, "table:%s%05d", opts->table_name, i)); } /* Create the log-like-table URI. */ len = strlen("table:") + strlen(opts->table_name) + strlen("_log_table") + 1; wtperf->log_table_uri = dmalloc(len); - snprintf( - wtperf->log_table_uri, len, "table:%s_log_table", opts->table_name); + testutil_check(__wt_snprintf(wtperf->log_table_uri, + len, "table:%s_log_table", opts->table_name)); } static int @@ -1909,7 +1973,8 @@ create_tables(WTPERF *wtperf) } for (i = 0; i < opts->table_count_idle; i++) { - snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i); + testutil_check(__wt_snprintf( + buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); if ((ret = session->create( session, buf, opts->table_config)) != 0) { lprintf(wtperf, ret, 0, @@ -1938,8 +2003,9 @@ create_tables(WTPERF *wtperf) return (ret); } if (opts->index) { - snprintf(buf, 512, "index:%s:val_idx", - wtperf->uris[i] + strlen("table:")); + testutil_check(__wt_snprintf(buf, 512, + "index:%s:val_idx", + wtperf->uris[i] + strlen("table:"))); if ((ret = session->create( session, buf, "columns=(val)")) != 0) { lprintf(wtperf, ret, 0, @@ -2124,15 +2190,15 @@ start_all_runs(WTPERF *wtperf) */ len = strlen(wtperf->home) + 5; next_wtperf->home = dmalloc(len); - snprintf( - next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i); + testutil_check(__wt_snprintf( + next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i)); if (opts->create != 0) recreate_dir(next_wtperf->home); len = strlen(wtperf->monitor_dir) + 5; next_wtperf->monitor_dir = dmalloc(len); - snprintf(next_wtperf->monitor_dir, - len, "%s/D%02d", wtperf->monitor_dir, (int)i); + testutil_check(__wt_snprintf(next_wtperf->monitor_dir, + len, "%s/D%02d", wtperf->monitor_dir, (int)i)); if (opts->create != 0 && strcmp(next_wtperf->home, next_wtperf->monitor_dir) != 0) recreate_dir(next_wtperf->monitor_dir); @@ -2244,6 +2310,8 @@ start_run(WTPERF *wtperf) opts->checkpoint_threads, checkpoint_worker) != 0) goto err; } + if (opts->pre_load_data && (ret = pre_load_data(wtperf)) != 0) + goto err; /* Execute the workload. */ if ((ret = execute_workload(wtperf)) != 0) goto err; @@ -2361,11 +2429,11 @@ main(int argc, char *argv[]) { CONFIG_OPTS *opts; WTPERF *wtperf, _wtperf; - size_t req_len, sreq_len; + size_t pos, req_len, sreq_len; bool monitor_set; int ch, ret; const char *cmdflags = "C:h:m:O:o:T:"; - const char *config_opts; + const char *append_comma, *config_opts; char *cc_buf, *path, *sess_cfg, *tc_buf, *user_cconfig, *user_tconfig; /* The first WTPERF structure (from which all others are derived). */ @@ -2479,9 +2547,9 @@ main(int argc, char *argv[]) */ req_len = strlen(",async=(enabled=true,threads=)") + 4; wtperf->async_config = dmalloc(req_len); - snprintf(wtperf->async_config, req_len, + testutil_check(__wt_snprintf(wtperf->async_config, req_len, ",async=(enabled=true,threads=%" PRIu32 ")", - opts->async_threads); + opts->async_threads)); } if ((ret = config_compress(wtperf)) != 0) goto err; @@ -2502,53 +2570,96 @@ main(int argc, char *argv[]) __wt_stream_set_line_buffer(stdout); /* Concatenate non-default configuration strings. */ - if (opts->verbose > 1 || user_cconfig != NULL || - opts->session_count_idle > 0 || wtperf->compress_ext != NULL || - wtperf->async_config != NULL) { - req_len = strlen(debug_cconfig) + 20; - if (user_cconfig != NULL) - req_len += strlen(user_cconfig); - if (wtperf->async_config != NULL) - req_len += strlen(wtperf->async_config); - if (wtperf->compress_ext != NULL) - req_len += strlen(wtperf->compress_ext); + if (user_cconfig != NULL || opts->session_count_idle > 0 || + wtperf->compress_ext != NULL || wtperf->async_config != NULL || + opts->in_memory) { + req_len = 20; + req_len += wtperf->async_config != NULL ? + strlen(wtperf->async_config) : 0; + req_len += wtperf->compress_ext != NULL ? + strlen(wtperf->compress_ext) : 0; if (opts->session_count_idle > 0) { - sreq_len = strlen(",session_max=") + 6; + sreq_len = strlen("session_max=") + 6; req_len += sreq_len; sess_cfg = dmalloc(sreq_len); - snprintf(sess_cfg, sreq_len, - ",session_max=%" PRIu32, + testutil_check(__wt_snprintf(sess_cfg, sreq_len, + "session_max=%" PRIu32, opts->session_count_idle + - wtperf->workers_cnt + opts->populate_threads + 10); + wtperf->workers_cnt + opts->populate_threads + 10)); } + req_len += opts->in_memory ? strlen("in_memory=true") : 0; + req_len += user_cconfig != NULL ? strlen(user_cconfig) : 0; cc_buf = dmalloc(req_len); - snprintf(cc_buf, req_len, "%s,%s,%s,%s,%s", - wtperf->async_config ? wtperf->async_config : "", - wtperf->compress_ext ? wtperf->compress_ext : "", - opts->verbose > 1 ? debug_cconfig : "", - sess_cfg != NULL ? sess_cfg : "", - user_cconfig != NULL ? user_cconfig : ""); - if (strlen(cc_buf) && (ret = + + pos = 0; + append_comma = ""; + if (wtperf->async_config != NULL && + strlen(wtperf->async_config) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->async_config)); + append_comma = ","; + } + if (wtperf->compress_ext != NULL && + strlen(wtperf->compress_ext) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->compress_ext)); + append_comma = ","; + } + if (opts->in_memory) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, "in_memory=true")); + append_comma = ","; + } + if (sess_cfg != NULL && strlen(sess_cfg) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, sess_cfg)); + append_comma = ","; + } + if (user_cconfig != NULL && strlen(user_cconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, user_cconfig)); + } + + if (strlen(cc_buf) != 0 && (ret = config_opt_name_value(wtperf, "conn_config", cc_buf)) != 0) goto err; } - if (opts->verbose > 1 || opts->index || + if (opts->index || user_tconfig != NULL || wtperf->compress_table != NULL) { - req_len = strlen(debug_tconfig) + 20; - if (user_tconfig != NULL) - req_len += strlen(user_tconfig); - if (wtperf->compress_table != NULL) - req_len += strlen(wtperf->compress_table); - if (opts->index) - req_len += strlen(INDEX_COL_NAMES); + req_len = 20; + req_len += wtperf->compress_table != NULL ? + strlen(wtperf->compress_table) : 0; + req_len += opts->index ? strlen(INDEX_COL_NAMES) : 0; + req_len += user_tconfig != NULL ? strlen(user_tconfig) : 0; tc_buf = dmalloc(req_len); - snprintf(tc_buf, req_len, "%s,%s,%s,%s", - opts->index ? INDEX_COL_NAMES : "", - wtperf->compress_table != NULL ? - wtperf->compress_table : "", - opts->verbose > 1 ? debug_tconfig : "", - user_tconfig ? user_tconfig : ""); - if (strlen(tc_buf) && (ret = + + pos = 0; + append_comma = ""; + if (wtperf->compress_table != NULL && + strlen(wtperf->compress_table) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->compress_table)); + append_comma = ","; + } + if (opts->index) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, INDEX_COL_NAMES)); + append_comma = ","; + } + if (user_tconfig != NULL && strlen(user_tconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, user_tconfig)); + } + + if (strlen(tc_buf) != 0 && (ret = config_opt_name_value(wtperf, "table_config", tc_buf)) != 0) goto err; } @@ -2556,8 +2667,9 @@ main(int argc, char *argv[]) req_len = strlen(opts->table_config) + strlen(LOG_PARTIAL_CONFIG) + 1; wtperf->partial_config = dmalloc(req_len); - snprintf(wtperf->partial_config, req_len, "%s%s", - opts->table_config, LOG_PARTIAL_CONFIG); + testutil_check(__wt_snprintf( + wtperf->partial_config, req_len, "%s%s", + opts->table_config, LOG_PARTIAL_CONFIG)); } /* * Set the config for reopen. If readonly add in that string. @@ -2570,11 +2682,12 @@ main(int argc, char *argv[]) req_len = strlen(opts->conn_config) + 1; wtperf->reopen_config = dmalloc(req_len); if (opts->readonly) - snprintf(wtperf->reopen_config, req_len, "%s%s", - opts->conn_config, READONLY_CONFIG); + testutil_check(__wt_snprintf( + wtperf->reopen_config, req_len, "%s%s", + opts->conn_config, READONLY_CONFIG)); else - snprintf(wtperf->reopen_config, - req_len, "%s", opts->conn_config); + testutil_check(__wt_snprintf( + wtperf->reopen_config, req_len, "%s", opts->conn_config)); /* Sanity-check the configuration. */ if ((ret = config_sanity(wtperf)) != 0) @@ -2587,7 +2700,8 @@ main(int argc, char *argv[]) /* Write a copy of the config. */ req_len = strlen(wtperf->home) + strlen("/CONFIG.wtperf") + 1; path = dmalloc(req_len); - snprintf(path, req_len, "%s/CONFIG.wtperf", wtperf->home); + testutil_check(__wt_snprintf( + path, req_len, "%s/CONFIG.wtperf", wtperf->home)); config_opt_log(opts, path); free(path); @@ -2712,7 +2826,8 @@ recreate_dir(const char *name) len = strlen(name) * 2 + 100; buf = dmalloc(len); - (void)snprintf(buf, len, "rm -rf %s && mkdir %s", name, name); + testutil_check(__wt_snprintf( + buf, len, "rm -rf %s && mkdir %s", name, name)); testutil_checkfmt(system(buf), "system: %s", buf); free(buf); } @@ -2779,14 +2894,43 @@ static uint64_t wtperf_rand(WTPERF_THREAD *thread) { CONFIG_OPTS *opts; + WT_CURSOR *rnd_cursor; WTPERF *wtperf; double S1, S2, U; uint64_t rval; + int ret; + char *key_buf; wtperf = thread->wtperf; opts = wtperf->opts; /* + * If we have a random cursor set up then use it. + */ + if ((rnd_cursor = thread->rand_cursor) != NULL) { + if ((ret = rnd_cursor->next(rnd_cursor)) != 0) { + lprintf(wtperf, ret, 0, "worker: rand next failed"); + /* 0 is outside the expected range. */ + return (0); + } + if ((ret = rnd_cursor->get_key(rnd_cursor, &key_buf)) != 0) { + lprintf(wtperf, ret, 0, + "worker: rand next key retrieval"); + return (0); + } + /* + * Resetting the cursor is not fatal. We still return the + * value we retrieved above. We do it so that we don't + * leave a cursor positioned. + */ + if ((ret = rnd_cursor->reset(rnd_cursor)) != 0) + lprintf(wtperf, ret, 0, + "worker: rand cursor reset failed"); + extract_key(key_buf, &rval); + return (rval); + } + + /* * Use WiredTiger's random number routine: it's lock-free and fairly * good. */ diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 81d74e134f6..3efb8ab700e 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -66,6 +66,9 @@ typedef struct { uint64_t throttle; /* Maximum operations/second */ /* Number of operations per transaction. Zero for autocommit */ int64_t ops_per_txn; + int64_t pause; /* Time between scans */ + int64_t read_range; /* Range of reads */ + int32_t table_index; /* Table to focus ops on */ int64_t truncate; /* Truncate ratio */ uint64_t truncate_pct; /* Truncate Percent */ uint64_t truncate_count; /* Truncate Count */ @@ -225,6 +228,7 @@ typedef struct { struct __wtperf_thread { /* Per-thread structure */ WTPERF *wtperf; /* Enclosing configuration */ + WT_CURSOR *rand_cursor; /* Random key cursor */ WT_RAND_STATE rnd; /* Random number generation state */ diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index 680eb53a90e..90f70457407 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -110,6 +110,8 @@ DEF_OPT_AS_UINT32(database_count, 1, DEF_OPT_AS_BOOL(drop_tables, 0, "Whether to drop all tables at the end of the run, and report time taken" " to do the drop.") +DEF_OPT_AS_BOOL(in_memory, 0, + "Whether to create the database in-memory.") DEF_OPT_AS_UINT32(icount, 5000, "number of records to initially populate. If multiple tables are " "configured the count is spread evenly across all tables.") @@ -145,12 +147,13 @@ DEF_OPT_AS_UINT32(populate_ops_per_txn, 0, "phase, zero for auto-commit") DEF_OPT_AS_UINT32(populate_threads, 1, "number of populate threads, 1 for bulk load") +DEF_OPT_AS_BOOL(pre_load_data, 0, + "Scan all data prior to starting the workload phase to warm the cache") DEF_OPT_AS_UINT32(random_range, 0, "if non zero choose a value from within this range as the key for " "insert operations") DEF_OPT_AS_BOOL(random_value, 0, "generate random content for the value") DEF_OPT_AS_BOOL(range_partition, 0, "partition data by range (vs hash)") -DEF_OPT_AS_UINT32(read_range, 0, "scan a range of keys after each search") DEF_OPT_AS_BOOL(readonly, 0, "reopen the connection between populate and workload phases in readonly " "mode. Requires reopen_connection turned on (default). Requires that " @@ -192,9 +195,10 @@ DEF_OPT_AS_STRING(threads, "", "workload configuration: each 'count' " "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' " "which would create 2 threads doing nothing but reads and 8 threads " "each doing 50% inserts and 25% reads and updates. Allowed configuration " - "values are 'count', 'throttle', 'update_delta', 'reads', 'inserts', " - "'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are " - "also behavior modifiers, supported modifiers are 'ops_per_txn'") + "values are 'count', 'throttle', 'update_delta', 'reads', 'read_range', " + "'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. " + "There are also behavior modifiers, supported modifiers are " + "'ops_per_txn'") DEF_OPT_AS_CONFIG_STRING(transaction_config, "", "WT_SESSION.begin_transaction configuration string, applied during the " "populate phase when populate_ops_per_txn is nonzero") |