Merge branch 'develop' into mongodb-3.4mongodb-3.3.6

8 files changed, 140 insertions, 131 deletions

diff --git a/src/block/block_ckpt.c b/src/block/block_ckpt.c
index a861a21876b..716121faa06 100644
--- a/src/block/block_ckpt.c
+++ b/src/block/block_ckpt.c
@@ -140,12 +140,10 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block,
 	 * will unnecessarily allocate buffer space.
 	 */
 	if (!checkpoint && !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) {
-		/*
-		 * The truncate might fail if there's a file mapping (if there's
-		 * an open checkpoint on the file), that's OK.
-		 */
 		WT_ERR(__wt_verbose(session, WT_VERB_CHECKPOINT,
 		    "truncate file to %" PRIuMAX, (uintmax_t)ci->file_size));
+
+		/* The truncate might fail, and that's OK. */
 		WT_ERR_BUSY_OK(
 		    __wt_block_truncate(session, block, ci->file_size));
 	}
@@ -190,10 +188,7 @@ __wt_block_checkpoint_unload(
 	 * checkpoints.
 	 */
 	if (!checkpoint) {
-		/*
-		 * The truncate might fail if there's a file mapping (if there's
-		 * an open checkpoint on the file), that's OK.
-		 */
+		/* The truncate might fail, and that's OK. */
 		WT_TRET_BUSY_OK(
 		    __wt_block_truncate(session, block, block->size));
 
diff --git a/src/block/block_ext.c b/src/block/block_ext.c
index caafcc77c48..6d67a66be5f 100644
--- a/src/block/block_ext.c
+++ b/src/block/block_ext.c
@@ -1362,9 +1362,8 @@ __wt_block_extlist_truncate(
 	block->size = size;
 
 	/*
-	 * Truncate the file. The truncate might fail if there's a file mapping
-	 * (if there's an open checkpoint on the file), that's OK, we'll ignore
-	 * those blocks.
+	 * Truncate the file. The truncate might fail, and that's OK, we simply
+	 * ignore those blocks.
 	 */
 	WT_RET(__wt_verbose(session, WT_VERB_BLOCK,
 	    "truncate file from %" PRIdMAX " to %" PRIdMAX,
diff --git a/src/block/block_map.c b/src/block/block_map.c
index b16fe7f8423..ce6fe8602f5 100644
--- a/src/block/block_map.c
+++ b/src/block/block_map.c
@@ -13,24 +13,16 @@
  *	Map a segment of the file in, if possible.
  */
 int
-__wt_block_map(
-    WT_SESSION_IMPL *session, WT_BLOCK *block, void *mapp, size_t *maplenp,
-    void **mappingcookie)
+__wt_block_map(WT_SESSION_IMPL *session, WT_BLOCK *block,
+    void *mapped_regionp, size_t *lengthp, void *mapped_cookiep)
 {
 	WT_DECL_RET;
+	WT_FILE_HANDLE *handle;
 
-	*(void **)mapp = NULL;
-	*maplenp = 0;
+	*(void **)mapped_regionp = NULL;
+	*lengthp = 0;
+	*(void **)mapped_cookiep = NULL;
 
-#ifdef WORDS_BIGENDIAN
-	/*
-	 * The underlying objects are little-endian, mapping objects isn't
-	 * currently supported on big-endian systems.
-	 */
-	WT_UNUSED(session);
-	WT_UNUSED(block);
-	WT_UNUSED(mappingcookie);
-#else
 	/* Map support is configurable. */
 	if (!S2C(session)->mmap)
 		return (0);
@@ -51,15 +43,23 @@ __wt_block_map(
 		return (0);
 
 	/*
+	 * There may be no underlying functionality.
+	 */
+	handle = block->fh->handle;
+	if (handle->map == NULL)
+		return (0);
+
+	/*
 	 * Map the file into memory.
 	 * Ignore not-supported errors, we'll read the file through the cache
 	 * if map fails.
 	 */
-	ret = block->fh->fh_map(
-	    session, block->fh, mapp, maplenp, mappingcookie);
-	if (ret == ENOTSUP)
+	ret = handle->map(handle,
+	    (WT_SESSION *)session, mapped_regionp, lengthp, mapped_cookiep);
+	if (ret == ENOTSUP) {
+		*(void **)mapped_regionp = NULL;
 		ret = 0;
-#endif
+	}
 
 	return (ret);
 }
@@ -69,11 +69,13 @@ __wt_block_map(
  *	Unmap any mapped-in segment of the file.
  */
 int
-__wt_block_unmap(
-    WT_SESSION_IMPL *session, WT_BLOCK *block, void *map, size_t maplen,
-    void **mappingcookie)
+__wt_block_unmap(WT_SESSION_IMPL *session,
+    WT_BLOCK *block, void *mapped_region, size_t length, void *mapped_cookie)
 {
+	WT_FILE_HANDLE *handle;
+
 	/* Unmap the file from memory. */
-	return (block->fh->fh_map_unmap(
-	    session, block->fh, map, maplen, mappingcookie));
+	handle = block->fh->handle;
+	return (handle->unmap(handle,
+	    (WT_SESSION *)session, mapped_region, length, mapped_cookie));
 }
diff --git a/src/block/block_mgr.c b/src/block/block_mgr.c
index 06150a0f062..465952d8ca5 100644
--- a/src/block/block_mgr.c
+++ b/src/block/block_mgr.c
@@ -103,7 +103,7 @@ __bm_checkpoint_load(WT_BM *bm, WT_SESSION_IMPL *session,
 		 * of being read into cache buffers.
 		 */
 		WT_RET(__wt_block_map(session,
-		    bm->block, &bm->map, &bm->maplen, &bm->mappingcookie));
+		    bm->block, &bm->map, &bm->maplen, &bm->mapped_cookie));
 
 		/*
 		 * If this handle is for a checkpoint, that is, read-only, there
@@ -149,7 +149,7 @@ __bm_checkpoint_unload(WT_BM *bm, WT_SESSION_IMPL *session)
 	/* Unmap any mapped segment. */
 	if (bm->map != NULL)
 		WT_TRET(__wt_block_unmap(session,
-		    bm->block, bm->map, bm->maplen, &bm->mappingcookie));
+		    bm->block, bm->map, bm->maplen, &bm->mapped_cookie));
 
 	/* Unload the checkpoint. */
 	WT_TRET(__wt_block_checkpoint_unload(session, bm->block, !bm->is_live));
@@ -302,6 +302,20 @@ __bm_is_mapped(WT_BM *bm, WT_SESSION_IMPL *session)
 }
 
 /*
+ * __bm_map_discard --
+ *	Discard a mapped segment.
+ */
+static int
+__bm_map_discard(WT_BM *bm, WT_SESSION_IMPL *session, void *map, size_t len)
+{
+	WT_FILE_HANDLE *handle;
+
+	handle = bm->block->fh->handle;
+	return (handle->map_discard(
+	    handle, (WT_SESSION *)session, map, len, bm->mapped_cookie));
+}
+
+/*
  * __bm_salvage_end --
  *	End a block manager salvage.
  */
@@ -413,19 +427,7 @@ __bm_stat(WT_BM *bm, WT_SESSION_IMPL *session, WT_DSRC_STATS *stats)
 static int
 __bm_sync(WT_BM *bm, WT_SESSION_IMPL *session, bool block)
 {
-	WT_DECL_RET;
-
-	if (!block && !bm->block->nowait_sync_available)
-		return (0);
-
-	if ((ret = __wt_fsync(session, bm->block->fh, block)) == 0)
-		return (0);
-
-	/* Ignore ENOTSUP, but don't try again. */
-	if (ret != ENOTSUP)
-		return (ret);
-	bm->block->nowait_sync_available = false;
-	return (0);
+	return (__wt_fsync(session, bm->block->fh, block));
 }
 
 /*
@@ -544,6 +546,7 @@ __bm_method_set(WT_BM *bm, bool readonly)
 	bm->compact_start = __bm_compact_start;
 	bm->free = __bm_free;
 	bm->is_mapped = __bm_is_mapped;
+	bm->map_discard = __bm_map_discard;
 	bm->preload = __wt_bm_preload;
 	bm->read = __wt_bm_read;
 	bm->salvage_end = __bm_salvage_end;
diff --git a/src/block/block_open.c b/src/block/block_open.c
index f4da5ca7c05..e58bef30a6d 100644
--- a/src/block/block_open.c
+++ b/src/block/block_open.c
@@ -33,7 +33,6 @@ __wt_block_manager_create(
 	WT_FH *fh;
 	int suffix;
 	bool exists;
-	char *path;
 
 	/*
 	 * Create the underlying file and open a handle.
@@ -44,7 +43,7 @@ __wt_block_manager_create(
 	 * in our space. Move any existing files out of the way and complain.
 	 */
 	for (;;) {
-		if ((ret = __wt_open(session, filename, WT_FILE_TYPE_DATA,
+		if ((ret = __wt_open(session, filename, WT_OPEN_FILE_TYPE_DATA,
 		    WT_OPEN_CREATE | WT_OPEN_EXCLUSIVE, &fh)) == 0)
 			break;
 		WT_ERR_TEST(ret != EEXIST, ret);
@@ -54,10 +53,10 @@ __wt_block_manager_create(
 		for (suffix = 1;; ++suffix) {
 			WT_ERR(__wt_buf_fmt(
 			    session, tmp, "%s.%d", filename, suffix));
-			WT_ERR(__wt_exist(session, tmp->data, &exists));
+			WT_ERR(__wt_fs_exist(session, tmp->data, &exists));
 			if (!exists) {
-				WT_ERR(
-				    __wt_rename(session, filename, tmp->data));
+				WT_ERR(__wt_fs_rename(
+				    session, filename, tmp->data));
 				WT_ERR(__wt_msg(session,
 				    "unexpected file %s found, renamed to %s",
 				    filename, (char *)tmp->data));
@@ -82,14 +81,12 @@ __wt_block_manager_create(
 	 * Some filesystems require that we sync the directory to be confident
 	 * that the file will appear.
 	 */
-	if (ret == 0 && (ret = __wt_filename(session, filename, &path)) == 0) {
-		ret = __wt_directory_sync(session, path);
-		__wt_free(session, path);
-	}
+	if (ret == 0)
+		WT_TRET(__wt_fs_directory_sync(session, filename));
 
 	/* Undo any create on error. */
 	if (ret != 0)
-		WT_TRET(__wt_remove(session, filename));
+		WT_TRET(__wt_fs_remove(session, filename));
 
 err:	__wt_scr_free(session, &tmp);
 
@@ -156,8 +153,7 @@ __wt_block_open(WT_SESSION_IMPL *session,
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	uint64_t bucket, hash;
-
-	WT_UNUSED(readonly);
+	uint32_t flags;
 
 	WT_RET(__wt_verbose(session, WT_VERB_BLOCK, "open: %s", filename));
 
@@ -204,12 +200,18 @@ __wt_block_open(WT_SESSION_IMPL *session,
 	/* Set the file extension information. */
 	block->extend_len = conn->data_extend_len;
 
-	/* Set the asynchronous flush, preload availability. */
-	block->nowait_sync_available = true;
-	block->preload_available = true;
-
-	/* Open the underlying file handle. */
-	WT_ERR(__wt_open(session, filename, WT_FILE_TYPE_DATA, 0, &block->fh));
+	/*
+	 * Open the underlying file handle.
+	 *
+	 * "direct_io=checkpoint" configures direct I/O for readonly data files.
+	 */
+	flags = 0;
+	if (readonly && FLD_ISSET(conn->direct_io, WT_DIRECT_IO_CHECKPOINT))
+		LF_SET(WT_OPEN_DIRECTIO);
+	if (!readonly && FLD_ISSET(conn->direct_io, WT_DIRECT_IO_DATA))
+		LF_SET(WT_OPEN_DIRECTIO);
+	WT_ERR(__wt_open(
+	    session, filename, WT_OPEN_FILE_TYPE_DATA, flags, &block->fh));
 
 	/* Set the file's size. */
 	WT_ERR(__wt_filesize(session, block->fh, &block->size));
@@ -422,5 +424,5 @@ int
 __wt_block_manager_named_size(
     WT_SESSION_IMPL *session, const char *name, wt_off_t *sizep)
 {
-	return (__wt_filesize_name(session, name, false, sizep));
+	return (__wt_fs_size(session, name, sizep));
 }
diff --git a/src/block/block_read.c b/src/block/block_read.c
index 6f0c41c1b5c..7304f6ff4bc 100644
--- a/src/block/block_read.c
+++ b/src/block/block_read.c
@@ -19,44 +19,32 @@ __wt_bm_preload(
 	WT_BLOCK *block;
 	WT_DECL_ITEM(tmp);
 	WT_DECL_RET;
+	WT_FILE_HANDLE *handle;
 	wt_off_t offset;
 	uint32_t cksum, size;
 	bool mapped;
 
 	WT_UNUSED(addr_size);
+
 	block = bm->block;
 
 	WT_STAT_FAST_CONN_INCR(session, block_preload);
 
-	/* Preload the block. */
-	if (block->preload_available) {
-		/* Crack the cookie. */
-		WT_RET(__wt_block_buffer_to_addr(
-		    block, addr, &offset, &size, &cksum));
-
-		mapped = bm->map != NULL &&
-		    offset + size <= (wt_off_t)bm->maplen;
-		if (mapped)
-			ret = block->fh->fh_map_preload(session,
-			    block->fh, (uint8_t *)bm->map + offset, size);
-		else
-			ret = block->fh->fh_advise(session,
-			    block->fh, (wt_off_t)offset,
-			    (wt_off_t)size, POSIX_FADV_WILLNEED);
-		if (ret == 0)
-			return (0);
-
-		/* Ignore ENOTSUP, but don't try again. */
-		if (ret != ENOTSUP)
-			return (ret);
-		block->preload_available = false;
-	}
+	/* Crack the cookie. */
+	WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &cksum));
 
-	/*
-	 * If preload isn't supported, do it the slow way; don't call the
-	 * underlying read routine directly, we don't know for certain if
-	 * this is a mapped range.
-	 */
+	handle = block->fh->handle;
+	mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen;
+	if (mapped && handle->map_preload != NULL)
+		ret = handle->map_preload(handle, (WT_SESSION *)session,
+		    (uint8_t *)bm->map + offset, size, bm->mapped_cookie);
+	if (!mapped && handle->fadvise != NULL)
+		ret = handle->fadvise(handle, (WT_SESSION *)session,
+		    (wt_off_t)offset, (wt_off_t)size, WT_FILE_HANDLE_WILLNEED);
+	if (ret != EBUSY && ret != ENOTSUP)
+		return (ret);
+
+	/* If preload isn't supported, do it the slow way. */
 	WT_RET(__wt_scr_alloc(session, 0, &tmp));
 	ret = __wt_bm_read(bm, session, tmp, addr, addr_size);
 	__wt_scr_free(session, &tmp);
@@ -74,6 +62,7 @@ __wt_bm_read(WT_BM *bm, WT_SESSION_IMPL *session,
 {
 	WT_BLOCK *block;
 	WT_DECL_RET;
+	WT_FILE_HANDLE *handle;
 	wt_off_t offset;
 	uint32_t cksum, size;
 	bool mapped;
@@ -87,23 +76,17 @@ __wt_bm_read(WT_BM *bm, WT_SESSION_IMPL *session,
 	/*
 	 * Map the block if it's possible.
 	 */
+	handle = block->fh->handle;
 	mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen;
-	if (mapped) {
+	if (mapped && handle->map_preload != NULL) {
 		buf->data = (uint8_t *)bm->map + offset;
 		buf->size = size;
-		if (block->preload_available) {
-			ret = block->fh->fh_map_preload(
-			    session, block->fh, buf->data, buf->size);
-
-			/* Ignore ENOTSUP, but don't try again. */
-			if (ret != ENOTSUP)
-				return (ret);
-			block->preload_available = false;
-		}
+		ret = handle->map_preload(handle, (WT_SESSION *)session,
+		    buf->data, buf->size,bm->mapped_cookie);
 
 		WT_STAT_FAST_CONN_INCR(session, block_map_read);
 		WT_STAT_FAST_CONN_INCRV(session, block_byte_map_read, size);
-		return (0);
+		return (ret);
 	}
 
 #ifdef HAVE_DIAGNOSTIC
diff --git a/src/block/block_vrfy.c b/src/block/block_vrfy.c
index 6570184ca10..a8e59ad0af7 100644
--- a/src/block/block_vrfy.c
+++ b/src/block/block_vrfy.c
@@ -15,7 +15,7 @@ static int __verify_filefrag_add(
 	WT_SESSION_IMPL *, WT_BLOCK *, const char *, wt_off_t, wt_off_t, bool);
 static int __verify_filefrag_chk(WT_SESSION_IMPL *, WT_BLOCK *);
 static int __verify_last_avail(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *);
-static int __verify_last_truncate(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *);
+static int __verify_set_file_size(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *);
 
 /* The bit list ignores the first block: convert to/from a frag/offset. */
 #define	WT_wt_off_TO_FRAG(block, off)					\
@@ -49,8 +49,8 @@ __wt_block_verify_start(WT_SESSION_IMPL *session,
 			return (0);
 	}
 
-	/* Truncate the file to the size of the last checkpoint. */
-	WT_RET(__verify_last_truncate(session, block, ckpt));
+	/* Set the size of the file to the size of the last checkpoint. */
+	WT_RET(__verify_set_file_size(session, block, ckpt));
 
 	/*
 	 * We're done if the file has no data pages (this happens if we verify
@@ -144,11 +144,11 @@ err:	__wt_block_ckpt_destroy(session, ci);
 }
 
 /*
- * __verify_last_truncate --
- *	Truncate the file to the last checkpoint's size.
+ * __verify_set_file_size --
+ *	Set the file size to the last checkpoint's size.
  */
 static int
-__verify_last_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt)
+__verify_set_file_size(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt)
 {
 	WT_BLOCK_CKPT *ci, _ci;
 	WT_DECL_RET;
@@ -156,7 +156,13 @@ __verify_last_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt)
 	ci = &_ci;
 	WT_RET(__wt_block_ckpt_init(session, ci, ckpt->name));
 	WT_ERR(__wt_block_buffer_to_ckpt(session, block, ckpt->raw.data, ci));
-	WT_ERR_BUSY_OK(__wt_block_truncate(session, block, ci->file_size));
+
+	/*
+	 * Verify is read-only. Set the block's file size information as if we
+	 * truncated the file during checkpoint load, so references to blocks
+	 * after last checkpoint's file size fail.
+	 */
+	block->size = block->extend_size = ci->file_size;
 
 err:	__wt_block_ckpt_destroy(session, ci);
 	return (ret);
diff --git a/src/block/block_write.c b/src/block/block_write.c
index 134272b52f9..4f1224f3c13 100644
--- a/src/block/block_write.c
+++ b/src/block/block_write.c
@@ -15,6 +15,24 @@
 int
 __wt_block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t len)
 {
+	/*
+	 * Backups are done by copying files outside of WiredTiger, potentially
+	 * by system utilities. We cannot truncate the file during the backup
+	 * window, we might surprise an application.
+	 *
+	 * Stop block truncation. This affects files that aren't involved in the
+	 * backup (for example, doing incremental backups, which only copies log
+	 * files, or targeted backups, stops all truncation). We may want a more
+	 * targeted solution at some point.
+	 */
+	if (S2C(session)->hot_backup)
+		return (EBUSY);
+
+	/*
+	 * Additionally, the truncate might fail if there's a file mapping (if
+	 * there's an open checkpoint on the file), in which case the underlying
+	 * function returns EBUSY.
+	 */
 	WT_RET(__wt_ftruncate(session, block->fh, len));
 
 	block->size = block->extend_size = len;
@@ -30,27 +48,28 @@ int
 __wt_block_discard(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t added_size)
 {
 	WT_DECL_RET;
+	WT_FILE_HANDLE *handle;
 
+	/* The file may not support this call. */
+	handle = block->fh->handle;
+	if (handle->fadvise == NULL)
+		return (0);
+
+	/* The call may not be configured. */
 	if (block->os_cache_max == 0)
 		return (0);
 
 	/*
 	 * We're racing on the addition, but I'm not willing to serialize on it
-	 * in the standard read path with more evidence it's needed.
+	 * in the standard read path without evidence it's needed.
 	 */
 	if ((block->os_cache += added_size) <= block->os_cache_max)
 		return (0);
 
 	block->os_cache = 0;
-	WT_ERR(block->fh->fh_advise(session,
-	    block->fh, (wt_off_t)0, (wt_off_t)0, POSIX_FADV_DONTNEED));
-	return (0);
-
-err:	/* Ignore ENOTSUP, but don't try again. */
-	if (ret != ENOTSUP)
-		return (ret);
-	block->os_cache_max = 0;
-	return (0);
+	ret = handle->fadvise(handle, (WT_SESSION *)session,
+	    (wt_off_t)0, (wt_off_t)0, WT_FILE_HANDLE_DONTNEED);
+	return (ret == EBUSY || ret == ENOTSUP ? 0 : ret);
 }
 
 /*
@@ -62,6 +81,7 @@ __wt_block_extend(WT_SESSION_IMPL *session, WT_BLOCK *block,
     WT_FH *fh, wt_off_t offset, size_t align_size, bool *release_lockp)
 {
 	WT_DECL_RET;
+	WT_FILE_HANDLE *handle;
 	bool locked;
 
 	/*
@@ -107,7 +127,8 @@ __wt_block_extend(WT_SESSION_IMPL *session, WT_BLOCK *block,
 	 * based on the filesystem type, fall back to ftruncate in that case,
 	 * and remember that ftruncate requires locking.
 	 */
-	if (fh->fallocate_available != WT_FALLOCATE_NOT_AVAILABLE) {
+	handle = fh->handle;
+	if (handle->fallocate != NULL || handle->fallocate_nolock != NULL) {
 		/*
 		 * Release any locally acquired lock if not needed to extend the
 		 * file, extending the file may require updating on-disk file's
@@ -115,7 +136,7 @@ __wt_block_extend(WT_SESSION_IMPL *session, WT_BLOCK *block,
 		 * configure for file extension on systems that require locking
 		 * over the extend call.)
 		 */
-		if (!fh->fallocate_requires_locking && *release_lockp) {
+		if (handle->fallocate_nolock != NULL && *release_lockp) {
 			*release_lockp = locked = false;
 			__wt_spin_unlock(session, &block->live_lock);
 		}
@@ -131,8 +152,7 @@ __wt_block_extend(WT_SESSION_IMPL *session, WT_BLOCK *block,
 		if ((ret = __wt_fallocate(
 		    session, fh, block->size, block->extend_len * 2)) == 0)
 			return (0);
-		if (ret != ENOTSUP)
-			return (ret);
+		WT_RET_ERROR_OK(ret, ENOTSUP);
 	}
 
 	/*
@@ -155,9 +175,8 @@ __wt_block_extend(WT_SESSION_IMPL *session, WT_BLOCK *block,
 	 * The truncate might fail if there's a mapped file (in other words, if
 	 * there's an open checkpoint on the file), that's OK.
 	 */
-	if ((ret = __wt_ftruncate(session, fh, block->extend_size)) == EBUSY)
-		ret = 0;
-	return (ret);
+	WT_RET_BUSY_OK(__wt_ftruncate(session, fh, block->extend_size));
+	return (0);
 }
 
 /*