Import tools: 4f093ae71cdb4c6a6e9de7cd1dc67ea4405f0013 from branch v3.4r3.4.7-rc0r3.4.7

ref: 29b8883c56..4f093ae71c
for: 3.4.7

TOOLS-1109    failes to build on arm64 (syscall.Dup2 not supported)
TOOLS-1542    dump and export shouldn't count views before running
TOOLS-1563    windows tests are failing after `use mongodb 3.4 "current" tests`
TOOLS-1577    update the readme with information about mongoreplay
TOOLS-1713    Move mongoreplay evergreen config .evergreen.yml into common.yml
TOOLS-1741    mongoimport --uri throws errors when passed Atlas Connection String URI
TOOLS-1743    legacy24 and legacy26 dumprestore tests failing on master

7 files changed, 155 insertions, 31 deletions

diff --git a/src/mongo/gotools/common/db/namespaces.go b/src/mongo/gotools/common/db/namespaces.go
index 908687b1c56..8530b16aa5e 100644
--- a/src/mongo/gotools/common/db/namespaces.go
+++ b/src/mongo/gotools/common/db/namespaces.go
@@ -2,13 +2,23 @@ package db
 
 import (
 	"fmt"
-	"github.com/mongodb/mongo-tools/common/bsonutil"
+	"strings"
+
 	"github.com/mongodb/mongo-tools/common/log"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
-	"strings"
 )
 
+type CollectionInfo struct {
+	Name    string  `bson:"name"`
+	Type    string  `bson:"type"`
+	Options *bson.D `bson:"options"`
+}
+
+func (ci *CollectionInfo) IsView() bool {
+	return ci.Type == "view"
+}
+
 // IsNoCmd reeturns true if err indicates a query command is not supported,
 // otherwise, returns false.
 func IsNoCmd(err error) bool {
@@ -124,7 +134,7 @@ func getCollectionsPre28(database *mgo.Database, name string) (*mgo.Iter, error)
 	return iter, nil
 }
 
-func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) {
+func GetCollectionInfo(coll *mgo.Collection) (*CollectionInfo, error) {
 	iter, useFullName, err := GetCollections(coll.Database, coll.Name)
 	if err != nil {
 		return nil, err
@@ -134,33 +144,32 @@ func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) {
 	if useFullName {
 		comparisonName = coll.FullName
 	}
-	collInfo := &bson.D{}
+
+	collInfo := &CollectionInfo{}
 	for iter.Next(collInfo) {
-		name, err := bsonutil.FindValueByKey("name", collInfo)
-		if err != nil {
-			collInfo = nil
-			continue
-		}
-		if nameStr, ok := name.(string); ok {
-			if nameStr == comparisonName {
-				// we've found the collection we're looking for
-				break
+		if collInfo.Name == comparisonName {
+			if useFullName {
+				collName, err := StripDBFromNamespace(collInfo.Name, coll.Database.Name)
+				if err != nil {
+					return nil, err
+				}
+				collInfo.Name = collName
 			}
-		} else {
-			collInfo = nil
-			continue
+			break
 		}
 	}
+	if err := iter.Err(); err != nil {
+		return nil, err
+	}
+	return collInfo, nil
+}
 
-	if collInfo != nil {
-		optsInterface, _ := bsonutil.FindValueByKey("options", collInfo)
-		if optsInterface != nil {
-			optsD, ok := optsInterface.(bson.D)
-			if !ok {
-				return nil, fmt.Errorf("Cannot unmarshal collection options for collection %v.%v", coll.Database, coll.Name)
-			}
-			return &optsD, nil
-		}
+func StripDBFromNamespace(namespace string, dbName string) (string, error) {
+	namespacePrefix := dbName + "."
+	// if the collection info came from querying system.indexes (2.6 or earlier) then the
+	// "name" we get includes the db name as well, so we must remove it
+	if strings.HasPrefix(namespace, namespacePrefix) {
+		return namespace[len(namespacePrefix):], nil
 	}
-	return nil, iter.Err()
+	return "", fmt.Errorf("namespace '%v' format is invalid - expected to start with '%v'", namespace, namespacePrefix)
 }
diff --git a/src/mongo/gotools/common/db/namespaces_test.go b/src/mongo/gotools/common/db/namespaces_test.go
new file mode 100644
index 00000000000..0e390a2f43c
--- /dev/null
+++ b/src/mongo/gotools/common/db/namespaces_test.go
@@ -0,0 +1,52 @@
+package db
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+type stripDBFromNamespaceTestCase struct {
+	inputNamespace string
+	inputDBName    string
+
+	outputNamespace string
+	outputError     error
+}
+
+func TestStripDBFromNamespace(t *testing.T) {
+	Convey("When testing StripDBFromNamespace with cases", t, func() {
+		testCases := []stripDBFromNamespaceTestCase{
+			{
+				inputNamespace: "database.col",
+				inputDBName:    "database",
+
+				outputNamespace: "col",
+				outputError:     nil,
+			},
+			{
+				inputNamespace: "database2.col",
+				inputDBName:    "database",
+
+				outputNamespace: "",
+				outputError:     fmt.Errorf("namespace 'database2.col' format is invalid - expected to start with 'database.'"),
+			},
+			{
+				inputNamespace: "database.col",
+				inputDBName:    "notAPrefix",
+
+				outputNamespace: "",
+				outputError:     fmt.Errorf("namespace 'database.col' format is invalid - expected to start with 'notAPrefix.'"),
+			},
+		}
+		Convey("cases should match expected", func() {
+			for _, tc := range testCases {
+				resultNamespace, resultError := StripDBFromNamespace(tc.inputNamespace, tc.inputDBName)
+				So(resultError, ShouldResemble, tc.outputError)
+				So(resultNamespace, ShouldEqual, tc.outputNamespace)
+			}
+		})
+	})
+
+}
diff --git a/src/mongo/gotools/common/db/write_concern.go b/src/mongo/gotools/common/db/write_concern.go
index cf7a8e98d56..8759fb59ad1 100644
--- a/src/mongo/gotools/common/db/write_concern.go
+++ b/src/mongo/gotools/common/db/write_concern.go
@@ -127,11 +127,17 @@ func BuildWriteConcern(writeConcern string, nodeType NodeType, cs *connstring.Co
 	}
 
 	if cs != nil {
+		if cs.W == "" {
+			cs.W = "majority"
+		}
 		sessionSafety, err = constructSafetyFromConnString(cs)
 		if err != nil {
 			return nil, err
 		}
-	} else if writeConcern != "" {
+	} else {
+		if writeConcern == "" {
+			writeConcern = "majority"
+		}
 		sessionSafety, err = constructWCObject(writeConcern)
 		if err != nil {
 			return nil, err
diff --git a/src/mongo/gotools/common/db/write_concern_test.go b/src/mongo/gotools/common/db/write_concern_test.go
index bdc6ef69018..31a5baf1bfb 100644
--- a/src/mongo/gotools/common/db/write_concern_test.go
+++ b/src/mongo/gotools/common/db/write_concern_test.go
@@ -54,6 +54,13 @@ func TestBuildWriteConcern(t *testing.T) {
 				So(err, ShouldBeNil)
 				So(writeConcern.J, ShouldBeTrue)
 			})
+			// Regression test for TOOLS-1741
+			Convey("When passing an empty writeConcern and empty URI"+
+				"then write concern should default to being majority", func() {
+				writeConcern, err := BuildWriteConcern("", ReplSet, nil)
+				So(err, ShouldBeNil)
+				So(writeConcern.WMode, ShouldEqual, "majority")
+			})
 		})
 		Convey("and given a connection string", func() {
 			Convey("with a w value of 0, without j set, a nil write concern should be returned", func() {
diff --git a/src/mongo/gotools/common/intents/intent.go b/src/mongo/gotools/common/intents/intent.go
index 8f317a3716f..db7eb78a772 100644
--- a/src/mongo/gotools/common/intents/intent.go
+++ b/src/mongo/gotools/common/intents/intent.go
@@ -285,6 +285,7 @@ func (manager *Manager) putNormalIntentWithNamespace(ns string, intent *Intent)
 // Put inserts an intent into the manager with the same source namespace as
 // its destinations.
 func (manager *Manager) Put(intent *Intent) {
+	log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
 	manager.PutWithNamespace(intent.Namespace(), intent)
 }
 
diff --git a/src/mongo/gotools/common/intents/intent_prioritizer.go b/src/mongo/gotools/common/intents/intent_prioritizer.go
index 290a7c83d1e..bea15bb1ab9 100644
--- a/src/mongo/gotools/common/intents/intent_prioritizer.go
+++ b/src/mongo/gotools/common/intents/intent_prioritizer.go
@@ -59,8 +59,8 @@ func (legacy *legacyPrioritizer) Finish(*Intent) {
 //===== Longest Task First =====
 
 // longestTaskFirstPrioritizer returns intents in the order of largest -> smallest,
-// which is better at minimizing total runtime in parallel environments than
-// other simple orderings.
+// with views at the front of the list, which is better at minimizing total
+// runtime in parallel environments than other simple orderings.
 type longestTaskFirstPrioritizer struct {
 	sync.Mutex
 	queue []*Intent
@@ -68,7 +68,7 @@ type longestTaskFirstPrioritizer struct {
 
 // NewLongestTaskFirstPrioritizer returns an initialized LTP prioritizer
 func NewLongestTaskFirstPrioritizer(intents []*Intent) *longestTaskFirstPrioritizer {
-	sort.Sort(BySize(intents))
+	sort.Sort(BySizeAndView(intents))
 	return &longestTaskFirstPrioritizer{
 		queue: intents,
 	}
@@ -92,6 +92,22 @@ func (ltf *longestTaskFirstPrioritizer) Finish(*Intent) {
 	return
 }
 
+// BySizeAndView attaches the methods for sort.Interface for sorting intents
+// from largest to smallest size, taking into account if it's a view or not.
+type BySizeAndView []*Intent
+
+func (s BySizeAndView) Len() int      { return len(s) }
+func (s BySizeAndView) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s BySizeAndView) Less(i, j int) bool {
+	if s[i].IsView() && !s[j].IsView() {
+		return true
+	}
+	if !s[i].IsView() && s[j].IsView() {
+		return false
+	}
+	return s[i].Size > s[j].Size
+}
+
 // For sorting intents from largest to smallest size
 type BySize []*Intent
 
diff --git a/src/mongo/gotools/common/intents/intent_prioritizer_test.go b/src/mongo/gotools/common/intents/intent_prioritizer_test.go
index 2abd79e5641..2e153d3f9ea 100644
--- a/src/mongo/gotools/common/intents/intent_prioritizer_test.go
+++ b/src/mongo/gotools/common/intents/intent_prioritizer_test.go
@@ -2,9 +2,11 @@ package intents
 
 import (
 	"container/heap"
+	"testing"
+
 	"github.com/mongodb/mongo-tools/common/testutil"
 	. "github.com/smartystreets/goconvey/convey"
-	"testing"
+	"gopkg.in/mgo.v2/bson"
 )
 
 func TestLegacyPrioritizer(t *testing.T) {
@@ -109,6 +111,37 @@ func TestDBCounterCollectionSorting(t *testing.T) {
 	})
 }
 
+func TestBySizeAndView(t *testing.T) {
+	var prioritizer IntentPrioritizer
+
+	testutil.VerifyTestType(t, testutil.UnitTestType)
+
+	Convey("With a prioritizer initialized with on a set of intents", t, func() {
+		intents := []*Intent{
+			&Intent{C: "non-view2", Size: 32},
+			&Intent{C: "view", Size: 0,
+				Options: &bson.D{{"viewOn", true}},
+			},
+			&Intent{C: "non-view1", Size: 1024},
+			&Intent{C: "non-view3", Size: 2},
+			&Intent{C: "view", Size: 0,
+				Options: &bson.D{{"viewOn", true}},
+			},
+		}
+		prioritizer = NewLongestTaskFirstPrioritizer(intents)
+		Convey("getting the sorted intents should produce views first, followed by largest to smallest", func() {
+
+			So(prioritizer.Get().C, ShouldEqual, "view")
+			So(prioritizer.Get().C, ShouldEqual, "view")
+			So(prioritizer.Get().C, ShouldEqual, "non-view1")
+			So(prioritizer.Get().C, ShouldEqual, "non-view2")
+			So(prioritizer.Get().C, ShouldEqual, "non-view3")
+		})
+
+	})
+
+}
+
 func TestSimulatedMultiDBJob(t *testing.T) {
 	var prioritizer IntentPrioritizer