1 files changed, 94 insertions, 30 deletions

diff --git a/src/mongo/executor/network_interface_integration_test.cpp b/src/mongo/executor/network_interface_integration_test.cpp
index fdf1ac6f8ba..1722084ac3b 100644
--- a/src/mongo/executor/network_interface_integration_test.cpp
+++ b/src/mongo/executor/network_interface_integration_test.cpp
@@ -50,6 +50,7 @@
 #include "mongo/unittest/integration_test.h"
 #include "mongo/unittest/unittest.h"
 #include "mongo/util/assert_util.h"
+#include "mongo/util/fail_point.h"
 #include "mongo/util/scopeguard.h"
 
 namespace mongo {
@@ -160,7 +161,7 @@ public:
     }
 
     void setUp() override {
-        startNet(std::make_unique<WaitForIsMasterHook>(this));
+        startNet(std::make_unique<WaitForHelloHook>(this));
     }
 
     // NetworkInterfaceIntegrationFixture::tearDown() shuts down the NetworkInterface. We always
@@ -253,33 +254,33 @@ public:
         return ++numCurrentOpRan;
     }
 
-    struct IsMasterData {
+    struct HelloData {
         BSONObj request;
         RemoteCommandResponse response;
     };
-    IsMasterData waitForIsMaster() {
+    HelloData waitForHello() {
         stdx::unique_lock<Latch> lk(_mutex);
-        _isMasterCond.wait(lk, [this] { return _isMasterResult != boost::none; });
+        _helloCondVar.wait(lk, [this] { return _helloResult != boost::none; });
 
-        return std::move(*_isMasterResult);
+        return std::move(*_helloResult);
     }
 
-    bool hasIsMaster() {
+    bool hasHelloResult() {
         stdx::lock_guard<Latch> lk(_mutex);
-        return _isMasterResult != boost::none;
+        return _helloResult != boost::none;
     }
 
 private:
-    class WaitForIsMasterHook : public NetworkConnectionHook {
+    class WaitForHelloHook : public NetworkConnectionHook {
     public:
-        explicit WaitForIsMasterHook(NetworkInterfaceTest* parent) : _parent(parent) {}
+        explicit WaitForHelloHook(NetworkInterfaceTest* parent) : _parent(parent) {}
 
         Status validateHost(const HostAndPort& host,
                             const BSONObj& request,
-                            const RemoteCommandResponse& isMasterReply) override {
+                            const RemoteCommandResponse& helloReply) override {
             stdx::lock_guard<Latch> lk(_parent->_mutex);
-            _parent->_isMasterResult = IsMasterData{request, isMasterReply};
-            _parent->_isMasterCond.notify_all();
+            _parent->_helloResult = HelloData{request, helloReply};
+            _parent->_helloCondVar.notify_all();
             return Status::OK();
         }
 
@@ -296,8 +297,8 @@ private:
     };
 
     Mutex _mutex = MONGO_MAKE_LATCH("NetworkInterfaceTest::_mutex");
-    stdx::condition_variable _isMasterCond;
-    boost::optional<IsMasterData> _isMasterResult;
+    stdx::condition_variable _helloCondVar;
+    boost::optional<HelloData> _helloResult;
 };
 
 class NetworkInterfaceInternalClientTest : public NetworkInterfaceTest {
@@ -328,7 +329,7 @@ TEST_F(NetworkInterfaceTest, CancelLocally) {
 
         auto deferred = runCommand(cbh, makeTestCommand(kMaxWait, makeEchoCmdObj()));
 
-        waitForIsMaster();
+        waitForHello();
 
         fpb->waitForTimesEntered(fpb.initialTimesEntered() + 1);
 
@@ -503,13 +504,35 @@ TEST_F(NetworkInterfaceTest, LateCancel) {
     assertNumOps(0u, 0u, 0u, 1u);
 }
 
+TEST_F(NetworkInterfaceTest, ConnectionErrorDropsSingleConnection) {
+    FailPoint* failPoint =
+        globalFailPointRegistry().find("transportLayerASIOasyncConnectReturnsConnectionError");
+    auto timesEntered = failPoint->setMode(FailPoint::nTimes, 1);
+
+    auto cbh = makeCallbackHandle();
+    auto deferred = runCommand(cbh, makeTestCommand(kMaxWait, makeEchoCmdObj()));
+    // Wait for one of the connection attempts to fail with a `ConnectionError`.
+    failPoint->waitForTimesEntered(timesEntered + 1);
+    auto result = deferred.get();
+
+    ASSERT_OK(result.status);
+    ConnectionPoolStats stats;
+    net().appendConnectionStats(&stats);
+
+    ASSERT_EQ(stats.totalCreated, 2);
+    ASSERT_EQ(stats.totalInUse + stats.totalAvailable + stats.totalRefreshing, 1);
+    // Connection dropped during finishRefresh, so the dropped connection still
+    // counts toward the refreshed counter.
+    ASSERT_EQ(stats.totalRefreshed, 2);
+}
+
 TEST_F(NetworkInterfaceTest, AsyncOpTimeout) {
     // Kick off operation
     auto cb = makeCallbackHandle();
     auto request = makeTestCommand(Milliseconds{1000}, makeSleepCmdObj());
     auto deferred = runCommand(cb, request);
 
-    waitForIsMaster();
+    waitForHello();
 
     auto result = deferred.get();
 
@@ -533,13 +556,19 @@ TEST_F(NetworkInterfaceTest, AsyncOpTimeoutWithOpCtxDeadlineSooner) {
     serviceContext->registerClientObserver(std::make_unique<LockerNoopClientObserver>());
     auto client = serviceContext->makeClient("NetworkClient");
     auto opCtx = client->makeOperationContext();
-    opCtx->setDeadlineAfterNowBy(opCtxDeadline, ErrorCodes::ExceededTimeLimit);
+
+    auto stopWatch = serviceContext->getPreciseClockSource()->makeStopWatch();
+    opCtx->setDeadlineByDate(stopWatch.start() + opCtxDeadline, ErrorCodes::ExceededTimeLimit);
 
     auto request = makeTestCommand(requestTimeout, makeSleepCmdObj(), opCtx.get());
 
     auto deferred = runCommand(cb, request);
+    // The time returned in result.elapsed is measured from when the command started, which happens
+    // in runCommand. The delay between setting the deadline on opCtx and starting the command can
+    // be long enough that the assertion about opCtxDeadline fails.
+    auto networkStartCommandDelay = stopWatch.elapsed();
 
-    waitForIsMaster();
+    waitForHello();
 
     auto result = deferred.get();
 
@@ -551,9 +580,10 @@ TEST_F(NetworkInterfaceTest, AsyncOpTimeoutWithOpCtxDeadlineSooner) {
 
     ASSERT_EQ(ErrorCodes::ExceededTimeLimit, result.status);
     ASSERT(result.elapsed);
+
     // check that the request timeout uses the smaller of the operation context deadline and
     // the timeout specified in the request constructor.
-    ASSERT_GTE(result.elapsed.value(), opCtxDeadline);
+    ASSERT_GTE(result.elapsed.value() + networkStartCommandDelay, opCtxDeadline);
     ASSERT_LT(result.elapsed.value(), requestTimeout);
     assertNumOps(0u, 1u, 0u, 0u);
 }
@@ -569,12 +599,19 @@ TEST_F(NetworkInterfaceTest, AsyncOpTimeoutWithOpCtxDeadlineLater) {
     serviceContext->registerClientObserver(std::make_unique<LockerNoopClientObserver>());
     auto client = serviceContext->makeClient("NetworkClient");
     auto opCtx = client->makeOperationContext();
-    opCtx->setDeadlineAfterNowBy(opCtxDeadline, ErrorCodes::ExceededTimeLimit);
+
+    auto stopWatch = serviceContext->getPreciseClockSource()->makeStopWatch();
+    opCtx->setDeadlineByDate(stopWatch.start() + opCtxDeadline, ErrorCodes::ExceededTimeLimit);
+
     auto request = makeTestCommand(requestTimeout, makeSleepCmdObj(), opCtx.get());
 
     auto deferred = runCommand(cb, request);
+    // The time returned in result.elapsed is measured from when the command started, which happens
+    // in runCommand. The delay between setting the deadline on opCtx and starting the command can
+    // be long enough that the assertion about opCtxDeadline fails.
+    auto networkStartCommandDelay = stopWatch.elapsed();
 
-    waitForIsMaster();
+    waitForHello();
 
     auto result = deferred.get();
 
@@ -586,10 +623,12 @@ TEST_F(NetworkInterfaceTest, AsyncOpTimeoutWithOpCtxDeadlineLater) {
 
     ASSERT_EQ(ErrorCodes::NetworkInterfaceExceededTimeLimit, result.status);
     ASSERT(result.elapsed);
+
     // check that the request timeout uses the smaller of the operation context deadline and
     // the timeout specified in the request constructor.
     ASSERT_GTE(duration_cast<Milliseconds>(result.elapsed.value()), requestTimeout);
-    ASSERT_LT(duration_cast<Milliseconds>(result.elapsed.value()), opCtxDeadline);
+    ASSERT_LT(duration_cast<Milliseconds>(result.elapsed.value() + networkStartCommandDelay),
+              opCtxDeadline);
 
     assertNumOps(0u, 1u, 0u, 0u);
 }
@@ -733,13 +772,13 @@ TEST_F(NetworkInterfaceTest, SetAlarm) {
 }
 
 TEST_F(NetworkInterfaceInternalClientTest,
-       IsMasterRequestContainsOutgoingWireVersionInternalClientInfo) {
+       HelloRequestContainsOutgoingWireVersionInternalClientInfo) {
     auto deferred = runCommand(makeCallbackHandle(), makeTestCommand(kNoTimeout, makeEchoCmdObj()));
-    auto isMasterHandshake = waitForIsMaster();
+    auto helloHandshake = waitForHello();
 
-    // Verify that the isMaster reply has the expected internalClient data.
+    // Verify that the "hello" reply has the expected internalClient data.
     auto wireSpec = WireSpec::instance().get();
-    auto internalClientElem = isMasterHandshake.request["internalClient"];
+    auto internalClientElem = helloHandshake.request["internalClient"];
     ASSERT_EQ(internalClientElem.type(), BSONType::Object);
     auto minWireVersionElem = internalClientElem.Obj()["minWireVersion"];
     auto maxWireVersionElem = internalClientElem.Obj()["maxWireVersion"];
@@ -754,14 +793,14 @@ TEST_F(NetworkInterfaceInternalClientTest,
     assertNumOps(0u, 0u, 0u, 1u);
 }
 
-TEST_F(NetworkInterfaceTest, IsMasterRequestMissingInternalClientInfoWhenNotInternalClient) {
+TEST_F(NetworkInterfaceTest, HelloRequestMissingInternalClientInfoWhenNotInternalClient) {
     resetIsInternalClient(false);
 
     auto deferred = runCommand(makeCallbackHandle(), makeTestCommand(kNoTimeout, makeEchoCmdObj()));
-    auto isMasterHandshake = waitForIsMaster();
+    auto helloHandshake = waitForHello();
 
-    // Verify that the isMaster reply has the expected internalClient data.
-    ASSERT_FALSE(isMasterHandshake.request["internalClient"]);
+    // Verify that the "hello" reply has the expected internalClient data.
+    ASSERT_FALSE(helloHandshake.request["internalClient"]);
     // Verify that the ping op is counted as a success.
     auto res = deferred.get();
     ASSERT(res.elapsed);
@@ -939,6 +978,31 @@ TEST_F(NetworkInterfaceTest, TearDownWaitsForInProgress) {
     ASSERT_EQ(getInProgress(), 0);
 }
 
+TEST_F(NetworkInterfaceTest, RunCommandOnLeasedStream) {
+    auto cs = fixture();
+    auto target = cs.getServers().front();
+    auto leasedStream = net().leaseStream(target, transport::kGlobalSSLMode, kNoTimeout).get();
+    auto* client = leasedStream->getClient();
+
+    auto request = RemoteCommandRequest(target, "admin", makeEchoCmdObj(), nullptr, kNoTimeout);
+    auto deferred = client->runCommandRequest(request);
+
+    auto res = deferred.get();
+
+    ASSERT(res.elapsed);
+    uassertStatusOK(res.status);
+    leasedStream->indicateSuccess();
+    leasedStream->indicateUsed();
+
+    // This opmsg request expect the following reply, which is generated below
+    // { echo: { echo: 1, foo: "bar", $db: "admin" }, ok: 1.0 }
+    auto cmdObj = res.data.getObjectField("echo");
+    ASSERT_EQ(1, cmdObj.getIntField("echo"));
+    ASSERT_EQ("bar"_sd, cmdObj.getStringField("foo"));
+    ASSERT_EQ("admin"_sd, cmdObj.getStringField("$db"));
+    ASSERT_EQ(1, res.data.getIntField("ok"));
+}
+
 }  // namespace
 }  // namespace executor
 }  // namespace mongo