From: David Reiss <dreiss@apache.org>
Date: Tue, 9 Mar 2010 05:19:34 +0000 (+0000)
Subject: cpp: Fix for task completion notification deadlock under heavy server load
X-Git-Tag: 0.3.0~91
X-Git-Url: https://source.supwisdom.com/gerrit/gitweb?a=commitdiff_plain;h=83b8fdac0ff52f1f1d9f96772b956c807f668593;p=common%2Fthrift.git

cpp: Fix for task completion notification deadlock under heavy server load

Along with insuring that the event loop is able to see task completion
notifications, we need to process pending notifications as soon as they
are available to avoid filling the notification pipe (the pipe can hold
512 notifications).

git-svn-id: https://svn.apache.org/repos/asf/incubator/thrift/trunk@920668 13f79535-47bb-0310-9956-ffa450edef68
---

diff --git a/lib/cpp/src/server/TNonblockingServer.cpp b/lib/cpp/src/server/TNonblockingServer.cpp
index 26e97c96..eb071a90 100644
--- a/lib/cpp/src/server/TNonblockingServer.cpp
+++ b/lib/cpp/src/server/TNonblockingServer.cpp
@@ -570,12 +570,12 @@ void TNonblockingServer::handleEvent(int fd, short which) {
       nTotalConnectionsDropped_++;
       if (overloadAction_ == T_OVERLOAD_CLOSE_ON_ACCEPT) {
         close(clientSocket);
-        continue;
+        return;
       } else if (overloadAction_ == T_OVERLOAD_DRAIN_TASK_QUEUE) {
         if (!drainPendingTask()) {
           // Nothing left to discard, so we drop connection instead.
           close(clientSocket);
-          continue;
+          return;
         }
       }
     }
@@ -718,6 +718,13 @@ void TNonblockingServer::createNotificationPipe() {
     GlobalOutput.perror("TNonblockingServer::createNotificationPipe ", errno);
       throw TException("can't create notification pipe");
   }
+  int flags;
+  if ((flags = fcntl(notificationPipeFDs_[0], F_GETFL, 0)) < 0 ||
+      fcntl(notificationPipeFDs_[0], F_SETFL, flags | O_NONBLOCK) < 0) {
+    close(notificationPipeFDs_[0]);
+    close(notificationPipeFDs_[1]);
+    throw TException("TNonblockingServer::createNotificationPipe() O_NONBLOCK");
+  }
 }
 
 /**
diff --git a/lib/cpp/src/server/TNonblockingServer.h b/lib/cpp/src/server/TNonblockingServer.h
index 2650dd11..b5476496 100644
--- a/lib/cpp/src/server/TNonblockingServer.h
+++ b/lib/cpp/src/server/TNonblockingServer.h
@@ -722,13 +722,17 @@ class TConnection {
    */
   static void taskHandler(int fd, short /* which */, void* /* v */) {
     TConnection* connection;
-    if (read(fd, (void*)&connection, sizeof(TConnection*))
-        != sizeof(TConnection*)) {
+    ssize_t nBytes;
+    while ((nBytes = read(fd, (void*)&connection, sizeof(TConnection*)))
+        == sizeof(TConnection*)) {
+      connection->transition();
+    }
+    if (nBytes > 0) {
+      throw TException("TConnection::taskHandler unexpected partial read");
+    }
+    if (errno != EWOULDBLOCK && errno != EAGAIN) {
       GlobalOutput.perror("TConnection::taskHandler read failed, resource leak", errno);
-      return;
     }
-
-    connection->transition();
   }
 
   /**