cpp: Fix for task completion notification deadlock under heavy server load

Along with insuring that the event loop is able to see task completion
notifications, we need to process pending notifications as soon as they
are available to avoid filling the notification pipe (the pipe can hold
512 notifications).

git-svn-id: https://svn.apache.org/repos/asf/incubator/thrift/trunk@920668 13f79535-47bb-0310-9956-ffa450edef68

diff --git a/lib/cpp/src/server/TNonblockingServer.cpp b/lib/cpp/src/server/TNonblockingServer.cpp
index 26e97c9..eb071a9 100644 (file)
--- a/lib/cpp/src/server/TNonblockingServer.cpp
+++ b/lib/cpp/src/server/TNonblockingServer.cpp
@@ -570,12 +570,12 @@ void TNonblockingServer::handleEvent(int fd, short which) {
       nTotalConnectionsDropped_++;
       if (overloadAction_ == T_OVERLOAD_CLOSE_ON_ACCEPT) {
         close(clientSocket);
-        continue;
+        return;
       } else if (overloadAction_ == T_OVERLOAD_DRAIN_TASK_QUEUE) {
         if (!drainPendingTask()) {
           // Nothing left to discard, so we drop connection instead.
           close(clientSocket);
-          continue;
+          return;
         }
       }
     }
@@ -718,6 +718,13 @@ void TNonblockingServer::createNotificationPipe() {
     GlobalOutput.perror("TNonblockingServer::createNotificationPipe ", errno);
       throw TException("can't create notification pipe");
   }
+  int flags;
+  if ((flags = fcntl(notificationPipeFDs_[0], F_GETFL, 0)) < 0 ||
+      fcntl(notificationPipeFDs_[0], F_SETFL, flags | O_NONBLOCK) < 0) {
+    close(notificationPipeFDs_[0]);
+    close(notificationPipeFDs_[1]);
+    throw TException("TNonblockingServer::createNotificationPipe() O_NONBLOCK");
+  }
 }
 
 /**

diff --git a/lib/cpp/src/server/TNonblockingServer.h b/lib/cpp/src/server/TNonblockingServer.h
index 2650dd1..b547649 100644 (file)
--- a/lib/cpp/src/server/TNonblockingServer.h
+++ b/lib/cpp/src/server/TNonblockingServer.h
@@ -722,13 +722,17 @@ class TConnection {
    */
   static void taskHandler(int fd, short /* which */, void* /* v */) {
     TConnection* connection;
-    if (read(fd, (void*)&connection, sizeof(TConnection*))
-        != sizeof(TConnection*)) {
+    ssize_t nBytes;
+    while ((nBytes = read(fd, (void*)&connection, sizeof(TConnection*)))
+        == sizeof(TConnection*)) {
+      connection->transition();
+    }
+    if (nBytes > 0) {
+      throw TException("TConnection::taskHandler unexpected partial read");
+    }
+    if (errno != EWOULDBLOCK && errno != EAGAIN) {
       GlobalOutput.perror("TConnection::taskHandler read failed, resource leak", errno);
-      return;
     }
-
-    connection->transition();
   }
 
   /**