From bf072d68ec1bee52be84f127bc71733a11e421b6 Mon Sep 17 00:00:00 2001
From: Steve Atherton <stevea@apple.com>
Date: Thu, 4 Jun 2020 01:32:12 -0700
Subject: [PATCH] Workarounds for strange behaviors in Boost ssl sockets on
 MacOS and Linux.  When writing to the ssl socket, write_some() would
 sometimes return BrokenPipe instead of WouldBlock unless onWriteable on the
 raw socket was checked first.  On MacOS, even with the onWriteable check
 using a send size greater than 2016 (determined experimentally) would still
 result in the error.  Also consolidated two identical copies of
 SendBufferIterator.

---
 fdbclient/HTTP.actor.cpp |  8 ++--
 flow/Net2.actor.cpp      | 91 +++++++++++++++-------------------------
 2 files changed, 36 insertions(+), 63 deletions(-)

diff --git a/fdbclient/HTTP.actor.cpp b/fdbclient/HTTP.actor.cpp
index 933dd15fff..3779b26ef5 100644
--- a/fdbclient/HTTP.actor.cpp
+++ b/fdbclient/HTTP.actor.cpp
@@ -352,6 +352,9 @@ namespace HTTP {
 			send_start = timer();
 
 			loop {
+				wait(conn->onWritable());
+				wait( delay( 0, TaskPriority::WriteSocket ) );
+
 				// If we already got a response, before finishing sending the request, then close the connection,
 				// set the Connection header to "close" as a hint to the caller that this connection can't be used
 				// again, and break out of the send loop.
@@ -372,11 +375,6 @@ namespace HTTP {
 				pContent->sent(len);
 				if(pContent->empty())
 					break;
-
-				if(len == 0) {
-					wait(conn->onWritable());
-					wait( delay( 0, TaskPriority::WriteSocket ) );
-				}
 			}
 
 			wait(responseReading);
diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index e5078d46dd..c77078dfcb 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -291,6 +291,35 @@ public:
 	}
 };
 
+struct SendBufferIterator {
+	typedef boost::asio::const_buffer value_type;
+	typedef std::forward_iterator_tag iterator_category;
+	typedef size_t difference_type;
+	typedef boost::asio::const_buffer* pointer;
+	typedef boost::asio::const_buffer& reference;
+
+	SendBuffer const* p;
+	int limit;
+
+	SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits<int>::max()) : p(p), limit(limit) {
+		ASSERT(limit > 0);
+	}
+
+	bool operator == (SendBufferIterator const& r) const { return p == r.p; }
+	bool operator != (SendBufferIterator const& r) const { return p != r.p; }
+	void operator++() {
+		limit -= p->bytes_written - p->bytes_sent;
+		if(limit > 0)
+			p = p->next;
+		else
+			p = NULL;
+	}
+
+	boost::asio::const_buffer operator*() const {
+		return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) );
+	}
+};
+
 class Connection : public IConnection, ReferenceCounted<Connection> {
 public:
 	virtual void addref() { ReferenceCounted<Connection>::addref(); }
@@ -415,35 +444,6 @@ private:
 	tcp::socket socket;
 	NetworkAddress peer_address;
 
-	struct SendBufferIterator {
-		typedef boost::asio::const_buffer value_type;
-		typedef std::forward_iterator_tag iterator_category;
-		typedef size_t difference_type;
-		typedef boost::asio::const_buffer* pointer;
-		typedef boost::asio::const_buffer& reference;
-
-		SendBuffer const* p;
-		int limit;
-
-		SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits<int>::max()) : p(p), limit(limit) {
-			ASSERT(limit > 0);
-		}
-
-		bool operator == (SendBufferIterator const& r) const { return p == r.p; }
-		bool operator != (SendBufferIterator const& r) const { return p != r.p; }
-		void operator++() {
-			limit -= p->bytes_written - p->bytes_sent;
-			if(limit > 0)
-				p = p->next;
-			else
-				p = NULL;
-		}
-
-		boost::asio::const_buffer operator*() const {
-			return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) );
-		}
-	};
-
 	void init() {
 		// Socket settings that have to be set after connect or accept succeeds
 		socket.non_blocking(true);
@@ -707,6 +707,10 @@ public:
 
 	// Writes as many bytes as possible from the given SendBuffer chain into the write buffer and returns the number of bytes written (might be 0)
 	virtual int write( SendBuffer const* data, int limit ) {
+#ifdef __APPLE__
+		// For some reason, writing ssl_sock with more than 2016 bytes when socket is writeable sometimes results in a broken pipe error.
+		limit = std::min(limit, 2016);
+#endif
 		boost::system::error_code err;
 		++g_net2->countWrites;
 
@@ -749,35 +753,6 @@ private:
 	NetworkAddress peer_address;
 	Reference<ReferencedObject<boost::asio::ssl::context>> sslContext;
 
-	struct SendBufferIterator {
-		typedef boost::asio::const_buffer value_type;
-		typedef std::forward_iterator_tag iterator_category;
-		typedef size_t difference_type;
-		typedef boost::asio::const_buffer* pointer;
-		typedef boost::asio::const_buffer& reference;
-
-		SendBuffer const* p;
-		int limit;
-
-		SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits<int>::max()) : p(p), limit(limit) {
-			ASSERT(limit > 0);
-		}
-
-		bool operator == (SendBufferIterator const& r) const { return p == r.p; }
-		bool operator != (SendBufferIterator const& r) const { return p != r.p; }
-		void operator++() {
-			limit -= p->bytes_written - p->bytes_sent;
-			if(limit > 0)
-				p = p->next;
-			else
-				p = NULL;
-		}
-
-		boost::asio::const_buffer operator*() const {
-			return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) );
-		}
-	};
-
 	void init() {
 		// Socket settings that have to be set after connect or accept succeeds
 		socket.non_blocking(true);