From 9f41a73a76efc8f46eb0ac523b2c891acdb8b07b Mon Sep 17 00:00:00 2001
From: Woo Xie <woo@fb.com>
Date: Thu, 11 Jun 2015 16:49:29 -0700
Subject: [PATCH] opt proxygen with new SSL_write_iovec function

Summary: this taks advantage of the SSL_write_iovec function to improve CPU.  According our testing, it saves 10+% CPU.

Reviewed By: @djwatson

Differential Revision: D2119223
---
 folly/io/async/AsyncSSLSocket.cpp | 58 ++++++++++++++++++++++++++++---
 folly/io/async/AsyncSSLSocket.h   |  4 +++
 2 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/folly/io/async/AsyncSSLSocket.cpp b/folly/io/async/AsyncSSLSocket.cpp
index 447f2543..66b4567c 100644
--- a/folly/io/async/AsyncSSLSocket.cpp
+++ b/folly/io/async/AsyncSSLSocket.cpp
@@ -220,6 +220,13 @@ void setup_SSL_CTX(SSL_CTX *ctx) {
                    SSL_MODE_ENABLE_PARTIAL_WRITE
                    );
 #endif
+// SSL_CTX_set_mode is a Macro
+#ifdef SSL_MODE_WRITE_IOVEC
+  SSL_CTX_set_mode(ctx,
+                   SSL_CTX_get_mode(ctx)
+                   | SSL_MODE_WRITE_IOVEC);
+#endif
+
 }
 
 BIO_METHOD eorAwareBioMethod;
@@ -1099,8 +1106,8 @@ AsyncSSLSocket::performRead(void* buf, size_t buflen) {
   ssize_t bytes = SSL_read(ssl_, buf, buflen);
   if (server_ && renegotiateAttempted_) {
     LOG(ERROR) << "AsyncSSLSocket(fd=" << fd_ << ", state=" << int(state_)
-               << ", sslstate=" << sslState_ << ", events=" << eventFlags_ << "): "
-               << "client intitiated SSL renegotiation not permitted";
+               << ", sslstate=" << sslState_ << ", events=" << eventFlags_
+               << "): client intitiated SSL renegotiation not permitted";
     // We pack our own SSLerr here with a dummy function
     errno = ERR_PACK(ERR_LIB_USER, TASYNCSSLSOCKET_F_PERFORM_READ,
                      SSL_CLIENT_RENEGOTIATION_ATTEMPT);
@@ -1117,8 +1124,8 @@ AsyncSSLSocket::performRead(void* buf, size_t buflen) {
       // need to write data if renegotiation is being performed.  We currently
       // don't support this and just fail the read.
       LOG(ERROR) << "AsyncSSLSocket(fd=" << fd_ << ", state=" << int(state_)
-                 << ", sslState=" << sslState_ << ", events=" << eventFlags_ << "): "
-                 << "unsupported SSL renegotiation during read",
+                 << ", sslState=" << sslState_ << ", events=" << eventFlags_
+                 << "): unsupported SSL renegotiation during read",
       errno = ERR_PACK(ERR_LIB_USER, TASYNCSSLSOCKET_F_PERFORM_READ,
                        SSL_INVALID_RENEGOTIATION);
       ERR_clear_error();
@@ -1242,6 +1249,12 @@ ssize_t AsyncSSLSocket::performWrite(const iovec* vec,
   bool cork = isSet(flags, WriteFlags::CORK);
   CorkGuard guard(fd_, count > 1, cork, &corked_);
 
+#ifdef SSL_MODE_WRITE_IOVEC
+  if (ssl_->mode & SSL_MODE_WRITE_IOVEC) {
+    return performWriteIovec(vec, count, flags, countWritten, partialWritten);
+  }
+#endif
+
   // Declare a buffer used to hold small write requests.  It could point to a
   // memory block either on stack or on heap. If it is on heap, we release it
   // manually when scope exits
@@ -1357,6 +1370,43 @@ ssize_t AsyncSSLSocket::performWrite(const iovec* vec,
   return totalWritten;
 }
 
+#ifdef SSL_MODE_WRITE_IOVEC
+ssize_t AsyncSSLSocket::performWriteIovec(const iovec* vec,
+                                          uint32_t count,
+                                          WriteFlags flags,
+                                          uint32_t* countWritten,
+                                          uint32_t* partialWritten) {
+  size_t tot = 0;
+  for (uint32_t j = 0; j < count; j++) {
+    tot += vec[j].iov_len;
+  }
+
+  ssize_t totalWritten = SSL_write_iovec(ssl_, vec, count);
+
+  *countWritten = 0;
+  *partialWritten = 0;
+  if (totalWritten <= 0) {
+    return interpretSSLError(totalWritten, SSL_get_error(ssl_, totalWritten));
+  } else {
+    ssize_t bytes = totalWritten, i = 0;
+    while (i < count && bytes >= (ssize_t)vec[i].iov_len) {
+      // we managed to write all of this buf
+      bytes -= vec[i].iov_len;
+      (*countWritten)++;
+      i++;
+    }
+    *partialWritten = bytes;
+
+    VLOG(4) << "SSL_write_iovec() writes " << tot
+            << ", returns " << totalWritten << " bytes"
+            << ", max_send_fragment=" << ssl_->max_send_fragment
+            << ", count=" << count << ", countWritten=" << *countWritten;
+
+    return totalWritten;
+  }
+}
+#endif
+
 int AsyncSSLSocket::eorAwareSSLWrite(SSL *ssl, const void *buf, int n,
                                       bool eor) {
   if (eor && SSL_get_wbio(ssl)->method == &eorAwareBioMethod) {
diff --git a/folly/io/async/AsyncSSLSocket.h b/folly/io/async/AsyncSSLSocket.h
index b72205de..0cc0dc92 100644
--- a/folly/io/async/AsyncSSLSocket.h
+++ b/folly/io/async/AsyncSSLSocket.h
@@ -692,6 +692,10 @@ class AsyncSSLSocket : public virtual AsyncSocket {
                        uint32_t* countWritten, uint32_t* partialWritten)
     override;
 
+  ssize_t performWriteIovec(const iovec* vec, uint32_t count,
+                            WriteFlags flags, uint32_t* countWritten,
+                            uint32_t* partialWritten);
+
   // This virtual wrapper around SSL_write exists solely for testing/mockability
   virtual int sslWriteImpl(SSL *ssl, const void *buf, int n) {
     return SSL_write(ssl, buf, n);
-- 
2.34.1