[llvm] [Support] Add CURL HTTP Client.

Provides an implementation of `HTTPClient` that wraps libcurl.

Reviewed By: dblaikie

Differential Revision: https://reviews.llvm.org/D112753

Author: noajshu

llvm/CMakeLists.txt

@@ -392,6 +392,8 @@ endif()
 
 set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
 
+set(LLVM_ENABLE_CURL "ON" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON")
+
 set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.")
 
 option(LLVM_ENABLE_Z3_SOLVER

llvm/include/llvm/Config/llvm-config.h.cmake

@@ -85,6 +85,9 @@
 /* Define if we have z3 and want to build it */
 #cmakedefine LLVM_WITH_Z3 ${LLVM_WITH_Z3}
 
+/* Define if we have curl and want to use it */
+#cmakedefine LLVM_ENABLE_CURL ${LLVM_ENABLE_CURL}
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 

llvm/include/llvm/Support/HTTPClient.h

@@ -77,6 +77,11 @@ class BufferedHTTPResponseHandler final : public HTTPResponseHandler {
 
 /// A reusable client that can perform HTTPRequests through a network socket.
 class HTTPClient {
+#ifdef LLVM_ENABLE_CURL
+  void *Curl = nullptr;
+  static bool IsInitialized;
+#endif
+
 public:
   HTTPClient();
   ~HTTPClient();

llvm/lib/Support/CMakeLists.txt

@@ -74,6 +74,11 @@ if(LLVM_WITH_Z3)
   set(system_libs ${system_libs} ${Z3_LIBRARIES})
 endif()
 
+# Link LibCURL if the user wants it
+if (LLVM_ENABLE_CURL)
+  set(system_libs ${system_libs} ${CURL_LIBRARIES})
+endif()
+
 # Override the C runtime allocator on Windows and embed it into LLVM tools & libraries
 if(LLVM_INTEGRATED_CRT_ALLOC)
   if (CMAKE_BUILD_TYPE AND NOT ${LLVM_USE_CRT_${uppercase_CMAKE_BUILD_TYPE}} MATCHES "^(MT|MTd)$")

llvm/lib/Support/HTTPClient.cpp

@@ -19,6 +19,9 @@
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
+#ifdef LLVM_ENABLE_CURL
+#include <curl/curl.h>
+#endif
 
 using namespace llvm;
 
@@ -81,12 +84,120 @@ Expected<HTTPResponseBuffer> HTTPClient::get(StringRef Url) {
   return perform(Request);
 }
 
+#ifdef LLVM_ENABLE_CURL
+
+bool HTTPClient::isAvailable() { return true; }
+
+bool HTTPClient::IsInitialized = false;
+
+void HTTPClient::initialize() {
+  if (!IsInitialized) {
+    curl_global_init(CURL_GLOBAL_ALL);
+    IsInitialized = true;
+  }
+}
+
+void HTTPClient::cleanup() {
+  if (IsInitialized) {
+    curl_global_cleanup();
+    IsInitialized = false;
+  }
+}
+
+void HTTPClient::setTimeout(std::chrono::milliseconds Timeout) {
+  if (Timeout < std::chrono::milliseconds(0))
+    Timeout = std::chrono::milliseconds(0);
+  curl_easy_setopt(Curl, CURLOPT_TIMEOUT_MS, Timeout.count());
+}
+
+/// CurlHTTPRequest and the curl{Header,Write}Function are implementation
+/// details used to work with Curl. Curl makes callbacks with a single
+/// customizable pointer parameter.
+struct CurlHTTPRequest {
+  CurlHTTPRequest(HTTPResponseHandler &Handler) : Handler(Handler) {}
+  void storeError(Error Err) {
+    ErrorState = joinErrors(std::move(Err), std::move(ErrorState));
+  }
+  HTTPResponseHandler &Handler;
+  llvm::Error ErrorState = Error::success();
+};
+
+static size_t curlHeaderFunction(char *Contents, size_t Size, size_t NMemb,
+                                 CurlHTTPRequest *CurlRequest) {
+  assert(Size == 1 && "The Size passed by libCURL to CURLOPT_HEADERFUNCTION "
+                      "should always be 1.");
+  if (Error Err =
+          CurlRequest->Handler.handleHeaderLine(StringRef(Contents, NMemb))) {
+    CurlRequest->storeError(std::move(Err));
+    return 0;
+  }
+  return NMemb;
+}
+
+static size_t curlWriteFunction(char *Contents, size_t Size, size_t NMemb,
+                                CurlHTTPRequest *CurlRequest) {
+  Size *= NMemb;
+  if (Error Err =
+          CurlRequest->Handler.handleBodyChunk(StringRef(Contents, Size))) {
+    CurlRequest->storeError(std::move(Err));
+    return 0;
+  }
+  return Size;
+}
+
+HTTPClient::HTTPClient() {
+  assert(IsInitialized &&
+         "Must call HTTPClient::initialize() at the beginning of main().");
+  if (Curl)
+    return;
+  assert((Curl = curl_easy_init()) && "Curl could not be initialized.");
+  // Set the callback hooks.
+  curl_easy_setopt(Curl, CURLOPT_WRITEFUNCTION, curlWriteFunction);
+  curl_easy_setopt(Curl, CURLOPT_HEADERFUNCTION, curlHeaderFunction);
+}
+
+HTTPClient::~HTTPClient() { curl_easy_cleanup(Curl); }
+
+Error HTTPClient::perform(const HTTPRequest &Request,
+                          HTTPResponseHandler &Handler) {
+  if (Request.Method != HTTPMethod::GET)
+    return createStringError(errc::invalid_argument,
+                             "Unsupported CURL request method.");
+
+  SmallString<128> Url = Request.Url;
+  curl_easy_setopt(Curl, CURLOPT_URL, Url.c_str());
+  curl_easy_setopt(Curl, CURLOPT_FOLLOWLOCATION, Request.FollowRedirects);
+
+  CurlHTTPRequest CurlRequest(Handler);
+  curl_easy_setopt(Curl, CURLOPT_WRITEDATA, &CurlRequest);
+  curl_easy_setopt(Curl, CURLOPT_HEADERDATA, &CurlRequest);
+  CURLcode CurlRes = curl_easy_perform(Curl);
+  if (CurlRes != CURLE_OK)
+    return joinErrors(std::move(CurlRequest.ErrorState),
+                      createStringError(errc::io_error,
+                                        "curl_easy_perform() failed: %s\n",
+                                        curl_easy_strerror(CurlRes)));
+  if (CurlRequest.ErrorState)
+    return std::move(CurlRequest.ErrorState);
+
+  unsigned Code;
+  curl_easy_getinfo(Curl, CURLINFO_RESPONSE_CODE, &Code);
+  if (Error Err = Handler.handleStatusCode(Code))
+    return joinErrors(std::move(CurlRequest.ErrorState), std::move(Err));
+
+  return std::move(CurlRequest.ErrorState);
+}
+
+#else
+
 HTTPClient::HTTPClient() = default;
 
 HTTPClient::~HTTPClient() = default;
 
 bool HTTPClient::isAvailable() { return false; }
 
+void HTTPClient::initialize() {}
+
 void HTTPClient::cleanup() {}
 
 void HTTPClient::setTimeout(std::chrono::milliseconds Timeout) {}
@@ -95,3 +206,5 @@ Error HTTPClient::perform(const HTTPRequest &Request,
                           HTTPResponseHandler &Handler) {
   llvm_unreachable("No HTTP Client implementation available.");
 }
+
+#endif

llvm/lib/Support/InitLLVM.cpp

@@ -8,6 +8,7 @@
 
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/HTTPClient.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Process.h"
@@ -58,6 +59,11 @@ InitLLVM::InitLLVM(int &Argc, const char **&Argv,
   Argc = Args.size() - 1;
   Argv = Args.data();
 #endif
+
+  HTTPClient::initialize();
 }
 
-InitLLVM::~InitLLVM() { llvm_shutdown(); }
+InitLLVM::~InitLLVM() {
+  HTTPClient::cleanup();
+  llvm_shutdown();
+}

llvm/unittests/Support/HTTPClient.cpp

@@ -86,3 +86,9 @@ TEST(BufferedHTTPResponseHandler, MalformedContentLength) {
   EXPECT_THAT_ERROR(Handler.handleBodyChunk("non-empty body content"),
                     Failed<llvm::StringError>());
 }
+
+#ifdef LLVM_ENABLE_CURL
+
+TEST(HTTPClient, isAvailable) { EXPECT_TRUE(HTTPClient::isAvailable()); }
+
+#endif