diff --git a/net/curl/inc/ROOT/RCurlConnection.hxx b/net/curl/inc/ROOT/RCurlConnection.hxx index 4a8857050fa8b..472bc46fae78d 100644 --- a/net/curl/inc/ROOT/RCurlConnection.hxx +++ b/net/curl/inc/ROOT/RCurlConnection.hxx @@ -75,6 +75,7 @@ private: void SetupErrorBuffer(); void SetOptions(); + void ResetHandle(); RResult SetUrl(const std::string &url); void Perform(RStatus &status); @@ -116,6 +117,8 @@ public: /// a valid batching of requests into multiple multi-range requests takes place automatically. /// The fNBytesRecv member of the ranges is only well-defined on success. RStatus SendRangesReq(std::size_t N, RUserRange *ranges); + /// Uploads data to the URL using an HTTP PUT request. + RStatus SendPutReq(const unsigned char *data, std::size_t length); const std::string &GetEscapedUrl() const { return fEscapedUrl; } diff --git a/net/curl/src/RCurlConnection.cxx b/net/curl/src/RCurlConnection.cxx index 4a2315319b07b..ffadb74289696 100644 --- a/net/curl/src/RCurlConnection.cxx +++ b/net/curl/src/RCurlConnection.cxx @@ -552,6 +552,59 @@ void ReverseDisplacements(std::vector &displacements, ROOT::Interna } } +/// State for the PUT upload read callback: tracks progress through the upload buffer. +struct RUploadState { + const unsigned char *fData = nullptr; + std::size_t fLength = 0; + std::size_t fOffset = 0; +}; + +/// CURLOPT_READFUNCTION callback for PUT uploads. Copies `requested` bytes from the upload +/// buffer into `buffer` and advances the offset. Returns 0 at end-of-data so that curl knows +/// the upload is complete. Since CURLOPT_INFILESIZE_LARGE is set, curl knows the total size +/// and will never request more bytes than remain. +std::size_t CallbackPutRead(char *buffer, std::size_t size, std::size_t nmemb, void *userdata) +{ + auto *state = static_cast(userdata); + R__ASSERT(state->fOffset <= state->fLength); + + std::size_t remaining = state->fLength - state->fOffset; + if (remaining == 0) + return 0; + + std::size_t requested = size * nmemb; + // CURL_READFUNC_ABORT (0x10000000) collides with a valid byte count at 256 MiB; + // assert that curl never asks for that much in a single callback invocation. + R__ASSERT(requested < CURL_READFUNC_ABORT); + R__ASSERT(requested <= remaining); + memcpy(buffer, state->fData + state->fOffset, requested); + state->fOffset += requested; + return requested; +} + +/// CURLOPT_SEEKFUNCTION callback for PUT uploads. Required because CURLOPT_FOLLOWLOCATION +/// is enabled: on a redirect curl needs to rewind the upload data before resending. +int CallbackPutSeek(void *userdata, curl_off_t offset, int origin) +{ + auto *state = static_cast(userdata); + // curl documents that it will only use SEEK_SET; guard against anything else defensively. + if (origin != SEEK_SET) + return CURL_SEEKFUNC_CANTSEEK; + if (offset < 0 || static_cast(offset) > state->fLength) + return CURL_SEEKFUNC_FAIL; + state->fOffset = static_cast(offset); + return CURL_SEEKFUNC_OK; +} + +/// Wrapper around curl_easy_setopt that asserts on failure. Most option-setting calls in this +/// file use valid options and values by construction, so failure indicates a programming error. +template +void SetCurlOption(void *handle, CURLoption option, T value) +{ + auto rc = curl_easy_setopt(handle, option, value); + R__ASSERT(rc == CURLE_OK); +} + std::string GetCurlErrorString(CURLcode code) { return std::string(curl_easy_strerror(code)) + " (" + std::to_string(code) + ")"; @@ -631,33 +684,42 @@ void ROOT::Internal::RCurlConnection::SetupErrorBuffer() { if (!fErrorBuffer) fErrorBuffer = std::make_unique(CURL_ERROR_SIZE); - auto rc = curl_easy_setopt(fHandle, CURLOPT_ERRORBUFFER, fErrorBuffer.get()); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_ERRORBUFFER, fErrorBuffer.get()); } void ROOT::Internal::RCurlConnection::SetOptions() { - int rc; - if (gDebug) { - rc = curl_easy_setopt(fHandle, CURLOPT_VERBOSE, 1); - R__ASSERT(rc == CURLE_OK); - rc = curl_easy_setopt(fHandle, CURLOPT_DEBUGFUNCTION, CallbackDebug); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_VERBOSE, 1); + SetCurlOption(fHandle, CURLOPT_DEBUGFUNCTION, CallbackDebug); } else { - rc = curl_easy_setopt(fHandle, CURLOPT_VERBOSE, 0); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_VERBOSE, 0); } static const std::string kUserAgent = GetUserAgentString(); - rc = curl_easy_setopt(fHandle, CURLOPT_USERAGENT, kUserAgent.c_str()); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_USERAGENT, kUserAgent.c_str()); + SetCurlOption(fHandle, CURLOPT_FOLLOWLOCATION, 1); + SetCurlOption(fHandle, CURLOPT_WRITEFUNCTION, CallbackData); +} - rc = curl_easy_setopt(fHandle, CURLOPT_FOLLOWLOCATION, 1); - R__ASSERT(rc == CURLE_OK); +/// Reset method-specific sticky curl options so that the easy handle is in a clean state +/// before configuring it for the next request (HEAD, GET, or PUT). +void ROOT::Internal::RCurlConnection::ResetHandle() +{ + SetCurlOption(fHandle, CURLOPT_NOBODY, 0L); + SetCurlOption(fHandle, CURLOPT_HTTPGET, 0L); + SetCurlOption(fHandle, CURLOPT_UPLOAD, 0L); + SetCurlOption(fHandle, CURLOPT_RANGE, static_cast(nullptr)); + SetCurlOption(fHandle, CURLOPT_READFUNCTION, static_cast(nullptr)); + SetCurlOption(fHandle, CURLOPT_READDATA, static_cast(nullptr)); + SetCurlOption(fHandle, CURLOPT_SEEKFUNCTION, static_cast(nullptr)); + SetCurlOption(fHandle, CURLOPT_SEEKDATA, static_cast(nullptr)); + SetCurlOption(fHandle, CURLOPT_INFILESIZE_LARGE, static_cast(-1)); - rc = curl_easy_setopt(fHandle, CURLOPT_WRITEFUNCTION, CallbackData); - R__ASSERT(rc == CURLE_OK); +#ifndef HAS_CURL_EASY_HEADER + SetCurlOption(fHandle, CURLOPT_HEADERFUNCTION, static_cast(nullptr)); + SetCurlOption(fHandle, CURLOPT_HEADERDATA, static_cast(nullptr)); +#endif } ROOT::RResult ROOT::Internal::RCurlConnection::SetUrl(const std::string &url) @@ -731,23 +793,14 @@ ROOT::Internal::RCurlConnection::RStatus ROOT::Internal::RCurlConnection::SendHe { remoteSize = kUnknownSize; - auto rc = curl_easy_setopt(fHandle, CURLOPT_NOBODY, 1); - R__ASSERT(rc == CURLE_OK); - rc = curl_easy_setopt(fHandle, CURLOPT_RANGE, NULL); // may have been set by a previous SendRangesReq() on the object - R__ASSERT(rc == CURLE_OK); - -#ifndef HAS_CURL_EASY_HEADER - rc = curl_easy_setopt(fHandle, CURLOPT_HEADERFUNCTION, NULL); - R__ASSERT(rc == CURLE_OK); - rc = curl_easy_setopt(fHandle, CURLOPT_HEADERDATA, NULL); - R__ASSERT(rc == CURLE_OK); -#endif + ResetHandle(); + SetCurlOption(fHandle, CURLOPT_NOBODY, 1); RStatus status; Perform(status); if (status) { curl_off_t length = -1; - rc = curl_easy_getinfo(fHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length); + auto rc = curl_easy_getinfo(fHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length); if (rc == CURLE_OK && length >= 0) remoteSize = length; } @@ -778,18 +831,15 @@ ROOT::Internal::RCurlConnection::SendRangesReq(std::size_t N, RUserRange *ranges return RStatus(RStatus::kSuccess); } - auto rc = curl_easy_setopt(fHandle, CURLOPT_HTTPGET, 1); - R__ASSERT(rc == CURLE_OK); + ResetHandle(); + SetCurlOption(fHandle, CURLOPT_HTTPGET, 1); RTransferState transfer(ranges, order, fHandle); - rc = curl_easy_setopt(fHandle, CURLOPT_WRITEDATA, &transfer); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_WRITEDATA, &transfer); #ifndef HAS_CURL_EASY_HEADER - rc = curl_easy_setopt(fHandle, CURLOPT_HEADERFUNCTION, CallbackHeader); - R__ASSERT(rc == CURLE_OK); - rc = curl_easy_setopt(fHandle, CURLOPT_HEADERDATA, &transfer); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_HEADERFUNCTION, CallbackHeader); + SetCurlOption(fHandle, CURLOPT_HEADERDATA, &transfer); #endif RStatus status; @@ -811,8 +861,7 @@ ROOT::Internal::RCurlConnection::SendRangesReq(std::size_t N, RUserRange *ranges for (std::size_t i = 1; i < nRanges; ++i) { rangeHeader += "," + requestRanges[b + i].ToString(); } - rc = curl_easy_setopt(fHandle, CURLOPT_RANGE, rangeHeader.c_str()); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_RANGE, rangeHeader.c_str()); if (b > 0) { const std::uint64_t lastByteRequested = requestRanges[b - 1].fLastByte; @@ -849,6 +898,26 @@ ROOT::Internal::RCurlConnection::SendRangesReq(std::size_t N, RUserRange *ranges return status; } +ROOT::Internal::RCurlConnection::RStatus +ROOT::Internal::RCurlConnection::SendPutReq(const unsigned char *data, std::size_t length) +{ + ResetHandle(); + + SetCurlOption(fHandle, CURLOPT_UPLOAD, 1L); + SetCurlOption(fHandle, CURLOPT_INFILESIZE_LARGE, static_cast(length)); + + RUploadState uploadState{data, length, 0}; + SetCurlOption(fHandle, CURLOPT_READFUNCTION, CallbackPutRead); + SetCurlOption(fHandle, CURLOPT_READDATA, &uploadState); + SetCurlOption(fHandle, CURLOPT_SEEKFUNCTION, CallbackPutSeek); + SetCurlOption(fHandle, CURLOPT_SEEKDATA, &uploadState); + + RStatus status; + Perform(status); + + return status; +} + void ROOT::Internal::RCurlConnection::SetCredentials(const RS3Credentials &credentials) { ClearCredentials(); @@ -876,13 +945,10 @@ void ROOT::Internal::RCurlConnection::ClearCredentials() if (!fCredentials) return; - CURLcode rc; switch (fCredentials->fType) { case EHTTPCredentialsType::kS3: - rc = curl_easy_setopt(fHandle, CURLOPT_AWS_SIGV4, NULL); - R__ASSERT(rc == CURLE_OK); - rc = curl_easy_setopt(fHandle, CURLOPT_USERPWD, NULL); - R__ASSERT(rc == CURLE_OK); + SetCurlOption(fHandle, CURLOPT_AWS_SIGV4, static_cast(nullptr)); + SetCurlOption(fHandle, CURLOPT_USERPWD, static_cast(nullptr)); break; default: R__ASSERT(false && "internal error: unknown credentials type"); } diff --git a/net/curl/test/curl_connection.cxx b/net/curl/test/curl_connection.cxx index 80fd922ae7afa..6ebb188563fa4 100644 --- a/net/curl/test/curl_connection.cxx +++ b/net/curl/test/curl_connection.cxx @@ -4,10 +4,67 @@ #include "TServerSocket.h" +#include +#include #include #include #include #include +#include + +/// Return a lower-cased copy of the input string. +static std::string ToLower(std::string s) +{ + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); }); + return s; +} + +/// Accept a PUT request: read headers + body, optionally respond to Expect: 100-continue, send 200 OK. +static void TaskRecvPut(TServerSocket *serverSocket, std::string *requestHeaders, std::string *requestBody) +{ + requestHeaders->clear(); + requestBody->clear(); + auto sock = serverSocket->Accept(); + + const char *eof = "\r\n\r\n"; + const std::size_t eofLen = strlen(eof); + std::size_t nextInEof = 0; + char c; + while (sock->RecvRaw(&c, 1)) { + requestHeaders->push_back(c); + if (c == eof[nextInEof]) { + if (++nextInEof == eofLen) + break; + } else { + nextInEof = 0; + } + } + + // If the client sent Expect: 100-continue, respond with HTTP 100 before reading the body + std::string headersLower = ToLower(*requestHeaders); + if (headersLower.find("expect: 100-continue") != std::string::npos) { + const char *continueResponse = "HTTP/1.1 100 Continue\r\n\r\n"; + sock->SendRaw(continueResponse, strlen(continueResponse)); + } + + // Parse content-length (case-insensitive) + std::size_t contentLength = 0; + auto pos = headersLower.find("content-length: "); + if (pos != std::string::npos) { + auto valStart = pos + strlen("content-length: "); + auto valEnd = headersLower.find("\r\n", valStart); + contentLength = std::stoul(headersLower.substr(valStart, valEnd - valStart)); + } + + if (contentLength > 0) { + requestBody->resize(contentLength); + sock->RecvRaw(&(*requestBody)[0], contentLength); + } + + const char *response = "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"; + sock->SendRaw(response, strlen(response)); + sock->Close(); +} static void TaskRecv(TServerSocket *serverSocket, std::string *request) { @@ -63,3 +120,64 @@ TEST(RCurlConnection, Cred) threadRecv.join(); EXPECT_EQ(std::string::npos, request.find("\r\nAuthorization: ")); } + +TEST(RCurlConnection, Put) +{ + TServerSocket sock(0, false, TServerSocket::kDefaultBacklog, -1, ESocketBindOption::kInaddrLoopback); + const std::string url = + std::string("http://") + sock.GetLocalInetAddress().GetHostAddress() + ":" + std::to_string(sock.GetLocalPort()); + + const unsigned char payload[] = "Hello, S3!"; + const std::size_t payloadLen = sizeof(payload) - 1; // exclude null terminator + + std::string headers; + std::string body; + std::thread threadRecv(TaskRecvPut, &sock, &headers, &body); + + ROOT::Internal::RCurlConnection conn(url); + auto status = conn.SendPutReq(payload, payloadLen); + + threadRecv.join(); + EXPECT_TRUE(static_cast(status)); + EXPECT_EQ(0u, headers.find("PUT ")); + + // Normalize headers to lower-case for case-insensitive matching + std::string headersLower = ToLower(headers); + auto clPos = headersLower.find("content-length: " + std::to_string(payloadLen)); + ASSERT_NE(std::string::npos, clPos) << "content-length header not found in request"; + + EXPECT_EQ(std::string(reinterpret_cast(payload), payloadLen), body); +} + +/// PUT with a payload larger than libcurl's internal Expect: 100-continue threshold (1 MB since curl 7.69). +/// Verifies that the server-side 100 Continue handshake works and all bytes arrive correctly. +TEST(RCurlConnection, PutLargeExpect100) +{ + TServerSocket sock(0, false, TServerSocket::kDefaultBacklog, -1, ESocketBindOption::kInaddrLoopback); + const std::string url = + std::string("http://") + sock.GetLocalInetAddress().GetHostAddress() + ":" + std::to_string(sock.GetLocalPort()); + + // 2 MB payload with a known repeating pattern + const std::size_t payloadLen = 2 * 1024 * 1024; + std::vector payload(payloadLen); + for (std::size_t i = 0; i < payloadLen; ++i) + payload[i] = static_cast(i & 0xFF); + + std::string headers; + std::string body; + std::thread threadRecv(TaskRecvPut, &sock, &headers, &body); + + ROOT::Internal::RCurlConnection conn(url); + auto status = conn.SendPutReq(payload.data(), payloadLen); + + threadRecv.join(); + EXPECT_TRUE(static_cast(status)); + EXPECT_EQ(0u, headers.find("PUT ")); + + std::string headersLower = ToLower(headers); + EXPECT_NE(std::string::npos, headersLower.find("expect: 100-continue")) + << "large upload should include Expect: 100-continue header"; + EXPECT_NE(std::string::npos, headersLower.find("content-length: " + std::to_string(payloadLen))); + ASSERT_EQ(payloadLen, body.size()); + EXPECT_EQ(0, memcmp(body.data(), payload.data(), payloadLen)); +} diff --git a/net/curl/test/curl_env.cxx b/net/curl/test/curl_env.cxx index bad292f6a3068..4cb07d6d731c4 100644 --- a/net/curl/test/curl_env.cxx +++ b/net/curl/test/curl_env.cxx @@ -11,8 +11,10 @@ #include "TCurlFile.h" #include "TSystem.h" +#include #include #include +#include TEST(RCurlConnection, CredFromEnv) { @@ -93,3 +95,58 @@ TEST(CurlFile, S3Credentials) gSystem->Unsetenv("S3_ACCESS_KEY"); gSystem->Unsetenv("S3_SECRET_KEY"); } + +TEST(CurlFile, S3PutAndRead) +{ + const auto testAccessKey = std::getenv("ROOT_TEST_S3_ACCESS_KEY"); + const auto testSecretKey = std::getenv("ROOT_TEST_S3_SECRET_KEY"); + if (!testAccessKey || testAccessKey[0] == '\0' || !testSecretKey || testSecretKey[0] == '\0') { + GTEST_SKIP() << "Missing S3 test credentials , skipping"; + } + if (ROOT::Internal::RCurlConnection::GetCurlVersion() <= 0x078100) { + GTEST_SKIP() << "libcurl <= 7.81 is known to produce an AWSv4 signature incompatible with Ceph S3"; + } + + const std::string url = "https://root-project-s3test.s3.cern.ch/test-curl-put-roundtrip.bin"; + + ROOT::Internal::RS3Credentials creds; + creds.fAccessKey = testAccessKey; + creds.fSecretKey = testSecretKey; + + // PUT a known payload + const unsigned char payload[] = "RCurlConnection::SendPutReq round-trip test"; + const std::size_t payloadLen = sizeof(payload) - 1; + + { + ROOT::Internal::RCurlConnection conn(url); + conn.SetCredentials(creds); + auto status = conn.SendPutReq(payload, payloadLen); + ASSERT_TRUE(static_cast(status)) << "PUT failed: " << status.fStatusMsg; + } + + // HEAD to verify size + { + ROOT::Internal::RCurlConnection conn(url); + conn.SetCredentials(creds); + std::uint64_t remoteSize = 0; + auto status = conn.SendHeadReq(remoteSize); + ASSERT_TRUE(static_cast(status)) << "HEAD failed: " << status.fStatusMsg; + EXPECT_EQ(static_cast(payloadLen), remoteSize); + } + + // GET (range read) to verify content + { + std::vector readback(payloadLen, 0); + ROOT::Internal::RCurlConnection::RUserRange range; + range.fDestination = readback.data(); + range.fOffset = 0; + range.fLength = payloadLen; + + ROOT::Internal::RCurlConnection conn(url); + conn.SetCredentials(creds); + auto status = conn.SendRangesReq(1, &range); + ASSERT_TRUE(static_cast(status)) << "GET failed: " << status.fStatusMsg; + EXPECT_EQ(payloadLen, range.fNBytesRecv); + EXPECT_EQ(0, memcmp(readback.data(), payload, payloadLen)); + } +}