aboutsummaryrefslogtreecommitdiff
path: root/src/httpfetch.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/httpfetch.cpp')
-rw-r--r--src/httpfetch.cpp226
1 files changed, 130 insertions, 96 deletions
diff --git a/src/httpfetch.cpp b/src/httpfetch.cpp
index 1883cf263..326b5052f 100644
--- a/src/httpfetch.cpp
+++ b/src/httpfetch.cpp
@@ -38,16 +38,17 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "noise.h"
std::mutex g_httpfetch_mutex;
-std::map<unsigned long, std::queue<HTTPFetchResult>> g_httpfetch_results;
+std::map<unsigned long, std::queue<HTTPFetchResult> > g_httpfetch_results;
PcgRandom g_callerid_randomness;
HTTPFetchRequest::HTTPFetchRequest() :
- timeout(g_settings->getS32("curl_timeout")), connect_timeout(timeout),
- useragent(std::string(PROJECT_NAME_C "/") + g_version_hash + " (" +
- porting::get_sysinfo() + ")")
+ timeout(g_settings->getS32("curl_timeout")),
+ connect_timeout(timeout),
+ useragent(std::string(PROJECT_NAME_C "/") + g_version_hash + " (" + porting::get_sysinfo() + ")")
{
}
+
static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result)
{
unsigned long caller = fetch_result.caller;
@@ -66,11 +67,11 @@ unsigned long httpfetch_caller_alloc()
// Check each caller ID except HTTPFETCH_DISCARD
const unsigned long discard = HTTPFETCH_DISCARD;
for (unsigned long caller = discard + 1; caller != discard; ++caller) {
- std::map<unsigned long, std::queue<HTTPFetchResult>>::iterator it =
- g_httpfetch_results.find(caller);
+ std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
+ it = g_httpfetch_results.find(caller);
if (it == g_httpfetch_results.end()) {
- verbosestream << "httpfetch_caller_alloc: allocating " << caller
- << std::endl;
+ verbosestream << "httpfetch_caller_alloc: allocating "
+ << caller << std::endl;
// Access element to create it
g_httpfetch_results[caller];
return caller;
@@ -92,18 +93,17 @@ unsigned long httpfetch_caller_alloc_secure()
unsigned long caller;
do {
- caller = (((u64)g_callerid_randomness.next()) << 32) |
- g_callerid_randomness.next();
+ caller = (((u64) g_callerid_randomness.next()) << 32) |
+ g_callerid_randomness.next();
if (--tries < 1) {
- FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller "
- "IDs");
+ FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs");
return HTTPFETCH_DISCARD;
}
} while (g_httpfetch_results.find(caller) != g_httpfetch_results.end());
- verbosestream << "httpfetch_caller_alloc_secure: allocating " << caller
- << std::endl;
+ verbosestream << "httpfetch_caller_alloc_secure: allocating "
+ << caller << std::endl;
// Access element to create it
g_httpfetch_results[caller];
@@ -112,7 +112,8 @@ unsigned long httpfetch_caller_alloc_secure()
void httpfetch_caller_free(unsigned long caller)
{
- verbosestream << "httpfetch_caller_free: freeing " << caller << std::endl;
+ verbosestream<<"httpfetch_caller_free: freeing "
+ <<caller<<std::endl;
httpfetch_request_clear(caller);
if (caller != HTTPFETCH_DISCARD) {
@@ -126,8 +127,8 @@ bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result)
MutexAutoLock lock(g_httpfetch_mutex);
// Check that caller exists
- std::map<unsigned long, std::queue<HTTPFetchResult>>::iterator it =
- g_httpfetch_results.find(caller);
+ std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
+ it = g_httpfetch_results.find(caller);
if (it == g_httpfetch_results.end())
return false;
@@ -152,7 +153,7 @@ bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result)
static size_t httpfetch_writefunction(
char *ptr, size_t size, size_t nmemb, void *userdata)
{
- std::ostringstream *stream = (std::ostringstream *)userdata;
+ std::ostringstream *stream = (std::ostringstream*)userdata;
size_t count = size * nmemb;
stream->write(ptr, count);
return count;
@@ -166,28 +167,28 @@ static size_t httpfetch_discardfunction(
class CurlHandlePool
{
- std::list<CURL *> handles;
+ std::list<CURL*> handles;
public:
CurlHandlePool() = default;
~CurlHandlePool()
{
- for (std::list<CURL *>::iterator it = handles.begin();
+ for (std::list<CURL*>::iterator it = handles.begin();
it != handles.end(); ++it) {
curl_easy_cleanup(*it);
}
}
- CURL *alloc()
+ CURL * alloc()
{
CURL *curl;
if (handles.empty()) {
curl = curl_easy_init();
if (curl == NULL) {
- errorstream << "curl_easy_init returned NULL"
- << std::endl;
+ errorstream<<"curl_easy_init returned NULL"<<std::endl;
}
- } else {
+ }
+ else {
curl = handles.front();
handles.pop_front();
}
@@ -207,10 +208,10 @@ public:
~HTTPFetchOngoing();
CURLcode start(CURLM *multi);
- const HTTPFetchResult *complete(CURLcode res);
+ const HTTPFetchResult * complete(CURLcode res);
- const HTTPFetchRequest &getRequest() const { return request; };
- const CURL *getEasyHandle() const { return curl; };
+ const HTTPFetchRequest &getRequest() const { return request; };
+ const CURL *getEasyHandle() const { return curl; };
private:
CurlHandlePool *pool;
@@ -223,11 +224,17 @@ private:
curl_httppost *post;
};
-HTTPFetchOngoing::HTTPFetchOngoing(
- const HTTPFetchRequest &request_, CurlHandlePool *pool_) :
- pool(pool_),
- curl(NULL), multi(NULL), request(request_), result(request_),
- oss(std::ios::binary), http_header(NULL), post(NULL)
+
+HTTPFetchOngoing::HTTPFetchOngoing(const HTTPFetchRequest &request_,
+ CurlHandlePool *pool_):
+ pool(pool_),
+ curl(NULL),
+ multi(NULL),
+ request(request_),
+ result(request_),
+ oss(std::ios::binary),
+ http_header(NULL),
+ post(NULL)
{
curl = pool->alloc();
if (curl == NULL) {
@@ -255,15 +262,21 @@ HTTPFetchOngoing::HTTPFetchOngoing(
// other protocols don't affect us.
// These settings were introduced in curl 7.19.4.
long protocols =
- CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP | CURLPROTO_FTPS;
+ CURLPROTO_HTTP |
+ CURLPROTO_HTTPS |
+ CURLPROTO_FTP |
+ CURLPROTO_FTPS;
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols);
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols);
#endif
// Set cURL options based on HTTPFetchRequest
- curl_easy_setopt(curl, CURLOPT_URL, request.url.c_str());
- curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, request.timeout);
- curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, request.connect_timeout);
+ curl_easy_setopt(curl, CURLOPT_URL,
+ request.url.c_str());
+ curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS,
+ request.timeout);
+ curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
+ request.connect_timeout);
if (!request.useragent.empty())
curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str());
@@ -272,10 +285,12 @@ HTTPFetchOngoing::HTTPFetchOngoing(
// ostringstream ongoing->oss, unless the data
// is to be discarded
if (request.caller == HTTPFETCH_DISCARD) {
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, httpfetch_discardfunction);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+ httpfetch_discardfunction);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
} else {
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, httpfetch_writefunction);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+ httpfetch_writefunction);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
}
@@ -286,7 +301,8 @@ HTTPFetchOngoing::HTTPFetchOngoing(
curl_httppost *last = NULL;
for (StringMap::iterator it = request.post_fields.begin();
it != request.post_fields.end(); ++it) {
- curl_formadd(&post, &last, CURLFORM_NAMELENGTH, it->first.size(),
+ curl_formadd(&post, &last,
+ CURLFORM_NAMELENGTH, it->first.size(),
CURLFORM_PTRNAME, it->first.c_str(),
CURLFORM_CONTENTSLENGTH, it->second.size(),
CURLFORM_PTRCONTENTS, it->second.c_str(),
@@ -305,12 +321,16 @@ HTTPFetchOngoing::HTTPFetchOngoing(
str += "=";
str += urlencode(post_field.second);
}
- curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, str.size());
- curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, str.c_str());
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
+ str.size());
+ curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS,
+ str.c_str());
} else {
curl_easy_setopt(curl, CURLOPT_POST, 1);
- curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, request.post_data.size());
- curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request.post_data.c_str());
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
+ request.post_data.size());
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
+ request.post_data.c_str());
// request.post_data must now *never* be
// modified until CURLOPT_POSTFIELDS is cleared
}
@@ -339,14 +359,15 @@ CURLcode HTTPFetchOngoing::start(CURLM *multi_)
CURLMcode mres = curl_multi_add_handle(multi_, curl);
if (mres != CURLM_OK) {
errorstream << "curl_multi_add_handle"
- << " returned error code " << mres << std::endl;
+ << " returned error code " << mres
+ << std::endl;
return CURLE_FAILED_INIT;
}
multi = multi_; // store for curl_multi_remove_handle
return CURLE_OK;
}
-const HTTPFetchResult *HTTPFetchOngoing::complete(CURLcode res)
+const HTTPFetchResult * HTTPFetchOngoing::complete(CURLcode res)
{
result.succeeded = (res == CURLE_OK);
result.timeout = (res == CURLE_OPERATION_TIMEDOUT);
@@ -355,16 +376,16 @@ const HTTPFetchResult *HTTPFetchOngoing::complete(CURLcode res)
// Get HTTP/FTP response code
result.response_code = 0;
if (curl && (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE,
- &result.response_code) != CURLE_OK)) {
+ &result.response_code) != CURLE_OK)) {
// We failed to get a return code, make sure it is still 0
result.response_code = 0;
}
if (res != CURLE_OK) {
- errorstream << request.url << " not found (" << curl_easy_strerror(res)
- << ")"
- << " (response code " << result.response_code << ")"
- << std::endl;
+ errorstream << request.url << " not found ("
+ << curl_easy_strerror(res) << ")"
+ << " (response code " << result.response_code << ")"
+ << std::endl;
}
return &result;
@@ -376,12 +397,14 @@ HTTPFetchOngoing::~HTTPFetchOngoing()
CURLMcode mres = curl_multi_remove_handle(multi, curl);
if (mres != CURLM_OK) {
errorstream << "curl_multi_remove_handle"
- << " returned error code " << mres << std::endl;
+ << " returned error code " << mres
+ << std::endl;
}
}
// Set safe options for the reusable cURL handle
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, httpfetch_discardfunction);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+ httpfetch_discardfunction);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL);
if (http_header) {
@@ -397,18 +420,17 @@ HTTPFetchOngoing::~HTTPFetchOngoing()
pool->free(curl);
}
+
class CurlFetchThread : public Thread
{
protected:
- enum RequestType
- {
+ enum RequestType {
RT_FETCH,
RT_CLEAR,
RT_WAKEUP,
};
- struct Request
- {
+ struct Request {
RequestType type;
HTTPFetchRequest fetch_request;
Event *event;
@@ -419,11 +441,12 @@ protected:
size_t m_parallel_limit;
// Variables exclusively used within thread
- std::vector<HTTPFetchOngoing *> m_all_ongoing;
+ std::vector<HTTPFetchOngoing*> m_all_ongoing;
std::list<HTTPFetchRequest> m_queued_fetches;
public:
- CurlFetchThread(int parallel_limit) : Thread("CurlFetch")
+ CurlFetchThread(int parallel_limit) :
+ Thread("CurlFetch")
{
if (parallel_limit >= 1)
m_parallel_limit = parallel_limit;
@@ -469,12 +492,13 @@ protected:
// see processQueued() for what happens next
- } else if (req.type == RT_CLEAR) {
+ }
+ else if (req.type == RT_CLEAR) {
unsigned long caller = req.fetch_request.caller;
// Abort all ongoing fetches for the caller
- for (std::vector<HTTPFetchOngoing *>::iterator it =
- m_all_ongoing.begin();
+ for (std::vector<HTTPFetchOngoing*>::iterator
+ it = m_all_ongoing.begin();
it != m_all_ongoing.end();) {
if ((*it)->getRequest().caller == caller) {
delete (*it);
@@ -485,15 +509,16 @@ protected:
}
// Also abort all queued fetches for the caller
- for (std::list<HTTPFetchRequest>::iterator it =
- m_queued_fetches.begin();
+ for (std::list<HTTPFetchRequest>::iterator
+ it = m_queued_fetches.begin();
it != m_queued_fetches.end();) {
if ((*it).caller == caller)
it = m_queued_fetches.erase(it);
else
++it;
}
- } else if (req.type == RT_WAKEUP) {
+ }
+ else if (req.type == RT_WAKEUP) {
// Wakeup: Nothing to do, thread is awake at this point
}
@@ -511,13 +536,15 @@ protected:
// Create ongoing fetch data and make a cURL handle
// Set cURL options based on HTTPFetchRequest
- HTTPFetchOngoing *ongoing = new HTTPFetchOngoing(request, pool);
+ HTTPFetchOngoing *ongoing =
+ new HTTPFetchOngoing(request, pool);
// Initiate the connection (curl_multi_add_handle)
CURLcode res = ongoing->start(m_multi);
if (res == CURLE_OK) {
m_all_ongoing.push_back(ongoing);
- } else {
+ }
+ else {
httpfetch_deliver_result(*ongoing->complete(res));
delete ongoing;
}
@@ -552,8 +579,8 @@ protected:
try {
Request req = m_requests.pop_front(timeout);
processRequest(req);
- } catch (ItemNotFoundException &e) {
}
+ catch (ItemNotFoundException &e) {}
}
}
@@ -572,18 +599,20 @@ protected:
FD_ZERO(&write_fd_set);
FD_ZERO(&exc_fd_set);
- mres = curl_multi_fdset(m_multi, &read_fd_set, &write_fd_set, &exc_fd_set,
- &max_fd);
+ mres = curl_multi_fdset(m_multi, &read_fd_set,
+ &write_fd_set, &exc_fd_set, &max_fd);
if (mres != CURLM_OK) {
- errorstream << "curl_multi_fdset"
- << " returned error code " << mres << std::endl;
+ errorstream<<"curl_multi_fdset"
+ <<" returned error code "<<mres
+ <<std::endl;
select_timeout = 0;
}
mres = curl_multi_timeout(m_multi, &select_timeout);
if (mres != CURLM_OK) {
- errorstream << "curl_multi_timeout"
- << " returned error code " << mres << std::endl;
+ errorstream<<"curl_multi_timeout"
+ <<" returned error code "<<mres
+ <<std::endl;
select_timeout = 0;
}
@@ -598,17 +627,19 @@ protected:
select_tv.tv_sec = select_timeout / 1000;
select_tv.tv_usec = (select_timeout % 1000) * 1000;
int retval = select(max_fd + 1, &read_fd_set,
- &write_fd_set, &exc_fd_set, &select_tv);
+ &write_fd_set, &exc_fd_set,
+ &select_tv);
if (retval == -1) {
-#ifdef _WIN32
- errorstream << "select returned error code "
- << WSAGetLastError() << std::endl;
-#else
- errorstream << "select returned error code "
- << errno << std::endl;
-#endif
+ #ifdef _WIN32
+ errorstream<<"select returned error code "
+ <<WSAGetLastError()<<std::endl;
+ #else
+ errorstream<<"select returned error code "
+ <<errno<<std::endl;
+ #endif
}
- } else {
+ }
+ else {
sleep_ms(select_timeout);
}
}
@@ -620,7 +651,7 @@ protected:
m_multi = curl_multi_init();
if (m_multi == NULL) {
- errorstream << "curl_multi_init returned NULL\n";
+ errorstream<<"curl_multi_init returned NULL\n";
return NULL;
}
@@ -651,14 +682,13 @@ protected:
/*
Handle completed async requests
*/
- if (still_ongoing < (int)m_all_ongoing.size()) {
+ if (still_ongoing < (int) m_all_ongoing.size()) {
CURLMsg *msg;
int msgs_in_queue;
msg = curl_multi_info_read(m_multi, &msgs_in_queue);
while (msg != NULL) {
processCurlMessage(msg);
- msg = curl_multi_info_read(
- m_multi, &msgs_in_queue);
+ msg = curl_multi_info_read(m_multi, &msgs_in_queue);
}
}
@@ -689,8 +719,9 @@ protected:
CURLMcode mres = curl_multi_cleanup(m_multi);
if (mres != CURLM_OK) {
- errorstream << "curl_multi_cleanup"
- << " returned error code " << mres << std::endl;
+ errorstream<<"curl_multi_cleanup"
+ <<" returned error code "<<mres
+ <<std::endl;
}
return NULL;
@@ -701,7 +732,8 @@ CurlFetchThread *g_httpfetch_thread = NULL;
void httpfetch_init(int parallel_limit)
{
- verbosestream << "httpfetch_init: parallel_limit=" << parallel_limit << std::endl;
+ verbosestream<<"httpfetch_init: parallel_limit="<<parallel_limit
+ <<std::endl;
CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT);
FATAL_ERROR_IF(res != CURLE_OK, "CURL init failed");
@@ -716,7 +748,7 @@ void httpfetch_init(int parallel_limit)
void httpfetch_cleanup()
{
- verbosestream << "httpfetch_cleanup: cleaning up" << std::endl;
+ verbosestream<<"httpfetch_cleanup: cleaning up"<<std::endl;
g_httpfetch_thread->stop();
g_httpfetch_thread->requestWakeUp();
@@ -744,7 +776,8 @@ static void httpfetch_request_clear(unsigned long caller)
}
}
-void httpfetch_sync(const HTTPFetchRequest &fetch_request, HTTPFetchResult &fetch_result)
+void httpfetch_sync(const HTTPFetchRequest &fetch_request,
+ HTTPFetchResult &fetch_result)
{
// Create ongoing fetch data and make a cURL handle
// Set cURL options based on HTTPFetchRequest
@@ -756,7 +789,7 @@ void httpfetch_sync(const HTTPFetchRequest &fetch_request, HTTPFetchResult &fetc
fetch_result = *ongoing.complete(res);
}
-#else // USE_CURL
+#else // USE_CURL
/*
USE_CURL is off:
@@ -775,7 +808,7 @@ void httpfetch_cleanup()
void httpfetch_async(const HTTPFetchRequest &fetch_request)
{
errorstream << "httpfetch_async: unable to fetch " << fetch_request.url
- << " because USE_CURL=0" << std::endl;
+ << " because USE_CURL=0" << std::endl;
HTTPFetchResult fetch_result(fetch_request); // sets succeeded = false etc.
httpfetch_deliver_result(fetch_result);
@@ -785,12 +818,13 @@ static void httpfetch_request_clear(unsigned long caller)
{
}
-void httpfetch_sync(const HTTPFetchRequest &fetch_request, HTTPFetchResult &fetch_result)
+void httpfetch_sync(const HTTPFetchRequest &fetch_request,
+ HTTPFetchResult &fetch_result)
{
errorstream << "httpfetch_sync: unable to fetch " << fetch_request.url
- << " because USE_CURL=0" << std::endl;
+ << " because USE_CURL=0" << std::endl;
fetch_result = HTTPFetchResult(fetch_request); // sets succeeded = false etc.
}
-#endif // USE_CURL
+#endif // USE_CURL
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-20 21:00:28 +0000