void DL_End( CURLcode res, CURLMcode resm )
{
CURLMsg *msg;
int msgs;
if( dl_verbose->integer == 0 && dl_showprogress->integer == 2 && !curlm )
Com_Printf( "\n" );
if( curlm )
{
// res = final download result
while( ( msg = curl_multi_info_read( curlm, &msgs ) ) )
{
if( msg->msg != CURLMSG_DONE )
{
if( dl_error[0] == '\0' )
Q_strncpyz( dl_error, "Download Interrupted.", sizeof(dl_error) );
}
else if( msg->easy_handle == curl )
{
if( msg->data.result != CURLE_OK );
res = msg->data.result;
}
else
{
Com_Printf( "Invalid cURL handle.\n" );
}
}
curl_multi_cleanup( curlm );
curlm = NULL;
}
if( curl )
{
curl_easy_cleanup( curl );
curl = NULL;
}
// get possible error messages
if( !*dl_error && res != CURLE_OK )
Q_strncpyz( dl_error, curl_easy_strerror(res), sizeof(dl_error) );
if( !*dl_error && resm != CURLM_OK )
Q_strncpyz( dl_error, curl_multi_strerror(resm), sizeof(dl_error) );
if( !*dl_error && !f )
Q_strncpyz( dl_error, "File is not opened.", sizeof(dl_error) );
if (f) {
FS_FCloseFile(f);
f = 0;
if (!*dl_error) { // download succeeded
char dest[MAX_OSPATH];
Com_Printf("Download complete, restarting filesystem.\n");
Q_strncpyz(dest, path, strlen(path)-3); // -4 +1 for the trailing \0
Q_strcat(dest, sizeof(dest), ".pk3");
if (!FS_FileExists(dest)) {
FS_SV_Rename(path, dest);
FS_Restart(clc.checksumFeed);
if (dl_showmotd->integer && *motd) {
Com_Printf("Server motd: %s\n", motd);
}
} else {
// normally such errors should be caught upon starting the transfer. Anyway better do
// it here again - the filesystem might have changed, plus this may help contain some
// bugs / exploitable flaws in the code.
Com_Printf("Failed to copy downloaded file to its location - file already exists.\n");
FS_HomeRemove(path);
}
} else {
FS_HomeRemove(path);
}
}
Cvar_Set( "cl_downloadName", "" ); // hide the ui downloading screen
Cvar_SetValue( "cl_downloadSize", 0 );
Cvar_SetValue( "cl_downloadCount", 0 );
Cvar_SetValue( "cl_downloadTime", 0 );
Cvar_Set( "cl_downloadMotd", "" );
if( *dl_error )
{
if( clc.state == CA_CONNECTED )
Com_Error( ERR_DROP, "%s\n", dl_error ); // download error while connecting, can not continue loading
else
Com_Printf( "%s\n", dl_error ); // download error while in game, do not disconnect
*dl_error = '\0';
}
else
{
if (strlen(Cvar_VariableString("cl_downloadDemo"))) {
Cbuf_AddText( va("demo %s\n", Cvar_VariableString("cl_downloadDemo") ) );
// download completed, request new gamestate to check possible new map if we are not already in game
} else if( clc.state == CA_CONNECTED)
CL_AddReliableCommand( "donedl", qfalse); // get new gamestate info from server
}
}
const string curl_multi::error_to_string(const CURLMcode code) const noexcept {
return curl_multi_strerror(code);
}
void
MultiAdapter::checkCurlStatus(CURLMcode theStatusCode, const std::string & theWhere) {
if (theStatusCode != CURLM_OK) {
throw asl::Exception(curl_multi_strerror(theStatusCode), theWhere);
}
};
enum async_ret_code resume_async_http_req(int fd, struct sip_msg *msg, void *_param)
{
CURLcode rc;
CURLMcode mrc;
rest_async_param *param = (rest_async_param *)_param;
int running = 0, max_fd;
long http_rc;
fd_set rset, wset, eset;
pv_value_t val;
int ret = 1, retr;
CURLM *multi_handle;
multi_handle = param->multi_list->multi_handle;
retr = 0;
do {
mrc = curl_multi_perform(multi_handle, &running);
if (mrc != CURLM_CALL_MULTI_PERFORM)
break;
LM_DBG("retry last perform...\n");
usleep(_async_resume_retr_itv);
retr += _async_resume_retr_itv;
} while (retr < _async_resume_retr_timeout);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_perform: %s\n", curl_multi_strerror(mrc));
return -1;
}
LM_DBG("running handles: %d\n", running);
if (running == 1) {
LM_DBG("transfer in progress...\n");
async_status = ASYNC_CONTINUE;
return 1;
}
if (running != 0) {
LM_BUG("non-zero running handles!! (%d)", running);
abort();
}
FD_ZERO(&rset);
mrc = curl_multi_fdset(multi_handle, &rset, &wset, &eset, &max_fd);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_fdset: %s\n", curl_multi_strerror(mrc));
ret = -1;
goto out;
}
if (max_fd == -1) {
if (FD_ISSET(fd, &rset)) {
LM_BUG("fd %d is still in rset!", fd);
abort();
}
} else if (FD_ISSET(fd, &rset)) {
LM_DBG("fd %d still transferring...\n", fd);
async_status = ASYNC_CONTINUE;
return 1;
}
curl_slist_free_all(param->header_list);
if (del_transfer(fd) != 0) {
LM_BUG("failed to delete fd %d", fd);
abort();
}
mrc = curl_multi_remove_handle(multi_handle, param->handle);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_remove_handle: %s\n", curl_multi_strerror(mrc));
/* default async status is ASYNC_DONE */
return -1;
}
put_multi(param->multi_list);
val.flags = PV_VAL_STR;
val.rs = param->body;
if (pv_set_value(msg, param->body_pv, 0, &val) != 0)
LM_ERR("failed to set output body pv\n");
if (param->ctype_pv) {
val.rs = param->ctype;
if (pv_set_value(msg, param->ctype_pv, 0, &val) != 0)
LM_ERR("failed to set output ctype pv\n");
}
if (param->code_pv) {
rc = curl_easy_getinfo(param->handle, CURLINFO_RESPONSE_CODE, &http_rc);
if (rc != CURLE_OK) {
LM_ERR("curl_easy_getinfo: %s\n", curl_easy_strerror(rc));
http_rc = 0;
}
LM_DBG("Last response code: %ld\n", http_rc);
val.flags = PV_VAL_INT|PV_TYPE_INT;
val.ri = (int)http_rc;
if (pv_set_value(msg, param->code_pv, 0, &val) != 0)
LM_ERR("failed to set output code pv\n");
}
out:
pkg_free(param->body.s);
if (param->ctype_pv && param->ctype.s)
pkg_free(param->ctype.s);
curl_easy_cleanup(param->handle);
if ( param->tparam ) {
pkg_free( param->tparam );
}
pkg_free(param);
/* default async status is ASYNC_DONE */
return ret;
}
std::string curlm_category_impl::message(int ev) const
{
return curl_multi_strerror(static_cast<CURLMcode>(ev));
}
/**
* Wait for the libcurl socket.
*
* @return -1 on error, 0 if no data is available yet, 1 if data is
* available
*/
static int
input_curl_select(struct input_curl *c, GError **error_r)
{
fd_set rfds, wfds, efds;
int max_fd, ret;
CURLMcode mcode;
struct timeval timeout = {
.tv_sec = 1,
.tv_usec = 0,
};
assert(!c->eof);
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
mcode = curl_multi_fdset(c->multi, &rfds, &wfds, &efds, &max_fd);
if (mcode != CURLM_OK) {
g_set_error(error_r, curl_quark(), mcode,
"curl_multi_fdset() failed: %s",
curl_multi_strerror(mcode));
return -1;
}
#if LIBCURL_VERSION_NUM >= 0x070f04
long timeout2;
mcode = curl_multi_timeout(c->multi, &timeout2);
if (mcode != CURLM_OK) {
g_warning("curl_multi_timeout() failed: %s\n",
curl_multi_strerror(mcode));
return -1;
}
if (timeout2 >= 0) {
if (timeout2 > 10000)
timeout2 = 10000;
timeout.tv_sec = timeout2 / 1000;
timeout.tv_usec = (timeout2 % 1000) * 1000;
}
#endif
ret = select(max_fd + 1, &rfds, &wfds, &efds, &timeout);
if (ret < 0)
g_set_error(error_r, g_quark_from_static_string("errno"),
errno,
"select() failed: %s\n", g_strerror(errno));
return ret;
}
static bool
fill_buffer(struct input_stream *is, GError **error_r)
{
struct input_curl *c = (struct input_curl *)is;
CURLMcode mcode = CURLM_CALL_MULTI_PERFORM;
while (!c->eof && g_queue_is_empty(c->buffers)) {
int running_handles;
bool bret;
if (mcode != CURLM_CALL_MULTI_PERFORM) {
/* if we're still here, there is no input yet
- wait for input */
int ret = input_curl_select(c, error_r);
if (ret <= 0)
/* no data yet or error */
return false;
}
mcode = curl_multi_perform(c->multi, &running_handles);
if (mcode != CURLM_OK && mcode != CURLM_CALL_MULTI_PERFORM) {
g_set_error(error_r, curl_quark(), mcode,
"curl_multi_perform() failed: %s",
curl_multi_strerror(mcode));
c->eof = true;
is->ready = true;
return false;
}
bret = input_curl_multi_info_read(c, error_r);
if (!bret)
return false;
}
return !g_queue_is_empty(c->buffers);
}
/**
* Mark a part of the buffer object as consumed.
*/
static struct buffer *
consume_buffer(struct buffer *buffer, size_t length)
{
assert(buffer != NULL);
assert(buffer->consumed < buffer->size);
buffer->consumed += length;
if (buffer->consumed < buffer->size)
return buffer;
assert(buffer->consumed == buffer->size);
g_free(buffer);
return NULL;
}
void handleError(CURLMcode code) {
if (code != CURLM_OK) {
throw std::runtime_error(std::string("CURL multi error: ") + curl_multi_strerror(code));
}
}
static int performN(KSI_NetworkClient *client, KSI_RequestHandle **arr, size_t arr_len) {
int res = KSI_UNKNOWN_ERROR;
CURLM *cm = NULL;
CURLMcode cres;
size_t i;
int count;
char buf[1024];
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = -1;
struct timeval timeout;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
if (client == NULL || (arr == NULL && arr_len != 0)) {
res = KSI_INVALID_ARGUMENT;
goto cleanup;
}
KSI_ERR_clearErrors(client->ctx);
KSI_LOG_debug(client->ctx, "Starting cURL multi perform.");
timeout.tv_sec = 0;
timeout.tv_usec = 100;
cm = curl_multi_init();
if (cm == NULL) {
KSI_pushError(client->ctx, res = KSI_OUT_OF_MEMORY, NULL);
goto cleanup;
}
for (i = 0; i < arr_len; i++) {
CurlNetHandleCtx *pctx = arr[i]->implCtx;
cres = curl_multi_add_handle(cm, pctx->curl);
if (cres != CURLM_OK) {
KSI_snprintf(buf, sizeof(buf), "Curl error occurred: %s", curl_multi_strerror(cres));
KSI_pushError(client->ctx, res = KSI_UNKNOWN_ERROR, buf);
goto cleanup;
}
}
curl_multi_setopt(cm, CURLMOPT_PIPELINING, 1);
do {
cres = curl_multi_fdset(cm, &fdread, &fdwrite, &fdexcep, &maxfd);
if (cres != CURLM_OK) {
KSI_snprintf(buf, sizeof(buf), "Curl error occurred: %s", curl_multi_strerror(cres));
KSI_pushError(client->ctx, res = KSI_UNKNOWN_ERROR, buf);
goto cleanup;
}
select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
cres = curl_multi_perform(cm, &count);
if (cres != CURLM_OK && cres != CURLM_CALL_MULTI_PERFORM) {
KSI_snprintf(buf, sizeof(buf), "Curl error occurred: %s", curl_multi_strerror(cres));
KSI_pushError(client->ctx, res = KSI_UNKNOWN_ERROR, buf);
goto cleanup;
}
} while (count > 0 || cres == CURLM_CALL_MULTI_PERFORM);
/* Remove the handles from the multi container. */
for (i = 0; i < arr_len; i++) {
CurlNetHandleCtx *pctx = arr[i]->implCtx;
cres = curl_multi_remove_handle(cm, pctx->curl);
if (cres != CURLM_OK) {
KSI_snprintf(buf, sizeof(buf), "Curl error occurred: %s", curl_multi_strerror(cres));
KSI_pushError(client->ctx, res = KSI_UNKNOWN_ERROR, buf);
goto cleanup;
}
arr[i]->response = pctx->raw;
pctx->raw = NULL;
arr[i]->response_length = pctx->len;
arr[i]->completed = true;
res = updateStatus(arr[i]);
if (res != KSI_OK) goto cleanup;
}
KSI_LOG_debug(client->ctx, "Finished cURL multi perform.");
res = KSI_OK;
cleanup:
if (cm != NULL) curl_multi_cleanup(cm);
return res;
}
static void
addTask( void * vtask )
{
struct tr_web_task * task = vtask;
const tr_handle * session = task->session;
if( session && session->web )
{
struct tr_web * web = session->web;
CURL * easy;
dbgmsg( "adding task #%lu [%s]", task->tag, task->url );
easy = curl_easy_init( );
if( !task->range && session->isProxyEnabled ) {
curl_easy_setopt( easy, CURLOPT_PROXY, session->proxy );
curl_easy_setopt( easy, CURLOPT_PROXYAUTH, CURLAUTH_ANY );
curl_easy_setopt( easy, CURLOPT_PROXYPORT, session->proxyPort );
curl_easy_setopt( easy, CURLOPT_PROXYTYPE,
getCurlProxyType( session->proxyType ) );
}
if( !task->range && session->isProxyAuthEnabled ) {
char * str = tr_strdup_printf( "%s:%s", session->proxyUsername,
session->proxyPassword );
curl_easy_setopt( easy, CURLOPT_PROXYUSERPWD, str );
tr_free( str );
}
curl_easy_setopt( easy, CURLOPT_DNS_CACHE_TIMEOUT, 360L );
curl_easy_setopt( easy, CURLOPT_CONNECTTIMEOUT, 60L );
curl_easy_setopt( easy, CURLOPT_FOLLOWLOCATION, 1L );
curl_easy_setopt( easy, CURLOPT_MAXREDIRS, 16L );
curl_easy_setopt( easy, CURLOPT_NOSIGNAL, 1L );
curl_easy_setopt( easy, CURLOPT_PRIVATE, task );
curl_easy_setopt( easy, CURLOPT_SSL_VERIFYHOST, 0L );
curl_easy_setopt( easy, CURLOPT_SSL_VERIFYPEER, 0L );
curl_easy_setopt( easy, CURLOPT_URL, task->url );
curl_easy_setopt( easy, CURLOPT_USERAGENT,
TR_NAME "/" LONG_VERSION_STRING );
curl_easy_setopt( easy, CURLOPT_VERBOSE,
getenv( "TR_CURL_VERBOSE" ) != NULL );
curl_easy_setopt( easy, CURLOPT_WRITEDATA, task );
curl_easy_setopt( easy, CURLOPT_WRITEFUNCTION, writeFunc );
if( task->range )
curl_easy_setopt( easy, CURLOPT_RANGE, task->range );
else /* don't set encoding on webseeds; it messes up binary data */
curl_easy_setopt( easy, CURLOPT_ENCODING, "" );
if( web->still_running >= MAX_CONCURRENT_TASKS ) {
tr_list_append( &web->easy_queue, easy );
dbgmsg( " >> enqueueing a task ... size is now %d",
tr_list_size( web->easy_queue ) );
} else {
const CURLMcode rc = curl_multi_add_handle( web->multi, easy );
if( rc == CURLM_OK )
++web->still_running;
else
tr_err( "%s", curl_multi_strerror( rc ) );
}
}
}
static void
gotstatus(int nnrun)
{
CURLMsg *msg;
int nmsg;
global.nrun = nnrun;
CURLM_syslog(LOG_DEBUG,
"gotstatus(): about to call curl_multi_info_read\n");
while ((msg = curl_multi_info_read(global.mhnd, &nmsg))) {
CURL_syslog(LOG_DEBUG,
"got curl multi_info message msg=%d\n",
msg->msg);
if (CURLMSG_DONE == msg->msg) {
CURL *chnd;
void *chandle = NULL;
CURLcode sta;
CURLMcode msta;
AsyncIO*IO;
chandle = NULL;;
chnd = msg->easy_handle;
sta = curl_easy_getinfo(chnd,
CURLINFO_PRIVATE,
&chandle);
if (sta) {
syslog(LOG_ERR,
"error asking curl for private"
" cookie of curl handle: %s\n",
curl_easy_strerror(sta));
continue;
}
IO = (AsyncIO *)chandle;
if (IO->ID == 0) {
EVCURL_syslog(LOG_ERR,
"Error, invalid IO context %p\n",
IO);
continue;
}
SetEVState(IO, eCurlGotStatus);
EVCURLM_syslog(LOG_DEBUG, "request complete\n");
IO->CitContext->lastcmd = IO->Now = ev_now(event_base);
ev_io_stop(event_base, &IO->recv_event);
ev_io_stop(event_base, &IO->send_event);
sta = msg->data.result;
if (sta) {
EVCURL_syslog(LOG_ERR,
"error description: %s\n",
IO->HttpReq.errdesc);
IO->HttpReq.CurlError = curl_easy_strerror(sta);
EVCURL_syslog(LOG_ERR,
"error performing request: %s\n",
IO->HttpReq.CurlError);
if (sta == CURLE_OPERATION_TIMEDOUT)
{
IO->SendBuf.fd = 0;
IO->RecvBuf.fd = 0;
}
}
sta = curl_easy_getinfo(chnd,
CURLINFO_RESPONSE_CODE,
&IO->HttpReq.httpcode);
if (sta)
EVCURL_syslog(LOG_ERR,
"error asking curl for "
"response code from request: %s\n",
curl_easy_strerror(sta));
EVCURL_syslog(LOG_DEBUG,
"http response code was %ld\n",
(long)IO->HttpReq.httpcode);
curl_slist_free_all(IO->HttpReq.headers);
IO->HttpReq.headers = NULL;
msta = curl_multi_remove_handle(global.mhnd, chnd);
if (msta)
EVCURL_syslog(LOG_ERR,
"warning problem detaching "
"completed handle from curl multi: "
"%s\n",
curl_multi_strerror(msta));
ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
IO->HttpReq.attached = 0;
switch(IO->SendDone(IO))
{
case eDBQuery:
FreeURL(&IO->ConnectMe);
QueueAnDBOperation(IO);
break;
case eSendDNSQuery:
case eReadDNSReply:
case eConnect:
case eSendReply:
case eSendMore:
case eSendFile:
case eReadMessage:
case eReadMore:
case eReadPayload:
case eReadFile:
break;
case eTerminateConnection:
case eAbort:
curl_easy_cleanup(IO->HttpReq.chnd);
IO->HttpReq.chnd = NULL;
FreeStrBuf(&IO->HttpReq.ReplyData);
FreeURL(&IO->ConnectMe);
RemoveContext(IO->CitContext);
IO->Terminate(IO);
}
}
}
}
/**
* Do some work on current fetches.
*
* Must be called regularly to make progress on fetches.
*/
static void fetch_curl_poll(lwc_string *scheme_ignored)
{
int running, queue;
CURLMcode codem;
CURLMsg *curl_msg;
if (nsoption_bool(suppress_curl_debug) == false) {
fd_set read_fd_set, write_fd_set, exc_fd_set;
int max_fd = -1;
int i;
FD_ZERO(&read_fd_set);
FD_ZERO(&write_fd_set);
FD_ZERO(&exc_fd_set);
codem = curl_multi_fdset(fetch_curl_multi,
&read_fd_set, &write_fd_set,
&exc_fd_set, &max_fd);
assert(codem == CURLM_OK);
LOG("Curl file descriptor states (maxfd=%i):", max_fd);
for (i = 0; i <= max_fd; i++) {
bool read = false;
bool write = false;
bool error = false;
if (FD_ISSET(i, &read_fd_set)) {
read = true;
}
if (FD_ISSET(i, &write_fd_set)) {
write = true;
}
if (FD_ISSET(i, &exc_fd_set)) {
error = true;
}
if (read || write || error) {
LOG(" fd %i: %s %s %s", i,
read ? "read" : " ",
write ? "write" : " ",
error ? "error" : " ");
}
}
}
/* do any possible work on the current fetches */
do {
codem = curl_multi_perform(fetch_curl_multi, &running);
if (codem != CURLM_OK && codem != CURLM_CALL_MULTI_PERFORM) {
LOG("curl_multi_perform: %i %s", codem, curl_multi_strerror(codem));
guit->misc->warning("MiscError", curl_multi_strerror(codem));
return;
}
} while (codem == CURLM_CALL_MULTI_PERFORM);
/* process curl results */
curl_msg = curl_multi_info_read(fetch_curl_multi, &queue);
while (curl_msg) {
switch (curl_msg->msg) {
case CURLMSG_DONE:
fetch_curl_done(curl_msg->easy_handle,
curl_msg->data.result);
break;
default:
break;
}
curl_msg = curl_multi_info_read(fetch_curl_multi, &queue);
}
}
static void
handle_transfer (GstCurlBaseSink * sink)
{
GstCurlBaseSinkClass *klass = GST_CURL_BASE_SINK_GET_CLASS (sink);
gint retval;
gint activated_fds;
gint running_handles;
gint timeout;
CURLMcode m_code;
CURLcode e_code;
GST_OBJECT_LOCK (sink);
timeout = sink->timeout;
GST_OBJECT_UNLOCK (sink);
GST_DEBUG_OBJECT (sink, "handling transfers");
/* Receiving CURLM_CALL_MULTI_PERFORM means that libcurl may have more data
available to send or receive - call simply curl_multi_perform before
poll() on more actions */
do {
m_code = curl_multi_perform (sink->multi_handle, &running_handles);
} while (m_code == CURLM_CALL_MULTI_PERFORM);
GST_DEBUG_OBJECT (sink, "running handles: %d", running_handles);
while (running_handles && (m_code == CURLM_OK)) {
if (klass->transfer_prepare_poll_wait) {
klass->transfer_prepare_poll_wait (sink);
}
activated_fds = gst_poll_wait (sink->fdset, timeout * GST_SECOND);
if (G_UNLIKELY (activated_fds == -1)) {
if (errno == EAGAIN || errno == EINTR) {
GST_DEBUG_OBJECT (sink, "interrupted by signal");
} else if (errno == EBUSY) {
GST_DEBUG_OBJECT (sink, "poll stopped");
retval = GST_FLOW_EOS;
GST_OBJECT_LOCK (sink);
if (gst_curl_base_sink_has_buffered_data_unlocked (sink))
GST_WARNING_OBJECT (sink,
"discarding render data due to thread close flag");
GST_OBJECT_UNLOCK (sink);
goto fail;
} else {
sink->error = g_strdup_printf ("poll failed: %s", g_strerror (errno));
retval = GST_FLOW_ERROR;
goto fail;
}
} else if (G_UNLIKELY (activated_fds == 0)) {
sink->error = g_strdup_printf ("poll timed out after %" GST_TIME_FORMAT,
GST_TIME_ARGS (timeout * GST_SECOND));
retval = GST_FLOW_ERROR;
goto fail;
}
/* readable/writable sockets */
do {
m_code = curl_multi_perform (sink->multi_handle, &running_handles);
} while (m_code == CURLM_CALL_MULTI_PERFORM);
GST_DEBUG_OBJECT (sink, "running handles: %d", running_handles);
}
if (m_code != CURLM_OK) {
sink->error = g_strdup_printf ("failed to write data: %s",
curl_multi_strerror (m_code));
retval = GST_FLOW_ERROR;
goto fail;
}
/* problems still might have occurred on individual transfers even when
* curl_multi_perform returns CURLM_OK */
if ((e_code = gst_curl_base_sink_transfer_check (sink)) != CURLE_OK) {
sink->error = g_strdup_printf ("failed to transfer data: %s",
curl_easy_strerror (e_code));
retval = GST_FLOW_ERROR;
goto fail;
}
gst_curl_base_sink_got_response_notify (sink);
GST_OBJECT_LOCK (sink);
if (sink->socket_type == CURLSOCKTYPE_ACCEPT) {
/* FIXME: remove this again once we can depend on libcurl > 7.44.0,
* see https://github.com/bagder/curl/issues/405.
*/
if (G_UNLIKELY (sink->fd.fd < 0)) {
sink->error = g_strdup_printf ("unknown error");
retval = GST_FLOW_ERROR;
GST_OBJECT_UNLOCK (sink);
goto fail;
}
if (!gst_poll_remove_fd (sink->fdset, &sink->fd)) {
sink->error = g_strdup_printf ("failed to remove fd");
retval = GST_FLOW_ERROR;
GST_OBJECT_UNLOCK (sink);
goto fail;
}
sink->fd.fd = -1;
}
GST_OBJECT_UNLOCK (sink);
return;
fail:
GST_OBJECT_LOCK (sink);
if (sink->flow_ret == GST_FLOW_OK) {
sink->flow_ret = retval;
}
GST_OBJECT_UNLOCK (sink);
return;
}
static void
handle_transfer (GstCurlBaseSink * sink)
{
GstCurlBaseSinkClass *klass = GST_CURL_BASE_SINK_GET_CLASS (sink);
gint retval;
gint activated_fds;
gint running_handles;
gint timeout;
CURLMcode m_code;
CURLcode e_code;
GST_OBJECT_LOCK (sink);
timeout = sink->timeout;
GST_OBJECT_UNLOCK (sink);
/* Receiving CURLM_CALL_MULTI_PERFORM means that libcurl may have more data
available to send or receive - call simply curl_multi_perform before
poll() on more actions */
do {
m_code = curl_multi_perform (sink->multi_handle, &running_handles);
} while (m_code == CURLM_CALL_MULTI_PERFORM);
while (running_handles && (m_code == CURLM_OK)) {
if (klass->transfer_prepare_poll_wait) {
klass->transfer_prepare_poll_wait (sink);
}
activated_fds = gst_poll_wait (sink->fdset, timeout * GST_SECOND);
if (G_UNLIKELY (activated_fds == -1)) {
if (errno == EAGAIN || errno == EINTR) {
GST_DEBUG_OBJECT (sink, "interrupted by signal");
} else if (errno == EBUSY) {
GST_DEBUG_OBJECT (sink, "poll stopped");
retval = GST_FLOW_EOS;
goto fail;
} else {
GST_DEBUG_OBJECT (sink, "poll failed: %s", g_strerror (errno));
GST_ELEMENT_ERROR (sink, RESOURCE, WRITE, ("poll failed"), (NULL));
retval = GST_FLOW_ERROR;
goto fail;
}
} else if (G_UNLIKELY (activated_fds == 0)) {
GST_DEBUG_OBJECT (sink, "poll timed out");
GST_ELEMENT_ERROR (sink, RESOURCE, WRITE, ("poll timed out"), (NULL));
retval = GST_FLOW_ERROR;
goto fail;
}
/* readable/writable sockets */
do {
m_code = curl_multi_perform (sink->multi_handle, &running_handles);
} while (m_code == CURLM_CALL_MULTI_PERFORM);
}
if (m_code != CURLM_OK) {
GST_DEBUG_OBJECT (sink, "curl multi error");
GST_ELEMENT_ERROR (sink, RESOURCE, WRITE, ("%s",
curl_multi_strerror (m_code)), (NULL));
retval = GST_FLOW_ERROR;
goto fail;
}
/* problems still might have occurred on individual transfers even when
* curl_multi_perform returns CURLM_OK */
if ((e_code = gst_curl_base_sink_transfer_check (sink)) != CURLE_OK) {
GST_DEBUG_OBJECT (sink, "curl easy error");
GST_ELEMENT_ERROR (sink, RESOURCE, WRITE, ("%s",
curl_easy_strerror (e_code)), (NULL));
retval = GST_FLOW_ERROR;
goto fail;
}
gst_curl_base_sink_got_response_notify (sink);
return;
fail:
GST_OBJECT_LOCK (sink);
if (sink->flow_ret == GST_FLOW_OK) {
sink->flow_ret = retval;
}
GST_OBJECT_UNLOCK (sink);
return;
}
extern "C" const char* HttpNative_MultiGetErrorString(PAL_CURLMcode code)
{
return curl_multi_strerror(static_cast<CURLMcode>(code));
}
// Actually starts a download by adding it to the curl multi handle.
static void start_download(dlqueue_t *entry, dlhandle_t *dl)
{
size_t len;
char temp[MAX_QPATH];
char escaped[MAX_QPATH * 4];
CURLMcode ret;
qerror_t err;
//yet another hack to accomodate filelists, how i wish i could push :(
//NULL file handle indicates filelist.
if (entry->type == DL_LIST) {
dl->file = NULL;
dl->path[0] = 0;
//filelist paths are absolute
escape_path(entry->path, escaped);
} else {
len = Q_snprintf(dl->path, sizeof(dl->path), "%s/%s.tmp", fs_gamedir, entry->path);
if (len >= sizeof(dl->path)) {
Com_EPrintf("[HTTP] Refusing oversize temporary file path.\n");
goto fail;
}
//prepend quake path with gamedir
len = Q_snprintf(temp, sizeof(temp), "%s/%s", http_gamedir(), entry->path);
if (len >= sizeof(temp)) {
Com_EPrintf("[HTTP] Refusing oversize server file path.\n");
goto fail;
}
escape_path(temp, escaped);
err = FS_CreatePath(dl->path);
if (err < 0) {
Com_EPrintf("[HTTP] Couldn't create path to '%s': %s\n", dl->path, Q_ErrorString(err));
goto fail;
}
//don't bother with http resume... too annoying if server doesn't support it.
dl->file = fopen(dl->path, "wb");
if (!dl->file) {
Com_EPrintf("[HTTP] Couldn't open '%s' for writing: %s\n", dl->path, strerror(errno));
goto fail;
}
}
len = Q_snprintf(dl->url, sizeof(dl->url), "%s%s", download_server, escaped);
if (len >= sizeof(dl->url)) {
Com_EPrintf("[HTTP] Refusing oversize download URL.\n");
goto fail;
}
dl->buffer = NULL;
dl->size = 0;
dl->position = 0;
dl->queue = entry;
if (!dl->curl)
dl->curl = curl_easy_init();
curl_easy_setopt(dl->curl, CURLOPT_ENCODING, "");
#ifdef _DEBUG
if (cl_http_debug->integer) {
curl_easy_setopt(dl->curl, CURLOPT_DEBUGFUNCTION, debug_func);
curl_easy_setopt(dl->curl, CURLOPT_VERBOSE, 1);
}
#endif
curl_easy_setopt(dl->curl, CURLOPT_NOPROGRESS, 0);
if (dl->file) {
curl_easy_setopt(dl->curl, CURLOPT_WRITEDATA, dl->file);
curl_easy_setopt(dl->curl, CURLOPT_WRITEFUNCTION, NULL);
} else {
curl_easy_setopt(dl->curl, CURLOPT_WRITEDATA, dl);
curl_easy_setopt(dl->curl, CURLOPT_WRITEFUNCTION, recv_func);
}
curl_easy_setopt(dl->curl, CURLOPT_FAILONERROR, 1);
curl_easy_setopt(dl->curl, CURLOPT_PROXY, cl_http_proxy->string);
curl_easy_setopt(dl->curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(dl->curl, CURLOPT_MAXREDIRS, 5);
curl_easy_setopt(dl->curl, CURLOPT_PROGRESSFUNCTION, progress_func);
curl_easy_setopt(dl->curl, CURLOPT_PROGRESSDATA, dl);
curl_easy_setopt(dl->curl, CURLOPT_USERAGENT, com_version->string);
curl_easy_setopt(dl->curl, CURLOPT_REFERER, download_referer);
curl_easy_setopt(dl->curl, CURLOPT_URL, dl->url);
ret = curl_multi_add_handle(curl_multi, dl->curl);
if (ret != CURLM_OK) {
Com_EPrintf("[HTTP] Failed to add download handle: %s\n",
curl_multi_strerror(ret));
fail:
CL_FinishDownload(entry);
// see if we have more to dl
CL_RequestNextDownload();
return;
}
Com_DPrintf("[HTTP] Fetching %s...\n", dl->url);
entry->state = DL_RUNNING;
curl_handles++;
}
static void *
http_client_main( void *args ) {
UNUSED( args );
debug( "Starting HTTP client thread ( tid = %#lx ).", pthread_self() );
init_curl();
while ( 1 ) {
int n_msgs = 0;
CURLMsg *msg = curl_multi_info_read( curl_handle, &n_msgs );
while ( msg != NULL ) {
if ( msg->msg != CURLMSG_DONE ) {
continue;
}
http_transaction *transaction = lookup_http_transaction( msg->easy_handle );
if ( transaction != NULL ) {
send_http_transaction_to_main( transaction, msg );
}
else {
warn( "Failed to find HTTP transaction record ( easy_handle = %p ).", msg->easy_handle );
}
msg = curl_multi_info_read( curl_handle, &n_msgs );
}
fd_set fd_read;
FD_ZERO( &fd_read );
fd_set fd_write;
FD_ZERO( &fd_write );
fd_set fd_excep;
FD_ZERO( &fd_excep );
int fd_max = -1;
long curl_timeout = -1;
struct timespec timeout = { 10, 0 };
if ( curl_running ) {
CURLMcode retval = curl_multi_timeout( curl_handle, &curl_timeout );
if ( retval != CURLM_OK && retval != CURLM_CALL_MULTI_PERFORM ) {
error( "Failed to get timeout value from a handle ( handle = %p, error = %s ).", curl_handle, curl_multi_strerror( retval ) );
}
if ( curl_timeout >= 0 ) {
timeout.tv_sec = curl_timeout / 1000;
if ( timeout.tv_sec > 1 ) {
timeout.tv_sec = 1;
}
else {
timeout.tv_nsec = ( curl_timeout % 1000 ) * 1000000;
}
}
retval = curl_multi_fdset( curl_handle, &fd_read, &fd_write, &fd_excep, &fd_max );
if ( retval != CURLM_OK && retval != CURLM_CALL_MULTI_PERFORM ) {
error( "Failed to retrieve fd set from a handle ( handle = %p, error = %s ).", curl_handle, curl_multi_strerror( retval ) );
}
}
if ( http_client_efd >= 0 ) {
#define sizeof( _x ) ( ( int ) sizeof( _x ) )
FD_SET( http_client_efd, &fd_read );
#undef sizeof
if ( http_client_efd > fd_max ) {
fd_max = http_client_efd;
}
}
if ( main_efd >= 0 && main_notify_count > 0 ) {
#define sizeof( _x ) ( ( int ) sizeof( _x ) )
FD_SET( main_efd, &fd_write );
#undef sizeof
if ( main_efd > fd_max ) {
fd_max = main_efd;
}
}
int ret = pselect( fd_max + 1, &fd_read, &fd_write, &fd_excep, &timeout, NULL );
switch ( ret ) {
case -1:
{
if ( errno == EINTR ) {
continue;
}
char buf[ 256 ];
memset( buf, '\0', sizeof( buf ) );
char *error_string = strerror_r( errno, buf, sizeof( buf ) - 1 );
warn( "Failed to select ( errno = %s [%d] ).", error_string, errno );
}
break;
case 0:
default:
{
if ( curl_running ) {
CURLMcode retval = curl_multi_perform( curl_handle, &curl_running );
if ( retval != CURLM_OK && retval != CURLM_CALL_MULTI_PERFORM ) {
error( "Failed to run a multi handle ( handle = %p, error = %s ).", curl_handle, curl_multi_strerror( retval ) );
}
}
#define sizeof( _x ) ( ( int ) sizeof( _x ) )
if ( http_client_efd >= 0 && FD_ISSET( http_client_efd, &fd_read ) ) {
retrieve_http_transactions_from_main();
}
if ( main_efd >= 0 && FD_ISSET( main_efd, &fd_write ) ) {
notify_main_actually();
}
#undef sizeof
}
break;
}
}
finalize_curl();
debug( "Stopping HTTP client thread ( tid = %#lx ).", pthread_self() );
return NULL;
}
void Downloader::groupBatchDownload(const DownloadUnits &units)
{
CURLM* multi_handle = curl_multi_init();
int still_running = 0;
for (auto it = units.cbegin(); it != units.cend(); ++it)
{
DownloadUnit unit = it->second;
std::string srcUrl = unit.srcUrl;
std::string storagePath = unit.storagePath;
std::string customId = unit.customId;
FileDescriptor *fDesc = new FileDescriptor();
ProgressData *data = new ProgressData();
prepareDownload(srcUrl, storagePath, customId, unit.resumeDownload, fDesc, data);
if (fDesc->fp != NULL)
{
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, srcUrl.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fileWriteFunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fDesc->fp);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, batchDownloadProgressFunc);
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, data);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
if (_connectionTimeout) curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, _connectionTimeout);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, LOW_SPEED_LIMIT);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, LOW_SPEED_TIME);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, MAX_REDIRS);
// Resuming download support
if (_supportResuming && unit.resumeDownload)
{
// Check already downloaded size for current download unit
long size = _fileUtils->getFileSize(storagePath + TEMP_EXT);
if (size != -1)
{
curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, size);
}
}
fDesc->curl = curl;
CURLMcode code = curl_multi_add_handle(multi_handle, curl);
if (code != CURLM_OK)
{
// Avoid memory leak
fclose(fDesc->fp);
delete data;
delete fDesc;
std::string msg = StringUtils::format("Unable to add curl handler for %s: [curl error]%s", customId.c_str(), curl_multi_strerror(code));
this->notifyError(msg, code, customId);
}
else
{
// Add to list for tracking
_progDatas.push_back(data);
_files.push_back(fDesc);
}
}
}
// Query multi perform
CURLMcode curlm_code = CURLM_CALL_MULTI_PERFORM;
while(CURLM_CALL_MULTI_PERFORM == curlm_code) {
curlm_code = curl_multi_perform(multi_handle, &still_running);
}
if (curlm_code != CURLM_OK) {
std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code));
this->notifyError(msg, curlm_code);
}
else
{
bool failed = false;
while (still_running > 0 && !failed)
{
// set a suitable timeout to play around with
struct timeval select_tv;
long curl_timeo = -1;
select_tv.tv_sec = 1;
select_tv.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
select_tv.tv_sec = curl_timeo / 1000;
if(select_tv.tv_sec > 1)
select_tv.tv_sec = 1;
else
select_tv.tv_usec = (curl_timeo % 1000) * 1000;
}
int rc;
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
// FIXME: when jenkins migrate to ubuntu, we should remove this hack code
#if (CC_TARGET_PLATFORM == CC_PLATFORM_LINUX)
curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &select_tv);
#else
rc = curl_multi_wait(multi_handle, nullptr, 0, MAX_WAIT_MSECS, &maxfd);
#endif
switch(rc)
{
case -1:
failed = true;
break;
case 0:
default:
curlm_code = CURLM_CALL_MULTI_PERFORM;
while(CURLM_CALL_MULTI_PERFORM == curlm_code) {
curlm_code = curl_multi_perform(multi_handle, &still_running);
}
if (curlm_code != CURLM_OK) {
std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code));
this->notifyError(msg, curlm_code);
}
break;
}
}
}
// Clean up and close files
curl_multi_cleanup(multi_handle);
for (auto it = _files.begin(); it != _files.end(); ++it)
{
FILE *f = (*it)->fp;
fclose(f);
CURL *single = (CURL *)((*it)->curl);
curl_multi_remove_handle(multi_handle, single);
curl_easy_cleanup(single);
}
// Check unfinished files and notify errors, succeed files will be renamed from temporary file name to real name
for (auto it = _progDatas.begin(); it != _progDatas.end(); ++it) {
ProgressData *data = *it;
if (data->downloaded < data->totalToDownload || data->totalToDownload == 0)
{
this->notifyError(ErrorCode::NETWORK, "Unable to download file", data->customId);
}
else
{
_fileUtils->renameFile(data->path, data->name + TEMP_EXT, data->name);
}
}
clearBatchDownloadData();
}
/**
* start_async_http_req - performs an HTTP request, stores results in pvars
* - TCP connect phase is synchronous, due to libcurl limitations
* - TCP read phase is asynchronous, thanks to the libcurl multi interface
*
* @msg: sip message struct
* @method: HTTP verb
* @url: HTTP URL to be queried
* @req_body: Body of the request (NULL if not needed)
* @req_ctype: Value for the "Content-Type: " header of the request (same as ^)
* @out_handle: CURL easy handle used to perform the transfer
* @body: reply body; gradually reallocated as data arrives
* @ctype: will eventually hold the last "Content-Type" header of the reply
*/
int start_async_http_req(struct sip_msg *msg, enum rest_client_method method,
char *url, char *req_body, char *req_ctype,
CURL **out_handle, str *body, str *ctype)
{
CURL *handle;
CURLcode rc;
CURLMcode mrc;
fd_set rset, wset, eset;
int max_fd, fd, i;
long busy_wait, timeout;
long retry_time, check_time = 5; /* 5ms looping time */
int msgs_in_queue;
CURLMsg *cmsg;
if (transfers == FD_SETSIZE) {
LM_ERR("too many ongoing tranfers: %d\n", FD_SETSIZE);
clean_header_list;
return ASYNC_NO_IO;
}
handle = curl_easy_init();
if (!handle) {
LM_ERR("Init curl handle failed!\n");
clean_header_list;
return ASYNC_NO_IO;
}
w_curl_easy_setopt(handle, CURLOPT_URL, url);
switch (method) {
case REST_CLIENT_POST:
w_curl_easy_setopt(handle, CURLOPT_POST, 1);
w_curl_easy_setopt(handle, CURLOPT_POSTFIELDS, req_body);
if (req_ctype) {
sprintf(print_buff, "Content-Type: %s", req_ctype);
header_list = curl_slist_append(header_list, print_buff);
w_curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header_list);
}
break;
case REST_CLIENT_GET:
break;
default:
LM_ERR("Unsupported rest_client_method: %d, defaulting to GET\n", method);
}
if (header_list)
w_curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header_list);
w_curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, connection_timeout);
w_curl_easy_setopt(handle, CURLOPT_TIMEOUT, curl_timeout);
w_curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
w_curl_easy_setopt(handle, CURLOPT_FAILONERROR, 1);
w_curl_easy_setopt(handle, CURLOPT_STDERR, stdout);
w_curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_func);
w_curl_easy_setopt(handle, CURLOPT_WRITEDATA, body);
if (ctype) {
w_curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, header_func);
w_curl_easy_setopt(handle, CURLOPT_HEADERDATA, ctype);
}
if (ssl_capath)
w_curl_easy_setopt(handle, CURLOPT_CAPATH, ssl_capath);
if (!ssl_verifypeer)
w_curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L);
if (!ssl_verifyhost)
w_curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L);
curl_multi_add_handle(multi_handle, handle);
timeout = connection_timeout_ms;
/* obtain a read fd in "connection_timeout" seconds at worst */
for (timeout = connection_timeout_ms; timeout > 0; timeout -= busy_wait) {
mrc = curl_multi_perform(multi_handle, &running_handles);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_perform: %s\n", curl_multi_strerror(mrc));
goto error;
}
mrc = curl_multi_timeout(multi_handle, &retry_time);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_timeout: %s\n", curl_multi_strerror(mrc));
goto error;
}
if (retry_time == -1) {
LM_INFO("curl_multi_timeout() returned -1, pausing %ldms...\n",
sleep_on_bad_timeout);
busy_wait = sleep_on_bad_timeout;
usleep(1000UL * busy_wait);
continue;
}
if (retry_time > connection_timeout_ms)
LM_INFO("initial TCP connect: we must wait at least %ldms! Please "
"consider increasing 'connection_timeout'!\n", retry_time);
busy_wait = retry_time < timeout ? retry_time : timeout;
/**
* libcurl is currently stuck in internal operations (connect)
* we have to wait a bit until we receive a read fd
*/
for (i = 0; i < busy_wait; i += check_time) {
/* transfer may have already been completed!! */
while ((cmsg = curl_multi_info_read(multi_handle, &msgs_in_queue))) {
if (cmsg->easy_handle == handle && cmsg->msg == CURLMSG_DONE) {
LM_DBG("done, no need for async!\n");
clean_header_list;
*out_handle = handle;
return ASYNC_SYNC;
}
}
FD_ZERO(&rset);
mrc = curl_multi_fdset(multi_handle, &rset, &wset, &eset, &max_fd);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_fdset: %s\n", curl_multi_strerror(mrc));
goto error;
}
if (max_fd != -1) {
for (fd = 0; fd <= max_fd; fd++) {
if (FD_ISSET(fd, &rset)) {
LM_DBG(" >>>>>>>>>> fd %d ISSET(read)\n", fd);
if (is_new_transfer(fd)) {
LM_DBG("add fd to read list: %d\n", fd);
add_transfer(fd);
goto success;
}
}
}
}
usleep(1000UL * check_time);
}
}
LM_ERR("timeout while connecting to '%s' (%ld sec)\n", url, connection_timeout);
goto error;
success:
clean_header_list;
*out_handle = handle;
return fd;
error:
mrc = curl_multi_remove_handle(multi_handle, handle);
if (mrc != CURLM_OK)
LM_ERR("curl_multi_remove_handle: %s\n", curl_multi_strerror(mrc));
cleanup:
clean_header_list;
curl_easy_cleanup(handle);
return ASYNC_NO_IO;
}
static bool
input_curl_easy_init(struct input_curl *c, GError **error_r)
{
CURLcode code;
CURLMcode mcode;
c->eof = false;
c->easy = curl_easy_init();
if (c->easy == NULL) {
g_set_error(error_r, curl_quark(), 0,
"curl_easy_init() failed");
return false;
}
mcode = curl_multi_add_handle(c->multi, c->easy);
if (mcode != CURLM_OK) {
g_set_error(error_r, curl_quark(), mcode,
"curl_multi_add_handle() failed: %s",
curl_multi_strerror(mcode));
return false;
}
curl_easy_setopt(c->easy, CURLOPT_USERAGENT,
"Music Player Daemon " VERSION);
curl_easy_setopt(c->easy, CURLOPT_HEADERFUNCTION,
input_curl_headerfunction);
curl_easy_setopt(c->easy, CURLOPT_WRITEHEADER, c);
curl_easy_setopt(c->easy, CURLOPT_WRITEFUNCTION,
input_curl_writefunction);
curl_easy_setopt(c->easy, CURLOPT_WRITEDATA, c);
curl_easy_setopt(c->easy, CURLOPT_HTTP200ALIASES, http_200_aliases);
curl_easy_setopt(c->easy, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(c->easy, CURLOPT_MAXREDIRS, 5);
curl_easy_setopt(c->easy, CURLOPT_FAILONERROR, true);
curl_easy_setopt(c->easy, CURLOPT_ERRORBUFFER, c->error);
curl_easy_setopt(c->easy, CURLOPT_NOPROGRESS, 1l);
curl_easy_setopt(c->easy, CURLOPT_NOSIGNAL, 1l);
curl_easy_setopt(c->easy, CURLOPT_CONNECTTIMEOUT, 10l);
if (proxy != NULL)
curl_easy_setopt(c->easy, CURLOPT_PROXY, proxy);
if (proxy_port > 0)
curl_easy_setopt(c->easy, CURLOPT_PROXYPORT, (long)proxy_port);
if (proxy_user != NULL && proxy_password != NULL) {
char *proxy_auth_str =
g_strconcat(proxy_user, ":", proxy_password, NULL);
curl_easy_setopt(c->easy, CURLOPT_PROXYUSERPWD, proxy_auth_str);
g_free(proxy_auth_str);
}
code = curl_easy_setopt(c->easy, CURLOPT_URL, c->url);
if (code != CURLE_OK) {
g_set_error(error_r, curl_quark(), code,
"curl_easy_setopt() failed: %s",
curl_easy_strerror(code));
return false;
}
c->request_headers = NULL;
c->request_headers = curl_slist_append(c->request_headers,
"Icy-Metadata: 1");
curl_easy_setopt(c->easy, CURLOPT_HTTPHEADER, c->request_headers);
return true;
}
enum async_ret_code resume_async_http_req(int fd, struct sip_msg *msg, void *_param)
{
CURLcode rc;
CURLMcode mrc;
rest_async_param *param = (rest_async_param *)_param;
int running, max_fd;
long http_rc;
fd_set rset, wset, eset;
pv_value_t val;
mrc = curl_multi_perform(multi_handle, &running);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_perform: %s\n", curl_multi_strerror(mrc));
return -1;
}
LM_DBG("running handles: %d\n", running);
if (running == running_handles) {
async_status = ASYNC_CONTINUE;
return 1;
}
if (running > running_handles) {
LM_BUG("incremented handles!!");
/* default async status is DONE */
return -1;
}
running_handles = running;
FD_ZERO(&rset);
mrc = curl_multi_fdset(multi_handle, &rset, &wset, &eset, &max_fd);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_fdset: %s\n", curl_multi_strerror(mrc));
/* default async status is DONE */
return -1;
}
if (max_fd == -1) {
if (running_handles != 0) {
LM_BUG("running_handles == %d", running_handles);
abort();
/* default async status is DONE */
return -1;
}
if (FD_ISSET(fd, &rset)) {
LM_BUG("fd %d is still in rset!", fd);
abort();
/* default async status is DONE */
return -1;
}
} else if (FD_ISSET(fd, &rset)) {
LM_DBG("fd %d still transferring...\n", fd);
async_status = ASYNC_CONTINUE;
return 1;
}
if (del_transfer(fd) != 0) {
LM_BUG("failed to delete fd %d", fd);
abort();
/* default async status is DONE */
return -1;
}
mrc = curl_multi_remove_handle(multi_handle, param->handle);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_remove_handle: %s\n", curl_multi_strerror(mrc));
/* default async status is DONE */
return -1;
}
val.flags = PV_VAL_STR;
val.rs = param->body;
if (pv_set_value(msg, param->body_pv, 0, &val) != 0)
LM_ERR("failed to set output body pv\n");
if (param->ctype_pv) {
val.rs = param->ctype;
if (pv_set_value(msg, param->ctype_pv, 0, &val) != 0)
LM_ERR("failed to set output ctype pv\n");
}
if (param->code_pv) {
rc = curl_easy_getinfo(param->handle, CURLINFO_RESPONSE_CODE, &http_rc);
if (rc != CURLE_OK) {
LM_ERR("curl_easy_getinfo: %s\n", curl_easy_strerror(rc));
http_rc = 0;
}
LM_DBG("Last response code: %ld\n", http_rc);
val.flags = PV_VAL_INT|PV_TYPE_INT;
val.ri = (int)http_rc;
if (pv_set_value(msg, param->code_pv, 0, &val) != 0)
LM_ERR("failed to set output code pv\n");
}
pkg_free(param->body.s);
if (param->ctype_pv && param->ctype.s)
pkg_free(param->ctype.s);
curl_easy_cleanup(param->handle);
pkg_free(param);
/* default async status is DONE */
return 1;
}
/**
* start_async_http_req - performs an HTTP request, stores results in pvars
* - TCP connect phase is synchronous, due to libcurl limitations
* - TCP read phase is asynchronous, thanks to the libcurl multi interface
*
* @msg: sip message struct
* @method: HTTP verb
* @url: HTTP URL to be queried
* @req_body: Body of the request (NULL if not needed)
* @req_ctype: Value for the "Content-Type: " header of the request (same as ^)
* @async_parm: output param, will contain async handles
* @body: reply body; gradually reallocated as data arrives
* @ctype: will eventually hold the last "Content-Type" header of the reply
*/
int start_async_http_req(struct sip_msg *msg, enum rest_client_method method,
char *url, char *req_body, char *req_ctype,
rest_async_param *async_parm, str *body, str *ctype)
{
CURL *handle;
CURLcode rc;
CURLMcode mrc;
fd_set rset, wset, eset;
int max_fd, fd;
long busy_wait, timeout;
long retry_time;
OSS_CURLM *multi_list;
CURLM *multi_handle;
if (transfers == FD_SETSIZE) {
LM_ERR("too many ongoing transfers: %d\n", FD_SETSIZE);
goto cleanup;
}
handle = curl_easy_init();
if (!handle) {
LM_ERR("Init curl handle failed!\n");
goto cleanup;
}
w_curl_easy_setopt(handle, CURLOPT_URL, url);
if (curl_http_version != CURL_HTTP_VERSION_NONE)
w_curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, curl_http_version);
if (tls_dom) {
w_curl_easy_setopt(handle, CURLOPT_SSLCERT, tls_dom->cert.s);
w_curl_easy_setopt(handle, CURLOPT_SSLKEY, tls_dom->pkey.s);
tls_api.release_domain(tls_dom);
tls_dom = NULL;
}
switch (method) {
case REST_CLIENT_POST:
w_curl_easy_setopt(handle, CURLOPT_POST, 1);
w_curl_easy_setopt(handle, CURLOPT_POSTFIELDS, req_body);
if (req_ctype) {
sprintf(print_buff, "Content-Type: %s", req_ctype);
header_list = curl_slist_append(header_list, print_buff);
w_curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header_list);
}
break;
case REST_CLIENT_PUT:
w_curl_easy_setopt(handle, CURLOPT_CUSTOMREQUEST, "PUT");
w_curl_easy_setopt(handle, CURLOPT_POSTFIELDS, req_body);
if (req_ctype) {
sprintf(print_buff, "Content-Type: %s", req_ctype);
header_list = curl_slist_append(header_list, print_buff);
w_curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header_list);
}
break;
case REST_CLIENT_GET:
break;
default:
LM_ERR("Unsupported rest_client_method: %d, defaulting to GET\n", method);
}
if (header_list)
w_curl_easy_setopt(handle, CURLOPT_HTTPHEADER, header_list);
w_curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, connection_timeout);
w_curl_easy_setopt(handle, CURLOPT_TIMEOUT, curl_timeout);
w_curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
w_curl_easy_setopt(handle, CURLOPT_STDERR, stdout);
w_curl_easy_setopt(handle, CURLOPT_FAILONERROR, 0);
w_curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_func);
w_curl_easy_setopt(handle, CURLOPT_WRITEDATA, body);
if (ctype) {
w_curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, header_func);
w_curl_easy_setopt(handle, CURLOPT_HEADERDATA, ctype);
}
if (ssl_capath)
w_curl_easy_setopt(handle, CURLOPT_CAPATH, ssl_capath);
if (!ssl_verifypeer)
w_curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L);
if (!ssl_verifyhost)
w_curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L);
if ( rest_trace_enabled() ) {
async_parm->tparam = pkg_malloc(sizeof(rest_trace_param_t));
if ( !async_parm->tparam ) {
LM_ERR("no more pkg mem!\n");
clean_header_list;
return -1;
}
memset( async_parm->tparam, 0, sizeof *async_parm->tparam);
async_parm->tparam->callid = msg->callid->body;
w_curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, trace_rest_request_cb);
w_curl_easy_setopt(handle, CURLOPT_DEBUGDATA, async_parm->tparam);
}
multi_list = get_multi();
if (!multi_list) {
LM_INFO("failed to get a multi handle, doing blocking query\n");
rc = curl_easy_perform(handle);
clean_header_list;
async_parm->handle = handle;
return ASYNC_SYNC;
}
multi_handle = multi_list->multi_handle;
curl_multi_add_handle(multi_handle, handle);
timeout = connection_timeout_ms;
busy_wait = connect_poll_interval;
/* obtain a read fd in "connection_timeout" seconds at worst */
for (timeout = connection_timeout_ms; timeout > 0; timeout -= busy_wait) {
mrc = curl_multi_perform(multi_handle, &running_handles);
if (mrc != CURLM_OK && mrc != CURLM_CALL_MULTI_PERFORM) {
LM_ERR("curl_multi_perform: %s\n", curl_multi_strerror(mrc));
goto error;
}
mrc = curl_multi_timeout(multi_handle, &retry_time);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_timeout: %s\n", curl_multi_strerror(mrc));
goto error;
}
LM_DBG("libcurl TCP connect: we should wait up to %ldms "
"(timeout=%ldms, poll=%ldms)!\n", retry_time,
connection_timeout_ms, connect_poll_interval);
if (retry_time == -1) {
LM_DBG("curl_multi_timeout() returned -1, pausing %ldms...\n",
busy_wait);
goto busy_wait;
}
/* transfer may have already been completed!! */
if (running_handles == 0) {
LM_DBG("done, no need for async!\n");
clean_header_list;
async_parm->handle = handle;
mrc = curl_multi_remove_handle(multi_handle, handle);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_remove_handle: %s\n", curl_multi_strerror(mrc));
}
put_multi(multi_list);
return ASYNC_SYNC;
}
FD_ZERO(&rset);
mrc = curl_multi_fdset(multi_handle, &rset, &wset, &eset, &max_fd);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_fdset: %s\n", curl_multi_strerror(mrc));
goto error;
}
if (max_fd != -1) {
for (fd = 0; fd <= max_fd; fd++) {
if (FD_ISSET(fd, &rset)) {
LM_DBG("ongoing transfer on fd %d\n", fd);
if (is_new_transfer(fd)) {
LM_DBG(">>> add fd %d to ongoing transfers\n", fd);
add_transfer(fd);
goto success;
}
}
}
}
/*
* from curl_multi_timeout() docs: "retry_time" milliseconds "at most!"
* -> we'll wait only 1/10 of this time before retrying
*/
busy_wait = connect_poll_interval < timeout ?
connect_poll_interval : timeout;
busy_wait:
/* libcurl seems to be stuck in internal operations (TCP connect?) */
LM_DBG("busy waiting %ldms ...\n", busy_wait);
usleep(1000UL * busy_wait);
}
LM_ERR("timeout while connecting to '%s' (%ld sec)\n", url, connection_timeout);
goto error;
success:
async_parm->header_list = header_list;
async_parm->handle = handle;
async_parm->multi_list = multi_list;
header_list = NULL;
return fd;
error:
mrc = curl_multi_remove_handle(multi_handle, handle);
if (mrc != CURLM_OK) {
LM_ERR("curl_multi_remove_handle: %s\n", curl_multi_strerror(mrc));
}
put_multi(multi_list);
curl_easy_cleanup(handle);
cleanup:
clean_header_list;
if (tls_dom) {
tls_api.release_domain(tls_dom);
tls_dom = NULL;
}
return ASYNC_NO_IO;
}
static gboolean
lr_fastestmirror_perform(GSList *list,
LrFastestMirrorCb cb,
void *cbdata,
GError **err)
{
assert(!err || *err == NULL);
if (!list)
return TRUE;
CURLM *multihandle = curl_multi_init();
if (!multihandle) {
g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURL,
"curl_multi_init() error");
return FALSE;
}
// Add curl easy handles to multi handle
long handles_added = 0;
for (GSList *elem = list; elem; elem = g_slist_next(elem)) {
LrFastestMirror *mirror = elem->data;
if (mirror->curl) {
curl_multi_add_handle(multihandle, mirror->curl);
handles_added++;
}
}
if (handles_added == 0) {
curl_multi_cleanup(multihandle);
return TRUE;
}
cb(cbdata, LR_FMSTAGE_DETECTION, (void *) &handles_added);
int still_running;
gdouble elapsed_time = 0.0;
GTimer *timer = g_timer_new();
g_timer_start(timer);
do {
struct timeval timeout;
int rc, cm_rc;
int maxfd = -1;
long curl_timeout = -1;
fd_set fdread, fdwrite, fdexcep;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
// Set suitable timeout to play around with
timeout.tv_sec = 0;
timeout.tv_usec = HALF_OF_SECOND_IN_MICROS;
cm_rc = curl_multi_timeout(multihandle, &curl_timeout);
if (cm_rc != CURLM_OK) {
g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURLM,
"curl_multi_timeout() error: %s",
curl_multi_strerror(cm_rc));
curl_multi_cleanup(multihandle);
return FALSE;
}
// Set timeout to a reasonable value
if (curl_timeout >= 0) {
timeout.tv_sec = curl_timeout / 1000;
if (timeout.tv_sec >= 1) {
timeout.tv_sec = 0;
timeout.tv_usec = HALF_OF_SECOND_IN_MICROS;
} else {
timeout.tv_usec = (curl_timeout % 1000) * 1000;
if (timeout.tv_usec > HALF_OF_SECOND_IN_MICROS)
timeout.tv_usec = HALF_OF_SECOND_IN_MICROS;
}
}
// Get file descriptors from the transfers
cm_rc = curl_multi_fdset(multihandle, &fdread, &fdwrite,
&fdexcep, &maxfd);
if (cm_rc != CURLM_OK) {
g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURLM,
"curl_multi_fdset() error: %s",
curl_multi_strerror(cm_rc));
return FALSE;
}
rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
if (rc < 0) {
if (errno == EINTR) {
g_debug("%s: select() interrupted by signal", __func__);
} else {
g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_SELECT,
"select() error: %s", strerror(errno));
return FALSE;
}
}
curl_multi_perform(multihandle, &still_running);
// Break loop after some reasonable amount of time
elapsed_time = g_timer_elapsed(timer, NULL);
} while(still_running && elapsed_time < LENGT_OF_MEASUREMENT);
g_timer_destroy(timer);
// Remove curl easy handles from multi handle
// and calculate plain_connect_time
for (GSList *elem = list; elem; elem = g_slist_next(elem)) {
LrFastestMirror *mirror = elem->data;
CURL *curl = mirror->curl;
if (!curl)
continue;
// Remove handle
curl_multi_remove_handle(multihandle, curl);
// Calculate plain_connect_time
char *effective_url;
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url);
if (!effective_url) {
// No effective url is most likely an error
mirror->plain_connect_time = DBL_MAX;
} else if (g_str_has_prefix(effective_url, "file://")) {
// Local directories are considered to be the best mirrors
mirror->plain_connect_time = 0.0;
} else {
// Get connect time
double namelookup_time;
double connect_time;
double plain_connect_time;
curl_easy_getinfo(curl, CURLINFO_NAMELOOKUP_TIME, &namelookup_time);
curl_easy_getinfo(curl, CURLINFO_CONNECT_TIME, &connect_time);
if (connect_time == 0.0) {
// Zero connect time is most likely an error
plain_connect_time = DBL_MAX;
} else {
plain_connect_time = connect_time - namelookup_time;
}
mirror->plain_connect_time = plain_connect_time;
//g_debug("%s: name_lookup: %3.6f connect_time: %3.6f (%3.6f) | %s",
// __func__, namelookup_time, connect_time,
// mirror->plain_connect_time, mirror->url);
}
}
curl_multi_cleanup(multihandle);
return TRUE;
}
void CurlStreamFile::fillCache(std::streampos size){
#if 1
assert(size >= 0);
if(! _running || _cached >=size){
return ;
}
fd_set readfd, writefd, exceptfd;
int maxfd;
CURLMcode mcode;
timeval tv;
//hard-coded slect timeout
//this number is kept low to give more thread switch
//opportunities while waitting for a load
const long maxSleepUsec = 10000; //1/100 of a second
const unsigned int userTimeout = 60000;
WallClockTimer lastProgress;
while(_running){
fillCacheNonBlocking();
if(_cached>=size || !_running) break;
FD_ZERO(&readfd);
FD_ZERO(&writefd);
FD_ZERO(&exceptfd);
mcode = curl_multi_fdset(_mCurlHandle, &readfd, &writefd,
&exceptfd, &maxfd);
if(mcode != CURLM_OK){
throw SnailException(curl_multi_strerror(mcode));
}
if(maxfd<0){
//as of libcurl 7.21.x, the DNS resolving appears to be
//going on in the background, so curl_multi_fdset fails to
//return anything useful, So we use the user timeout value
//to give DNS enough time to resolve the lookup
if(userTimeout && lastProgress.elapsed()>userTimeout){
return ;
}else{
continue;
}
}//if(maxfd<0)
tv.tv_sec = 0;
tv.tv_usec = maxSleepUsec;
//wait for data on the filedescriptors until a timeout set in rc file
int ret = select(maxfd+1, &readfd, &writefd, &exceptfd, &tv);
#if !defined(WIN32)
if(ret == -1){
if(errno == EINTR){
cout<<"select() was interrupted by a singal"<<endl;
ret = 0;
}else{
std::ostringstream os;
os<<"error polling data from connection to"<<_url<<":"<<strerror(errno);
throw SnailException(os.str());
}
}
#endif
if(!ret){
//timeout check the clock to see
//if we expired the user timeout
if(userTimeout && lastProgress.elapsed() > userTimeout){
cout<<"timeout ("<<userTimeout<<") while loading from URL"<<_url<<endl;
return ;
}
}else{
lastProgress.restart();
}
}//while(....
processMessages();
#endif
}
/**
* Updates all registered GPollFD objects, unregisters old ones,
* registers new ones.
*/
static void
http_client_update_fds(void)
{
fd_set rfds, wfds, efds;
int max_fd;
CURLMcode mcode;
GSList *fds;
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
mcode = curl_multi_fdset(http_client.multi, &rfds, &wfds,
&efds, &max_fd);
if (mcode != CURLM_OK) {
g_warning("curl_multi_fdset() failed: %s\n",
curl_multi_strerror(mcode));
return;
}
fds = http_client.fds;
http_client.fds = NULL;
while (fds != NULL) {
GPollFD *poll_fd = fds->data;
gushort events = http_client_fd_events(poll_fd->fd, &rfds,
&wfds, &efds);
assert(poll_fd->events != 0);
fds = g_slist_remove(fds, poll_fd);
if (events != poll_fd->events)
g_source_remove_poll(http_client.source, poll_fd);
if (events != 0) {
if (events != poll_fd->events) {
poll_fd->events = events;
g_source_add_poll(http_client.source, poll_fd);
}
http_client.fds = g_slist_prepend(http_client.fds,
poll_fd);
} else {
g_free(poll_fd);
}
}
for (int fd = 0; fd <= max_fd; ++fd) {
gushort events = http_client_fd_events(fd, &rfds,
&wfds, &efds);
if (events != 0) {
GPollFD *poll_fd = g_new(GPollFD, 1);
poll_fd->fd = fd;
poll_fd->events = events;
g_source_add_poll(http_client.source, poll_fd);
http_client.fds = g_slist_prepend(http_client.fds,
poll_fd);
}
}
}
