/* Read the request from a client socket.
*/
int fetch_request(t_session *session) {
char *new_reqbuf, *strstart, *strend;
long max_request_size, bytes_read, header_length = -1, content_length = -1, chunk_size_pos;
int result = 200, write_bytes, poll_result, upload_handle = -1, retval;
time_t deadline;
struct pollfd poll_data;
bool keep_reading = true, store_on_disk = false, chunked_request = false;
if (session->request_limit == false) {
deadline = session->time + NO_REQUEST_LIMIT_TIME;
max_request_size = NO_REQUEST_LIMIT_SIZE;
} else if (session->kept_alive == 0) {
deadline = session->time + session->binding->time_for_1st_request;
max_request_size = session->binding->max_request_size;
} else {
deadline = session->time + session->binding->time_for_request;
max_request_size = session->binding->max_request_size;
}
do {
/* Check if requestbuffer contains a complete request.
*/
if (session->request != NULL) {
if (header_length == -1) {
if ((strstart = strstr(session->request, "\r\n\r\n")) != NULL) {
*(strstart + 2) = '\0';
header_length = strstart + 4 - session->request;
session->header_length = header_length;
determine_request_method(session);
store_on_disk = (session->request_method == PUT) && session->binding->enable_alter;
if (store_on_disk) {
if ((session->uploaded_file = (char*)malloc(session->config->upload_directory_len + 15)) != NULL) {
strcpy(session->uploaded_file, session->config->upload_directory);
strcpy(session->uploaded_file + session->config->upload_directory_len, "/upload_XXXXXX");
umask(S_IWGRP | S_IWOTH);
if ((upload_handle = mkstemp(session->uploaded_file)) == -1) {
free(session->uploaded_file);
session->uploaded_file = NULL;
}
}
if (session->uploaded_file == NULL) {
log_error(session, "can't create temporary file for PUT request");
result = 500;
break;
}
session->uploaded_size = session->bytes_in_buffer - header_length;
if (write_buffer(upload_handle, session->request + header_length, session->uploaded_size) == -1) {
result = 500;
break;
}
session->bytes_in_buffer = header_length;
}
}
}
if (header_length != -1) {
if ((content_length == -1) && (chunked_request == false)) {
if ((strstart = strcasestr(session->request, hs_conlen)) != NULL) {
/* Request has Content-Length
*/
strstart += 16;
if ((strend = strstr(strstart, "\r\n")) != NULL) {
*strend = '\0';
content_length = str_to_int(strstart);
*strend = '\r';
if ((content_length < 0) || (INT_MAX - content_length - 2 <= header_length)) {
result = 500;
break;
}
if (store_on_disk) {
/* Write to file on disk
*/
session->content_length = 0;
if (content_length > session->binding->max_upload_size) {
result = 413;
break;
}
session->buffer_size = header_length + REQUEST_BUFFER_CHUNK;
if ((new_reqbuf = (char*)realloc(session->request, session->buffer_size + 1)) != NULL) {
session->request = new_reqbuf;
} else {
session->error_cause = ec_SOCKET_READ_ERROR;
result = -1;
break;
}
} else {
/* Read into memory
*/
session->content_length = content_length;
if (header_length + content_length > max_request_size) {
session->error_cause = ec_MAX_REQUESTSIZE;
result = -1;
break;
}
if (header_length + content_length > session->buffer_size) {
session->buffer_size = header_length + content_length;
if ((new_reqbuf = (char*)realloc(session->request, session->buffer_size + 1)) != NULL) {
session->request = new_reqbuf;
} else {
session->error_cause = ec_SOCKET_READ_ERROR;
result = -1;
break;
}
}
}
}
} else if (strcasestr(session->request, hs_chunked) != NULL) {
/* Chunked transfer encoding
*/
if (store_on_disk) {
log_error(session, "Chunked transfer encoding for PUT requests not supported.");
result = -1;
break;
}
chunked_request = true;
chunk_size_pos = 0;
} else {
/* No content
*/
session->content_length = 0;
if (store_on_disk) {
result = 411;
}
break;
}
}
if (content_length > -1) {
if (store_on_disk) {
if (session->uploaded_size == content_length) {
/* Received a complete PUT request */
break;
}
} else {
if (session->bytes_in_buffer >= header_length + content_length) {
/* Received a complete request */
break;
}
}
} else if (chunked_request) {
/* All chunks uploaded
*/
retval = all_chunks_uploaded(session->request + session->header_length,
session->bytes_in_buffer - session->header_length,
&chunk_size_pos);
if (retval == -1) {
result = 400;
break;
} else if (retval == 1) {
if ((session->content_length = merge_chunks(session->request + session->header_length,
session->bytes_in_buffer - session->header_length,
&(session->bytes_in_buffer))) == -1) {
result = -1;
}
break;
}
}
}
}
#ifdef ENABLE_SSL
poll_result = session->binding->use_ssl ? ssl_pending(&(session->ssl_context)) : 0;
if (poll_result == 0) {
#endif
poll_data.fd = session->client_socket;
poll_data.events = POLL_EVENT_BITS;
poll_result = poll(&poll_data, 1, 1000);
#ifdef ENABLE_SSL
}
#endif
switch (poll_result) {
case -1:
if (errno != EINTR) {
if (session->bytes_in_buffer == 0) {
session->error_cause = ec_CLIENT_DISCONNECTED;
} else {
session->error_cause = ec_SOCKET_READ_ERROR;
}
result = -1;
keep_reading = false;
}
break;
case 0:
if (session->force_quit) {
session->error_cause = ec_FORCE_QUIT;
result = -1;
keep_reading = false;
} else if (time(NULL) > deadline) {
session->error_cause = ec_TIMEOUT;
result = -1;
keep_reading = false;
}
break;
default:
if ((content_length == -1) && ((session->buffer_size - session->bytes_in_buffer) < 256)) {
session->buffer_size += REQUEST_BUFFER_CHUNK;
if ((new_reqbuf = (char*)realloc(session->request, session->buffer_size + 1)) != NULL) {
session->request = new_reqbuf;
} else {
session->error_cause = ec_SOCKET_READ_ERROR;
result = -1;
keep_reading = false;
break;
}
}
/* Read from socket.
*/
#ifdef ENABLE_SSL
if (session->binding->use_ssl) {
bytes_read = ssl_receive(&(session->ssl_context), session->request + session->bytes_in_buffer,
session->buffer_size - session->bytes_in_buffer);
} else
#endif
bytes_read = recv(session->client_socket, session->request + session->bytes_in_buffer,
session->buffer_size - session->bytes_in_buffer, 0);
switch (bytes_read) {
case -1:
if (errno != EINTR) {
if (session->bytes_in_buffer == 0) {
session->error_cause = ec_CLIENT_DISCONNECTED;
} else {
session->error_cause = ec_SOCKET_READ_ERROR;
}
result = -1;
keep_reading = false;
}
break;
case 0:
session->error_cause = ec_CLIENT_DISCONNECTED;
result = -1;
keep_reading = false;
break;
default:
if (store_on_disk) {
/* Write to file on disk
*/
write_bytes = bytes_read;
if (session->uploaded_size + bytes_read > content_length) {
write_bytes -= ((session->uploaded_size + bytes_read) - content_length);
}
if (write_buffer(upload_handle, session->request + header_length, write_bytes) == -1) {
result = 500;
keep_reading = false;
break;
}
if ((session->uploaded_size += write_bytes) > session->binding->max_upload_size) {
keep_reading = false;
result = 413;
break;
}
if (write_bytes < bytes_read) {
memmove(session->request + header_length, session->request + header_length + write_bytes, bytes_read - write_bytes);
session->bytes_in_buffer += bytes_read - write_bytes;
keep_reading = false;
}
} else {
/* Read into memory
*/
session->bytes_in_buffer += bytes_read;
*(session->request + session->bytes_in_buffer) = '\0';
if (session->bytes_in_buffer > max_request_size) {
keep_reading = false;
session->error_cause = ec_MAX_REQUESTSIZE;
result = -1;
break;
}
}
}
}
} while (keep_reading);
if (upload_handle != -1) {
fsync(upload_handle);
close(upload_handle);
}
#ifdef ENABLE_TOMAHAWK
increment_transfer(TRANSFER_RECEIVED, header_length + content_length);
#endif
return result;
}
int ffdl_download_to_file_with_options(char *url, char *filename,
unsigned long long chunkSize,
long maxConnections, long timeout,
long rateLimit) {
// initialize curl
curl_global_init(CURL_GLOBAL_ALL);
int result = TRUE;
double filesize = ffdl_get_file_size_bytes(url);
#ifdef DEBUG
fprintf(stdout, "filesize: %f\n", filesize);
#endif
if (chunkSize == 0) {
chunkSize = c_defaultChunkSize;
}
if (maxConnections == 0) {
maxConnections = c_defaultMaxConnections;
}
if (timeout < 0) {
timeout = c_defaultTimeoutSecs;
}
if (rateLimit < 0) {
rateLimit = 0;
}
#ifdef DEBUG
fprintf(stdout, "Options: chunksize: %llu\tmaxConnections: %ld\ttimeout: "
"%ld\trateLimit: %ld\n",
chunkSize, maxConnections, timeout, rateLimit);
#endif
unsigned long long numberOfChunks = ceil(filesize / (double)(chunkSize));
unsigned long long currentStartOfPartition = 0;
unsigned long long currentPartitionSize = 0;
CURL **curlHandles = (CURL **)malloc(c_partitionSize * sizeof(CURL *));
FILE **fileDescriptors = (FILE **)malloc(c_partitionSize * sizeof(FILE *));
// Unfortunately most systems of a file descriptor limit of 1024 by default
// Each loop has at most 1000 file descriptors open at one time
// To make things worse, curl versions earlier than 7.23.0 have a bug where
// once the curl lib overflows it's own internal count of FD_SET,
// so even if you raise the system file descriptor limit, curl will segfault.
// The current version of ubuntu ships with libcurl 7.22.0 which has this bug.
for (currentStartOfPartition = 0; currentStartOfPartition < numberOfChunks;
currentStartOfPartition += c_partitionSize) {
currentPartitionSize =
min(c_partitionSize, numberOfChunks - currentStartOfPartition);
CURLM *curlMulti = curl_multi_init();
curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS,
maxConnections);
setup_curl_handlers(curlMulti, url, filename, currentStartOfPartition,
currentPartitionSize, chunkSize, timeout, rateLimit,
curlHandles, fileDescriptors);
perform_multi_curl_download(curlMulti, currentPartitionSize, timeout,
curlHandles);
clean_up_curl_multi_connections(curlMulti, currentPartitionSize,
curlHandles, fileDescriptors);
}
free(curlHandles);
free(fileDescriptors);
merge_chunks(numberOfChunks, chunkSize, filename);
curl_global_cleanup();
return result;
}
