size_t
CoolUrlFread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
{
size_t want;
want = nmemb * size;
fill_buffer(file,want,1);
// check if theres data in the buffer - if not fill_buffer()
// either errored or EOF
if(!file->buffer_pos)
return 0;
// ensure only available data is considered
if(file->buffer_pos < want)
want = file->buffer_pos;
// xfer data to caller */
memcpy(ptr, file->buffer, want);
use_buffer(file,want);
want = want / size; // number of items - nb correct op - checked
// with glibc code
return want;
}
static void
down (int i)
{
char buf[10000];
static void *last;
if (last && last < (void *) buf)
{
printf ("%d: %p < %p\n", i, last, buf);
marker_hit ();
}
last = buf;
if (i == 500)
{
if (munmap (reserved, RESERVED_SIZE) != 0)
abort ();
reserved = NULL;
}
if (i > 0)
{
use_buffer (buf);
down (i - 1);
}
else
marker_miss ();
}
char *
url_fgets(char *ptr, int size, URL_FILE *file)
{
int want = size - 1;/* always need to leave room for zero termination */
int loop;
switch(file->type)
{
case CFTYPE_FILE:
ptr = fgets(ptr,size,file->handle.file);
break;
case CFTYPE_CURL:
fill_buffer(file,want,1);
/* check if theres data in the buffer - if not fill either errored or
* EOF */
if(!file->buffer_pos)
return NULL;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/*buffer contains data */
/* look for newline or eof */
for(loop=0;loop < want;loop++)
{
if(file->buffer[loop] == '\n')
{
want=loop+1;/* include newline */
break;
}
}
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
ptr[want]=0;/* allways null terminate */
use_buffer(file,want);
/*printf("(fgets) return %d bytes %d left\n", want,file->buffer_pos);*/
break;
default: /* unknown or supported type - oh dear */
ptr=NULL;
errno=EBADF;
break;
}
return ptr;/*success */
}
static size_t curl_file_fread(ALLEGRO_FILE *f, void *ptr, size_t size)
{
CURL_FILE *cf = al_get_file_userdata(f);
fill_buffer(cf, size);
if (!cf->buffer_pos)
return 0;
if (cf->buffer_pos < size)
size = cf->buffer_pos;
memcpy(ptr, cf->buffer, size);
use_buffer(cf, size);
return size;
}
size_t
url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
{
size_t want;
switch(file->type)
{
case CFTYPE_FILE:
want=fread(ptr,size,nmemb,file->handle.file);
break;
case CFTYPE_CURL:
want = nmemb * size;
fill_buffer(file,want,1);
/* check if theres data in the buffer - if not fill_buffer()
* either errored or EOF */
if(!file->buffer_pos)
return 0;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
use_buffer(file,want);
want = want / size; /* number of items - nb correct op - checked
* with glibc code*/
/*printf("(fread) return %d bytes %d left\n", want,file->buffer_pos);*/
break;
default: /* unknown or supported type - oh dear */
want=0;
errno=EBADF;
break;
}
return want;
}
std::size_t read(unsigned char* aBuffer, std::size_t aByteCount)
{
fill_buffer(iCURLData.get(), aByteCount);
/* check if theres data in the buffer - if not fill_buffer()
* either errored or EOF */
if(!iCURLData->buffer_pos)
return 0;
/* ensure only available data is considered */
if(iCURLData->buffer_pos < aByteCount)
aByteCount = iCURLData->buffer_pos;
/* xfer data to caller */
memcpy(aBuffer, iCURLData->buffer, aByteCount);
use_buffer(iCURLData.get(), aByteCount);
return aByteCount; /* number of items */
}
char *
CoolUrlFgets(char *ptr, int size, URL_FILE *file)
{
int want = size - 1;// always need to leave room for zero termination
int loop;
fill_buffer(file,want,1);
//check if theres data in the buffer - if not fill either errored or
//EOF
if(!file->buffer_pos)
return NULL;
//ensure only available data is considered
if(file->buffer_pos < want)
want = file->buffer_pos;
//buffer contains data
// look for newline or eof
for(loop=0;loop < want;loop++)
{
if(file->buffer[loop] == '\n')
{
want=loop+1;//nclude newline
break;
}
}
//xfer data to caller
memcpy(ptr, file->buffer, want);
ptr[want]=0;//allways null terminate
use_buffer(file,want);
return ptr;//success
}
URLIO_FILE *urlio_fopen(const char *url, const char *operation) {
/* this code could check for URLs or types in the 'url' and
basically use the real fopen() for standard files */
URLIO_FILE *file = NULL;
URLIO_CONN *conn = NULL;
(void) operation;
const size_t want = 1;
double dSize;
#ifdef URLIO_VERBOSE
printf("fopen: %s\n", url);
#endif
file = (URLIO_FILE*) calloc(1, sizeof(URLIO_FILE));
if (!file)
return NULL;
file->handle.file = fopen(url, operation);
if (file->handle.file) {
file->type = CFTYPE_FILE; /* marked as FILE */
file->url = (char*) malloc((strlen(url) + 1) * sizeof(char));
strcpy(file->url, url);
file->pos = 0;
fseek(file->handle.file, 0, SEEK_END);
file->handle.conn->size = ftell(file->handle.file);
fseek(file->handle.file, 0, SEEK_SET);
} else {
file->type = CFTYPE_CURL; /* marked as URL */
int wanted_urlio_index = -1;
for (int i = 0; i < g_urlio_count; i++) {
if (!strcmp(g_urlio_list[i]->url, url)) {
printf("stream exists, reuse it\n");
wanted_urlio_index = i;
break;
}
}
if (wanted_urlio_index == -1) {
conn = (URLIO_CONN*) calloc(1, sizeof(URLIO_CONN));
conn->io[0].curl = curl_easy_init();
curl_easy_setopt(conn->io[0].curl, CURLOPT_URL, url);
curl_easy_setopt(conn->io[0].curl, CURLOPT_WRITEDATA,
&(conn->io[0]));
curl_easy_setopt(conn->io[0].curl, CURLOPT_VERBOSE, CURL_VERBOSE);
curl_easy_setopt(conn->io[0].curl, CURLOPT_WRITEFUNCTION,
write_callback);
conn->multi_handle = curl_multi_init();
curl_multi_add_handle(conn->multi_handle, conn->io[0].curl);
/* lets start the fetch */
curl_multi_perform(conn->multi_handle, &conn->still_running);
if ((conn->io[0].buffer_pos == 0) && (!conn->still_running)) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(conn->multi_handle, conn->io[0].curl);
/* cleanup */
curl_easy_cleanup(conn->io[0].curl);
free(file);
free(conn);
file = NULL;
} else {
for (int retry = 0; retry < RETRY_TIMES; retry++) {
fill_buffer(conn, want, 0);
/* check if there's data in the buffer - if not fill either error or
* EOF */
if (conn->io[0].buffer_pos) {
use_buffer(conn, want, 0);
curl_easy_getinfo(conn->io[0].curl,
CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dSize);
conn->size = (long) dSize;
#ifdef URLIO_VERBOSE
printf("stream opened, length: %zu\n", conn->size);
#endif
break;
}
/* halt transaction */
curl_multi_remove_handle(conn->multi_handle,
conn->io[0].curl);
/* restart */
curl_multi_add_handle(conn->multi_handle, conn->io[0].curl);
/* ditch buffer - write will recreate - resets stream pos*/
free(conn->io[0].buffer);
conn->io[0].buffer = NULL;
conn->io[0].buffer_pos = 0;
conn->io[0].buffer_len = 0;
}
/* check if there's data in the buffer - if not fill either error or
* EOF */
if (conn->io[0].buffer_pos) {
/* halt transaction */
curl_multi_remove_handle(conn->multi_handle,
conn->io[0].curl);
/* restart */
curl_multi_add_handle(conn->multi_handle, conn->io[0].curl);
/* ditch buffer - write will recreate - resets stream pos*/
free(conn->io[0].buffer);
conn->io[0].buffer = NULL;
conn->io[0].buffer_pos = 0;
conn->io[0].buffer_len = 0;
} else {
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(conn->multi_handle,
conn->io[0].curl);
/* cleanup */
curl_easy_cleanup(conn->io[0].curl);
free(file);
free(conn);
file = NULL;
}
}
conn->url = (char*) malloc((strlen(url) + 1) * sizeof(char));
strcpy(conn->url, url);
if (g_urlio_count == 0)
g_urlio_list = (URLIO_CONN**) malloc(sizeof(URLIO_CONN*));
else
g_urlio_list = (URLIO_CONN**) realloc(g_urlio_list,
(g_urlio_count + 1) * sizeof(URLIO_CONN*));
g_urlio_list[g_urlio_count] = conn;
wanted_urlio_index = g_urlio_count;
g_urlio_count++;
}
file->handle.conn = g_urlio_list[wanted_urlio_index];
}
if (file) {
file->url = (char*) malloc((strlen(url) + 1) * sizeof(char));
strcpy(file->url, url);
file->pos = 0;
}
return file;
}
static size_t download(void *ptr, size_t pos, size_t wanted, URLIO_CONN *conn) {
int wanted_bulk_count = (((pos % (CACHE_BULK_SIZE)) + wanted - 1)
/ (CACHE_BULK_SIZE)) + 1;
size_t wanted_bulk_id = (pos / (CACHE_BULK_SIZE)) * (CACHE_BULK_SIZE);
size_t residual = wanted;
size_t copied = 0L;
size_t want_per_thread[THREAD_NUM];
size_t want_total;
for (int i = 0; i < wanted_bulk_count; i++) {
int wanted_cache_index = -1;
int wanted_bulk_index = -1;
for (int j = 0; j < g_cache_count; j++) {
if (!strcmp(g_cache_list[j]->url, conn->url)) {
wanted_cache_index = j;
break;
}
}
if (wanted_cache_index != -1) {
for (int j = 0; j < g_cache_list[wanted_cache_index]->bulk_count;
j++) {
if (g_cache_list[wanted_cache_index]->bulk_id_list[j]
== wanted_bulk_id) {
printf("download: cache hit bulk id %zu\n", wanted_bulk_id);
wanted_bulk_index = j;
break;
}
}
}
if (wanted_bulk_index == -1) { /* missed bulk */
printf("download: cache is missed\n");
g_mutex_lock(&g_cache_lock);
for (int t = 0; t < THREAD_NUM; t++) {
if (pos + t * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
/* close the stream */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(conn->multi_handle, conn->io[t].curl);
/* cleanup */
curl_easy_cleanup(conn->io[t].curl);
/* reopen the stream and resume at a specific pos */
conn->io[t].curl = curl_easy_init();
curl_easy_setopt(conn->io[t].curl, CURLOPT_URL, conn->url);
curl_easy_setopt(conn->io[t].curl, CURLOPT_WRITEDATA,
&(conn->io[t]));
curl_easy_setopt(conn->io[t].curl, CURLOPT_VERBOSE,
CURL_VERBOSE);
curl_easy_setopt(conn->io[t].curl, CURLOPT_WRITEFUNCTION,
write_callback);
curl_easy_setopt(conn->io[t].curl, CURLOPT_RESUME_FROM_LARGE,
wanted_bulk_id+t*(CACHE_BULK_SIZE_PER_THREAD)); /* bulk id represents the pointer of the head of a bulk*/
// /* This tests if multi_handle has been initialzed. It should always be initialized.*/
// if (!urlio->multi_handle)
// urlio->multi_handle = curl_multi_init();
curl_multi_add_handle(conn->multi_handle, conn->io[t].curl);
}
/* lets start the fetch */
curl_multi_perform(conn->multi_handle, &conn->still_running);
for (int t = 0; t < THREAD_NUM; t++) {
if (pos + t * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
if ((conn->io[t].buffer_pos == 0) && (!conn->still_running)) {
/* if still_running is 0 now, we should return NULL */
for (int q = 0; q < THREAD_NUM; q++) {
if (pos + q * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(conn->multi_handle,
conn->io[q].curl);
/* cleanup */
curl_easy_cleanup(conn->io[q].curl);
}
g_mutex_unlock(&g_cache_lock);
return 0L;
}
}
/* check if there's data in the buffer - if not fill either error or
* EOF */
for (int t = 0; t < THREAD_NUM; t++) {
if (pos + t * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
if (conn->io[t].buffer_pos) {
/* halt transaction */
curl_multi_remove_handle(conn->multi_handle,
conn->io[t].curl);
/* restart */
curl_multi_add_handle(conn->multi_handle, conn->io[t].curl);
/* ditch buffer - write will recreate - resets stream pos*/
free(conn->io[t].buffer);
conn->io[t].buffer = NULL;
conn->io[t].buffer_pos = 0L;
conn->io[t].buffer_len = 0L;
}
}
/* This paragraph reads want of data from the stream */
for (int t = 0; t < THREAD_NUM; t++) {
if (pos + t * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
want_per_thread[t] = (CACHE_BULK_SIZE_PER_THREAD);
fill_buffer(conn, want_per_thread[t], t);
/* check if there's data in the buffer - if not fill_buffer()
* either error or EOF */
if (!conn->io[t].buffer_pos) {
g_mutex_unlock(&g_cache_lock);
return 0L;
}
/* ensure only available data is considered */
if (conn->io[t].buffer_pos < want_per_thread[t])
want_per_thread[t] = conn->io[t].buffer_pos;
}
want_total = 0L;
for (int t = 0; t < THREAD_NUM; t++) {
if (pos + t * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
want_total += want_per_thread[t];
}
/* Expand cache memory */
if (wanted_cache_index == -1) {
if (!g_cache_list)
g_cache_list = (URLIO_CACHE**) malloc(sizeof(URLIO_CACHE*));
else
g_cache_list = (URLIO_CACHE**) realloc(g_cache_list,
(g_cache_count + 1) * sizeof(URLIO_CACHE*));
g_cache_list[g_cache_count] = (URLIO_CACHE*) calloc(1,
sizeof(URLIO_CACHE));
g_cache_list[g_cache_count]->url = (char*) malloc(
(strlen(conn->url) + 1) * sizeof(char));
strcpy(g_cache_list[g_cache_count]->url, conn->url);
wanted_cache_index = g_cache_count;
g_cache_count++;
}
/* Expand cache memory */
if (g_cache_list[wanted_cache_index]->bulk_count == 0) {
g_cache_list[wanted_cache_index]->bulk_list = (char**) malloc(
sizeof(char*));
g_cache_list[wanted_cache_index]->bulk_id_list =
(size_t*) malloc(sizeof(size_t));
g_cache_list[wanted_cache_index]->bulk_size_list =
(size_t*) malloc(sizeof(size_t));
g_cache_list[wanted_cache_index]->bulk_list[g_cache_list[wanted_cache_index]->bulk_count] =
(char*) malloc(want_total * sizeof(char));
} else {
g_cache_list[wanted_cache_index]->bulk_list = (char**) realloc(
g_cache_list[wanted_cache_index]->bulk_list,
(g_cache_list[wanted_cache_index]->bulk_count + 1)
* sizeof(char*));
g_cache_list[wanted_cache_index]->bulk_id_list =
(size_t*) realloc(
g_cache_list[wanted_cache_index]->bulk_id_list,
(g_cache_list[wanted_cache_index]->bulk_count
+ 1) * sizeof(size_t));
g_cache_list[wanted_cache_index]->bulk_size_list =
(size_t*) realloc(
g_cache_list[wanted_cache_index]->bulk_size_list,
(g_cache_list[wanted_cache_index]->bulk_count
+ 1) * sizeof(size_t));
g_cache_list[wanted_cache_index]->bulk_list[g_cache_list[wanted_cache_index]->bulk_count] =
(char*) malloc(want_total * sizeof(char));
}
for (int t = 0; t < THREAD_NUM; t++) {
if (pos + t * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
/* xfer data to caller */
memcpy(
g_cache_list[wanted_cache_index]->bulk_list[g_cache_list[wanted_cache_index]->bulk_count]
+ t * (CACHE_BULK_SIZE_PER_THREAD),
conn->io[t].buffer, want_per_thread[t]);
}
g_cache_list[wanted_cache_index]->bulk_size_list[g_cache_list[wanted_cache_index]->bulk_count] =
want_total;
g_cache_list[wanted_cache_index]->bulk_id_list[g_cache_list[wanted_cache_index]->bulk_count] =
wanted_bulk_id;
for (int t = 0; t < THREAD_NUM; t++) {
if (pos + t * (CACHE_BULK_SIZE_PER_THREAD) > conn->size)
break;
use_buffer(conn, want_per_thread[t], t);
}
wanted_bulk_index = g_cache_list[wanted_cache_index]->bulk_count;
g_cache_list[wanted_cache_index]->bulk_count++;
printf("download: obtained bulk id %zu\n", wanted_bulk_id);
g_mutex_unlock(&g_cache_lock);
}
size_t copy_ptr = (i == 0) ? pos % (CACHE_BULK_SIZE) : 0;
size_t copy_size =
(residual
< g_cache_list[wanted_cache_index]->bulk_size_list[wanted_bulk_index]
- copy_ptr) ?
residual :
(g_cache_list[wanted_cache_index]->bulk_size_list[wanted_bulk_index]
- copy_ptr);
size_t ptr_ptr = wanted - residual;
memcpy((char*) ptr + ptr_ptr,
&g_cache_list[wanted_cache_index]->bulk_list[wanted_bulk_index][copy_ptr],
copy_size);
copied += copy_size;
residual -= copy_size;
wanted_bulk_id += CACHE_BULK_SIZE;
}
return copied;
}
