/* Start downloading */
void axel_start(axel_t *axel)
{
int i;
/* HTTP might've redirected and FTP handles wildcards, so
re-scan the URL for every conn */
for (i = 0; i < axel->conf->num_connections; i++)
{
conn_set(&axel->conn[i], axel->url->text);
axel->url = axel->url->next;
axel->conn[i].local_if = axel->conf->interfaces->text;
axel->conf->interfaces = axel->conf->interfaces->next;
axel->conn[i].conf = axel->conf;
if (i)
{
axel->conn[i].supported = 1;
}
}
if( axel->conf->verbose > 0 )
{
axel_message(axel, _("Starting download"));
}
for (i = 0; i < axel->conf->num_connections; i++)
{
if (axel->conn[i].currentbyte <= axel->conn[i].lastbyte)
{
if ( axel->conf->verbose >= 2)
{
axel_message(axel, _("Connection %i downloading from %s:%i using interface %s"),
i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
}
axel->conn[i].state = 1;
#if WIN32
axel->conn[i].setup_thread = CreateThread(NULL, 0, setup_thread_cb, &axel->conn[i], 0, NULL);
if (NULL == axel->conn[i].setup_thread)
#else
if (0 != pthread_create(axel->conn[i].setup_thread, NULL, setup_thread_cb, &axel->conn[i]))
#endif
{
axel_message(axel, _("thread error in axel_start!!!"));
axel->ready = -1;
}
else
{
axel->conn[i].last_transfer = gettime();
}
}
}
/* The real downloading will start now, so let's start counting */
axel->start_time = gettime();
axel->ready = 0;
}
/* Create a new axel_t structure */
axel_t *axel_new( conf_t *conf, int count, void *url )
{
search_t *res;
axel_t *axel;
url_t *u;
char *s;
int i;
axel = malloc( sizeof( axel_t ) );
memset( axel, 0, sizeof( axel_t ) );
*axel->conf = *conf;
axel->conn = malloc( sizeof( conn_t ) * axel->conf->num_connections );
memset( axel->conn, 0, sizeof( conn_t ) * axel->conf->num_connections );
if( axel->conf->max_speed > 0 )
{
if( (float) axel->conf->max_speed / axel->conf->buffer_size < 0.5 )
{
if( axel->conf->verbose >= 2 )
axel_message( axel, _("Buffer resized for this speed.") );
axel->conf->buffer_size = axel->conf->max_speed;
}
axel->delay_time = (int) ( (float) 1000000 / axel->conf->max_speed * axel->conf->buffer_size * axel->conf->num_connections );
}
if( buffer == NULL )
buffer = malloc( max( MAX_STRING, axel->conf->buffer_size ) );
if( count == 0 )
{
axel->url = malloc( sizeof( url_t ) );
axel->url->next = axel->url;
strncpy( axel->url->text, (char *) url, MAX_STRING );
}
else
{
res = (search_t *) url;
u = axel->url = malloc( sizeof( url_t ) );
for( i = 0; i < count; i ++ )
{
strncpy( u->text, res[i].url, MAX_STRING );
if( i < count - 1 )
{
u->next = malloc( sizeof( url_t ) );
u = u->next;
}
else
{
u->next = axel->url;
}
}
}
axel->conn[0].conf = axel->conf;
if( !conn_set( &axel->conn[0], axel->url->text ) )
{
axel_message( axel, _("Could not parse URL.\n") );
axel->ready = -1;
return( axel );
}
axel->conn[0].local_if = axel->conf->interfaces->text;
axel->conf->interfaces = axel->conf->interfaces->next;
strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
http_decode( axel->filename );
if( *axel->filename == 0 ) /* Index page == no fn */
strncpy( axel->filename, axel->conf->default_filename, MAX_STRING );
if( ( s = strchr( axel->filename, '?' ) ) != NULL && axel->conf->strip_cgi_parameters )
*s = 0; /* Get rid of CGI parameters */
if( !conn_init( &axel->conn[0] ) )
{
axel_message( axel, axel->conn[0].message );
axel->ready = -1;
return( axel );
}
/* This does more than just checking the file size, it all depends
on the protocol used. */
if( !conn_info( &axel->conn[0] ) )
{
axel_message( axel, axel->conn[0].message );
axel->ready = -1;
return( axel );
}
s = conn_url( axel->conn );
strncpy( axel->url->text, s, MAX_STRING );
if( ( axel->size = axel->conn[0].size ) != INT_MAX )
{
if( axel->conf->verbose > 0 )
axel_message( axel, _("File size: %lld bytes"), axel->size );
}
/* Wildcards in URL --> Get complete filename */
if( strchr( axel->filename, '*' ) || strchr( axel->filename, '?' ) )
strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
return( axel );
}
/* Main 'loop' */
void axel_do( axel_t *axel )
{
fd_set fds[1];
int hifd, i, j;
long long int size;
struct timeval timeval[1];
/* Create statefile if necessary */
if( gettime() > axel->next_state )
{
save_state( axel );
axel->next_state = gettime() + axel->conf->save_state_interval;
}
/* Wait for data on (one of) the connections */
FD_ZERO( fds );
hifd = 0;
for( i = 0; i < axel->conf->num_connections; i ++ )
{
if( axel->conn[i].enabled )
FD_SET( axel->conn[i].fd, fds );
hifd = max( hifd, axel->conn[i].fd );
}
if( hifd == 0 )
{
/* No connections yet. Wait... */
usleep( 100000 );
goto conn_check;
}
else
{
timeval->tv_sec = 0;
timeval->tv_usec = 100000;
/* A select() error probably means it was interrupted
by a signal, or that something else's very wrong... */
if( select( hifd + 1, fds, NULL, NULL, timeval ) == -1 )
{
axel->ready = -1;
return;
}
}
/* Handle connections which need attention */
for( i = 0; i < axel->conf->num_connections; i ++ )
if( axel->conn[i].enabled ) {
if( FD_ISSET( axel->conn[i].fd, fds ) )
{
axel->conn[i].last_transfer = gettime();
size = read( axel->conn[i].fd, buffer, axel->conf->buffer_size );
if( size == -1 )
{
if( axel->conf->verbose )
{
axel_message( axel, _("Error on connection %i! "
"Connection closed"), i );
}
axel->conn[i].enabled = 0;
conn_disconnect( &axel->conn[i] );
continue;
}
else if( size == 0 )
{
if( axel->conf->verbose )
{
/* Only abnormal behaviour if: */
if( axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX )
{
axel_message( axel, _("Connection %i unexpectedly closed"), i );
}
else
{
axel_message( axel, _("Connection %i finished"), i );
}
}
if( !axel->conn[0].supported )
{
axel->ready = 1;
}
axel->conn[i].enabled = 0;
conn_disconnect( &axel->conn[i] );
continue;
}
/* j == Bytes to go */
j = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
if( j < size )
{
if( axel->conf->verbose )
{
axel_message( axel, _("Connection %i finished"), i );
}
axel->conn[i].enabled = 0;
conn_disconnect( &axel->conn[i] );
size = j;
/* Don't terminate, still stuff to write! */
}
/* This should always succeed.. */
lseek( axel->outfd, axel->conn[i].currentbyte, SEEK_SET );
if( write( axel->outfd, buffer, size ) != size )
{
axel_message( axel, _("Write error!") );
axel->ready = -1;
return;
}
axel->conn[i].currentbyte += size;
axel->bytes_done += size;
}
else
{
if( gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout )
{
if( axel->conf->verbose )
axel_message( axel, _("Connection %i timed out"), i );
conn_disconnect( &axel->conn[i] );
axel->conn[i].enabled = 0;
}
} }
if( axel->ready )
return;
conn_check:
/* Look for aborted connections and attempt to restart them. */
for( i = 0; i < axel->conf->num_connections; i ++ )
{
if( !axel->conn[i].enabled && axel->conn[i].currentbyte < axel->conn[i].lastbyte )
{
if( axel->conn[i].state == 0 )
{
conn_set( &axel->conn[i], axel->url->text );
axel->url = axel->url->next;
/* axel->conn[i].local_if = axel->conf->interfaces->text;
axel->conf->interfaces = axel->conf->interfaces->next; */
if( axel->conf->verbose >= 2 )
axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) == 0 )
{
axel->conn[i].state = 1;
axel->conn[i].last_transfer = gettime();
}
else
{
axel_message( axel, _("pthread error!!!") );
axel->ready = -1;
}
}
else
{
if( gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay )
{
pthread_cancel( *axel->conn[i].setup_thread );
axel->conn[i].state = 0;
}
}
}
}
/* Calculate current average speed and finish_time */
axel->bytes_per_second = (int) ( (double) ( axel->bytes_done - axel->start_byte ) / ( gettime() - axel->start_time ) );
axel->finish_time = (int) ( axel->start_time + (double) ( axel->size - axel->start_byte ) / axel->bytes_per_second );
/* Check speed. If too high, delay for some time to slow things
down a bit. I think a 5% deviation should be acceptable. */
if( axel->conf->max_speed > 0 )
{
if( (float) axel->bytes_per_second / axel->conf->max_speed > 1.05 )
axel->delay_time += 10000;
else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) && ( axel->delay_time >= 10000 ) )
axel->delay_time -= 10000;
else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) )
axel->delay_time = 0;
usleep( axel->delay_time );
}
/* Ready? */
if( axel->bytes_done == axel->size )
axel->ready = 1;
}
/* Open a local file to store the downloaded data */
int axel_open( axel_t *axel )
{
int i, fd;
long long int j;
if( axel->conf->verbose > 0 )
axel_message( axel, _("Opening output file %s"), axel->filename );
snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
axel->outfd = -1;
/* Check whether server knows about RESTart and switch back to
single connection download if necessary */
if( !axel->conn[0].supported )
{
axel_message( axel, _("Server unsupported, "
"starting from scratch with one connection.") );
axel->conf->num_connections = 1;
axel->conn = realloc( axel->conn, sizeof( conn_t ) );
axel_divide( axel );
}
else if( ( fd = open( buffer, O_RDONLY ) ) != -1 )
{
read( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
axel->conn = realloc( axel->conn, sizeof( conn_t ) * axel->conf->num_connections );
memset( axel->conn + 1, 0, sizeof( conn_t ) * ( axel->conf->num_connections - 1 ) );
axel_divide( axel );
read( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
for( i = 0; i < axel->conf->num_connections; i ++ )
read( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
axel_message( axel, _("State file found: %lld bytes downloaded, %lld to go."),
axel->bytes_done, axel->size - axel->bytes_done );
close( fd );
if( ( axel->outfd = open( axel->filename, O_WRONLY, 0666 ) ) == -1 )
{
axel_message( axel, _("Error opening local file") );
return( 0 );
}
}
/* If outfd == -1 we have to start from scrath now */
if( axel->outfd == -1 )
{
axel_divide( axel );
if( ( axel->outfd = open( axel->filename, O_CREAT | O_WRONLY, 0666 ) ) == -1 )
{
axel_message( axel, _("Error opening local file") );
return( 0 );
}
/* And check whether the filesystem can handle seeks to
past-EOF areas.. Speeds things up. :) AFAIK this
should just not happen: */
if( lseek( axel->outfd, axel->size, SEEK_SET ) == -1 && axel->conf->num_connections > 1 )
{
/* But if the OS/fs does not allow to seek behind
EOF, we have to fill the file with zeroes before
starting. Slow.. */
axel_message( axel, _("Crappy filesystem/OS.. Working around. :-(") );
lseek( axel->outfd, 0, SEEK_SET );
memset( buffer, 0, axel->conf->buffer_size );
j = axel->size;
while( j > 0 )
{
write( axel->outfd, buffer, min( j, axel->conf->buffer_size ) );
j -= axel->conf->buffer_size;
}
}
}
return( 1 );
}
/* Main 'loop' */
void axel_do(axel_t *axel)
{
#if WIN32
WSAEVENT hEventObject = WSACreateEvent();
DWORD byte;
#else
fd_set fds[1];
struct timeval timeval[1];
int hifd;
#endif
int i;
long long int remaining, size;
/* Create statefile if necessary */
if (axel->next_state < gettime())
{
save_state(axel);
axel->next_state = gettime() + axel->conf->save_state_interval;
}
/* Wait for data on (one of) the connections */
#if !WIN32
FD_ZERO(fds);
hifd = 0;
for (i = 0; i < axel->conf->num_connections; i++)
{
if (axel->conn[i].enabled)
{
FD_SET(axel->conn[i].fd, fds);
}
hifd = max(hifd, axel->conn[i].fd);
}
if (0 == hifd)
{
#ifdef DEBUG
printf("DEBUG no connection yet. Wait...\n");
#endif
/* No connections yet. Wait... */
usleep(100000);
goto conn_check;
}
else
{
timeval->tv_sec = 0;
timeval->tv_usec = 100000;
/* A select() error probably means it was interrupted
by a signal, or that something else's very wrong... */
if (-1 == select(hifd + 1, fds, NULL, NULL, timeval))
{
axel->ready = -1;
return;
}
}
#endif
/* Handle connections which need attention */
for (i = 0; i < axel->conf->num_connections; i++)
{
if (axel->conn[i].enabled)
{
#if WIN32
if (is_readable(axel, axel->conn[i].fd, hEventObject))
#else
if (FD_ISSET(axel->conn[i].fd, fds))
#endif
{
axel->conn[i].last_transfer = gettime();
#if WIN32
memset(buffer, 0, max(MAX_STRING, axel->conf->buffer_size));
size = recv(axel->conn[i].fd, buffer, axel->conf->buffer_size, 0);
#else
size = read(axel->conn[i].fd, buffer, axel->conf->buffer_size);
#endif
#if WIN32
if (SOCKET_ERROR == size)
#else
if (-1 == size)
#endif
{
#if !WIN32
if (axel->conf->verbose)
{
axel_message( axel, _("Error on connection %i! "
"Connection closed"), i );
}
#endif
axel->conn[i].enabled = 0;
conn_disconnect(&axel->conn[i]);
continue;
}
else if (0 == size)
{
if (axel->conf->verbose)
{
/* Only abnormal behaviour if: */
if (axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX)
{
axel_message(axel, _("Connection %i unexpectedly closed"), i);
}
else
{
axel_message(axel, _("Connection %i finished"), i);
}
}
if (!axel->conn[0].supported)
{
axel->ready = 1;
}
axel->conn[i].enabled = 0;
conn_disconnect(&axel->conn[i]);
continue;
}
/* remaining == Bytes to go */
remaining = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
if (remaining < size)
{
if (axel->conf->verbose)
{
axel_message(axel, _("Connection %i finished"), i);
}
axel->conn[i].enabled = 0;
conn_disconnect(&axel->conn[i]);
size = remaining;
/* Don't terminate, still stuff to write! */
}
/* This should always succeed.. */
#if WIN32
SetFilePointer(axel->outfd, axel->conn[i].currentbyte, NULL, FILE_BEGIN);
if (0 == WriteFile(axel->outfd, buffer, size, &byte, NULL))
#else
lseek(axel->outfd, axel->conn[i].currentbyte, SEEK_SET);
if (write(axel->outfd, buffer, size) != size)
#endif
{
axel_message(axel, _("Write error!"));
axel->ready = -1;
return;
}
axel->conn[i].currentbyte += size;
axel->bytes_done += size;
}
else
{
if (gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout)
{
if (axel->conf->verbose)
{
axel_message(axel, _("Connection %i timed out"), i);
}
conn_disconnect(&axel->conn[i]);
axel->conn[i].enabled = 0;
}
}
}
}
if (axel->ready)
{
return;
}
conn_check:
/* Look for aborted connections and attempt to restart them. */
for (i = 0; i < axel->conf->num_connections; i++)
{
if (!axel->conn[i].enabled
&& axel->conn[i].currentbyte < axel->conn[i].lastbyte)
{
if (0 == axel->conn[i].state)
{
// Wait for termination of this thread
#if WIN32
WaitForSingleObject(axel->conn[i].setup_thread, INFINITE);
CloseHandle(axel->conn[i].setup_thread);
#else
pthread_join(*(axel->conn[i].setup_thread), NULL);
#endif
conn_set(&axel->conn[i], axel->url->text);
axel->url = axel->url->next;
if (axel->conf->verbose >= 2)
{
axel_message(axel, _("Connection %i downloading from %s:%i using interface %s"),
i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if);
}
axel->conn[i].state = 1;
#if WIN32
axel->conn[i].setup_thread = CreateThread(NULL, 0, setup_thread_cb, &axel->conn[i], 0, NULL);
if (NULL != axel->conn[i].setup_thread)
#else
if (pthread_create(axel->conn[i].setup_thread, NULL, setup_thread_cb, &axel->conn[i]) == 0)
#endif
{
axel->conn[i].last_transfer = gettime();
}
else
{
axel_message(axel, _("thread error in axel_do!!!"));
axel->ready = -1;
}
}
else
{
if (gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay)
{
#if WIN32
TerminateThread(axel->conn[i].setup_thread, 0);
CloseHandle(axel->conn[i].setup_thread);
#else
pthread_cancel(*axel->conn[i].setup_thread);
#endif
axel->conn[i].state = 0;
}
}
}
}
/* Calculate current average speed and finish_time */
axel->bytes_per_second = (int)((double)(axel->bytes_done - axel->start_byte) / (gettime() - axel->start_time));
axel->finish_time = (int)(axel->start_time + (double)(axel->size - axel->start_byte) / axel->bytes_per_second);
/* Check speed. If too high, delay for some time to slow things
down a bit. I think a 5% deviation should be acceptable. */
if (0 < axel->conf->max_speed)
{
if (1.05 < (float) axel->bytes_per_second / axel->conf->max_speed)
{
axel->delay_time += 10000;
}
else if (((float)axel->bytes_per_second / axel->conf->max_speed < 0.95)
&& 10000 <= (axel->delay_time))
{
axel->delay_time -= 10000;
}
else if (((float)axel->bytes_per_second / axel->conf->max_speed < 0.95))
{
axel->delay_time = 0;
}
usleep(axel->delay_time);
}
/* Ready? */
if (axel->bytes_done == axel->size)
{
axel->ready = 1;
}
}
/* Open a local file to store the downloaded data */
int axel_open(axel_t *axel)
{
int i;
#if WIN32
HANDLE fd;
DWORD byte;
#else
int fd;
#endif
long long int j;
if (0 < axel->conf->verbose)
{
axel_message(axel, _("Opening output file %s"), axel->filename);
}
snprintf(buffer, MAX_STRING, "%s.st", axel->filename);
#if WIN32
axel->outfd = INVALID_HANDLE_VALUE;
#else
axel->outfd = -1;
#endif
/* Check whether server knows about RESTart and switch back to
single connection download if necessary */
if (!axel->conn[0].supported)
{
axel_message(axel, _("Server unsupported, "
"starting from scratch with one connection."));
axel->conf->num_connections = 1;
axel->conn = realloc(axel->conn, sizeof(conn_t));
axel_divide(axel);
}
#if WIN32
else if (INVALID_HANDLE_VALUE != (fd = CreateFile(buffer,
GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0)))
#else
else if ((fd = open(buffer, O_RDONLY)) != -1)
#endif
{
#if WIN32
ReadFile(fd, &axel->conf->num_connections, sizeof(axel->conf->num_connections), &byte, NULL);
#else
read(fd, &axel->conf->num_connections, sizeof(axel->conf->num_connections));
#endif
axel->conn = realloc(axel->conn, sizeof(conn_t) * axel->conf->num_connections);
memset(axel->conn + 1, 0, sizeof(conn_t) * (axel->conf->num_connections - 1));
axel_divide(axel);
#if WIN32
ReadFile(fd, &axel->bytes_done, sizeof(axel->bytes_done), &byte, NULL);
#else
read(fd, &axel->bytes_done, sizeof(axel->bytes_done));
#endif
for (i = 0; i < axel->conf->num_connections; i++)
{
#if WIN32
ReadFile(fd, &axel->conn[i].currentbyte, sizeof(axel->conn[i].currentbyte), &byte, NULL);
#else
read(fd, &axel->conn[i].currentbyte, sizeof(axel->conn[i].currentbyte));
#endif
}
axel_message(axel, _("State file found: %lld bytes downloaded, %lld to go."),
axel->bytes_done, axel->size - axel->bytes_done);
#if WIN32
CloseHandle(fd);
#else
close(fd);
#endif
#if WIN32
if (INVALID_HANDLE_VALUE == (axel->outfd = CreateFile(axel->filename,
GENERIC_WRITE, FILE_SHARE_READ, 0, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0)))
#else
if ((axel->outfd = open(axel->filename, O_WRONLY, 0666)) == -1)
#endif
{
axel_message(axel, _("Error opening local file"));
return 0;
}
}
/* If outfd == -1 we have to start from scrath now */
#if WIN32
if (INVALID_HANDLE_VALUE == axel->outfd)
#else
if (axel->outfd == -1)
#endif
{
axel_divide(axel);
#if WIN32
if (INVALID_HANDLE_VALUE == (axel->outfd = CreateFile(axel->filename,
GENERIC_WRITE, FILE_SHARE_READ, 0, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0)))
#else
if ((axel->outfd = open(axel->filename, O_CREAT | O_WRONLY, 0666)) == -1)
#endif
{
axel_message(axel, _("Error opening local file"));
return 0;
}
/* And check whether the filesystem can handle seeks to
past-EOF areas.. Speeds things up. :) AFAIK this
should just not happen: */
#if WIN32
if (INVALID_SET_FILE_POINTER == SetFilePointer(axel->outfd, axel->size, NULL, FILE_BEGIN)
&& 1 < axel->conf->num_connections)
#else
if (lseek(axel->outfd, axel->size, SEEK_SET) == -1 && axel->conf->num_connections > 1)
#endif
{
/* But if the OS/fs does not allow to seek behind
EOF, we have to fill the file with zeroes before
starting. Slow.. */
axel_message(axel, _("Crappy filesystem/OS.. Working around. :-("));
#if WIN32
SetFilePointer(axel->outfd, 0, NULL, FILE_BEGIN);
#else
lseek(axel->outfd, 0, SEEK_SET);
#endif
memset(buffer, 0, axel->conf->buffer_size);
j = axel->size;
while (0 < j)
{
#if WIN32
WriteFile(axel->outfd, buffer, min(j, axel->conf->buffer_size), &byte, NULL);
#else
write(axel->outfd, buffer, min(j, axel->conf->buffer_size));
#endif
j -= axel->conf->buffer_size;
}
}
}
return 1;
}
