static GTree* service_filegetter_import_download_specs(service_filegetter_tp sfg, service_filegetter_multi_args_tp args) { /* reads file with lines of the form: * fileserver.shd:8080:/5mb.urnd */ if(args->server_specification_filepath == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "please specify a path to a download specification file"); return NULL; } FILE* specs = fopen(args->server_specification_filepath, "r"); if(specs == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "could not open file"); perror(args->server_specification_filepath); return NULL; } gchar linebuffer[512]; GTree* dlTree = g_tree_new_full(_treeIntCompare, NULL, g_free, g_free); gint counter = 0; while(fgets(linebuffer, sizeof(linebuffer), specs) != NULL) { /* strip off the newline */ if (linebuffer[strlen(linebuffer) - 1] == '\n') { linebuffer[strlen(linebuffer) - 1] = '\0'; } gchar** tokens = g_strsplit((const gchar*) linebuffer, (const gchar*)":", 3); if(g_strrstr(tokens[2], ":") != NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "format of download specification file incorrect. expected something like \"fileserver.shd:8080:/5mb.urnd\" on each line"); g_strfreev(tokens); g_tree_destroy(dlTree); return NULL; } service_filegetter_server_args_t http; http.host = tokens[0]; http.port = tokens[1]; gchar* filepath = tokens[2]; service_filegetter_download_tp dl = service_filegetter_get_download_from_args(sfg, &http, &args->socks_proxy, filepath, args->hostbyname_cb); g_strfreev(tokens); if(dl == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "error parsing download specification file"); g_tree_destroy(dlTree); return NULL; } gint* key = g_new(gint, 1); *key = counter++; g_tree_insert(dlTree, key, dl); } fclose(specs); return dlTree; }
static service_filegetter_download_tp service_filegetter_get_download_from_args( service_filegetter_tp sfg, service_filegetter_server_args_tp http_server, service_filegetter_server_args_tp socks_proxy, gchar* filepath, service_filegetter_hostbyname_cb hostbyname_cb) { /* absolute file path to get from server */ if(filepath[0] != '/') { service_filegetter_log(sfg, SFG_CRITICAL, "filepath %s does not begin with '/'", filepath); return NULL; } /* we require http info */ if(http_server == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "no HTTP server specified"); return NULL; } GString* strbuf = g_string_new(http_server->host); gint hostlength = strbuf->len; gboolean isOnionAddress = g_strstr_len(strbuf->str, strbuf->len, ".onion") ? TRUE : FALSE; g_string_free(strbuf, TRUE); in_addr_t http_addr = 0; if(!isOnionAddress) { http_addr = service_filegetter_getaddr(sfg, http_server, hostbyname_cb); } in_port_t http_port = htons((in_port_t) atoi(http_server->port)); if((!isOnionAddress && http_addr == 0) || http_port == 0) { service_filegetter_log(sfg, SFG_CRITICAL, "HTTP server specified but 0"); return NULL; } /* there may not be a socks proxy, so NULL is ok */ in_addr_t socks_addr = 0; in_port_t socks_port = 0; if(socks_proxy != NULL) { socks_addr = service_filegetter_getaddr(sfg, socks_proxy, hostbyname_cb); socks_port = htons((in_port_t) atoi(socks_proxy->port)); } if(isOnionAddress && !socks_addr) { service_filegetter_log(sfg, SFG_WARNING, "it probably wont work to specify an .onion address without a Tor socks proxy"); return NULL; } /* validation successful */ service_filegetter_download_tp dl = calloc(1, sizeof(service_filegetter_download_t)); strncpy(dl->fspec.remote_path, filepath, sizeof(dl->fspec.remote_path)); strncpy(dl->sspec.http_hostname, http_server->host, sizeof(dl->sspec.http_hostname)); dl->sspec.http_addr = http_addr; dl->sspec.http_port = http_port; dl->sspec.socks_addr = socks_addr; dl->sspec.socks_port = socks_port; dl->sspec.useHostname = isOnionAddress; dl->sspec.hostnameLength = hostlength; return dl; }
enum filegetter_code service_filegetter_start_multi(service_filegetter_tp sfg, service_filegetter_multi_args_tp args, gint epolld, gint* sockd_out) { assert(sfg); assert(args); memset(sfg, 0, sizeof(service_filegetter_t)); sfg->type = SFG_MULTI; sfg->state = SFG_NONE; /* if null, we ignore logging */ sfg->log_cb = args->log_cb; /* not required if they only give us addresses. we complain later if needed */ sfg->hostbyname_cb = args->hostbyname_cb; sfg->sleep_cb = args->sleep_cb; if(sfg->sleep_cb == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "sleep callback function required"); return FG_ERR_INVALID; } if(args->thinktimes_cdf_filepath != NULL) { sfg->think_times = cdf_new(0, args->thinktimes_cdf_filepath); if(sfg->think_times == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "problem importing thinktime cdf."); return FG_ERR_INVALID; } } sfg->downloads = service_filegetter_import_download_specs(sfg, args); if(sfg->downloads == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "problem parsing server download specification file. is the format correct?"); cdf_free(sfg->think_times); return FG_ERR_INVALID; } gint runtime_seconds = atoi(args->runtime_seconds); if(runtime_seconds > 0) { clock_gettime(CLOCK_REALTIME, &sfg->expire); sfg->expire.tv_sec += runtime_seconds; } if(args->num_downloads) { sfg->downloads_requested = atoi(args->num_downloads); } return service_filegetter_launch(sfg, epolld, sockd_out); }
enum filegetter_code service_filegetter_stop(service_filegetter_tp sfg) { assert(sfg); service_filegetter_log(sfg, SFG_INFO, "shutting down filegetter"); enum filegetter_code result = FG_SUCCESS; if(sfg->think_times != NULL) { cdf_free(sfg->think_times); sfg->think_times = NULL; } if(sfg->downloads != NULL) { g_tree_destroy(sfg->downloads); sfg->downloads = NULL; } if(sfg->state != SFG_DONE) { result = filegetter_shutdown(&sfg->fg); sfg->current_download = NULL; sfg->state = SFG_DONE; } return result; }
enum filegetter_code service_filegetter_start_single(service_filegetter_tp sfg, service_filegetter_single_args_tp args, gint epolld, gint* sockd_out) { assert(sfg); assert(args); memset(sfg, 0, sizeof(service_filegetter_t)); sfg->type = SFG_SINGLE; sfg->state = SFG_NONE; /* if null, we ignore logging */ sfg->log_cb = args->log_cb; sfg->hostbyname_cb = args->hostbyname_cb; sfg->sleep_cb = args->sleep_cb; /* we download a single file, store our specification in current */ sfg->current_download = service_filegetter_get_download_from_args(sfg, &args->http_server, &args->socks_proxy, args->filepath, args->hostbyname_cb); if(sfg->current_download == NULL) { return FG_ERR_INVALID; } sfg->downloads_requested = atoi(args->num_downloads); if(sfg->downloads_requested <= 0) { service_filegetter_log(sfg, SFG_WARNING, "you didn't want to download anything?"); return FG_ERR_INVALID; } return service_filegetter_launch(sfg, epolld, sockd_out); }
static enum filegetter_code service_filegetter_download_next(service_filegetter_tp sfg) { assert(sfg); switch (sfg->type) { case SFG_MULTI: { /* get a new random download */ const gint position = (gint) (rand() % g_tree_nnodes(sfg->downloads)); sfg->current_download = g_tree_lookup(sfg->downloads, &position); if(sfg->current_download == NULL) { return FG_ERR_INVALID; } /* follow through to set the download */ } case SFG_SINGLE: { enum filegetter_code result = filegetter_download(&sfg->fg, &sfg->current_download->sspec, &sfg->current_download->fspec); service_filegetter_log(sfg, SFG_DEBUG, "filegetter set specs code: %s", filegetter_codetoa(result)); if(result == FG_SUCCESS) { sfg->state = SFG_DOWNLOADING; } return result; } default: { return FG_ERR_INVALID; } } }
static service_filegetter_download_tp service_filegetter_get_download_from_args( service_filegetter_tp sfg, service_filegetter_server_args_tp http_server, service_filegetter_server_args_tp socks_proxy, gchar* filepath, service_filegetter_hostbyname_cb hostbyname_cb) { /* absolute file path to get from server */ if(filepath[0] != '/') { service_filegetter_log(sfg, SFG_CRITICAL, "filepath %s does not begin with '/'", filepath); return NULL; } /* we require http info */ if(http_server == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "no HTTP server specified"); return NULL; } in_addr_t http_addr = service_filegetter_getaddr(sfg, http_server, hostbyname_cb); in_port_t http_port = htons((in_port_t) atoi(http_server->port)); if(http_addr == 0 || http_port == 0) { service_filegetter_log(sfg, SFG_CRITICAL, "HTTP server specified but 0"); return NULL; } /* there may not be a socks proxy, so NULL is ok */ in_addr_t socks_addr = 0; in_port_t socks_port = 0; if(socks_proxy != NULL) { socks_addr = service_filegetter_getaddr(sfg, socks_proxy, hostbyname_cb); socks_port = htons((in_port_t) atoi(socks_proxy->port)); } /* validation successful */ service_filegetter_download_tp dl = calloc(1, sizeof(service_filegetter_download_t)); strncpy(dl->fspec.remote_path, filepath, sizeof(dl->fspec.remote_path)); strncpy(dl->sspec.http_hostname, http_server->host, sizeof(dl->sspec.http_hostname)); dl->sspec.http_addr = http_addr; dl->sspec.http_port = http_port; dl->sspec.socks_addr = socks_addr; dl->sspec.socks_port = socks_port; return dl; }
static enum filegetter_code service_filegetter_launch(service_filegetter_tp sfg, gint epolld, gint* sockd_out) { /* inputs should be ok, start up the client */ enum filegetter_code result = filegetter_start(&sfg->fg, epolld); service_filegetter_log(sfg, SFG_DEBUG, "filegetter startup code: %s", filegetter_codetoa(result)); /* set our download specifications */ result = service_filegetter_download_next(sfg); if(result == FG_SUCCESS && sockd_out != NULL) { /* ready to activate */ *sockd_out = sfg->fg.sockd; } return result; }
static in_addr_t service_filegetter_getaddr(service_filegetter_tp sfg, service_filegetter_server_args_tp server, service_filegetter_hostbyname_cb hostname_cb) { /* check if we have an address as a string */ struct in_addr in; gint is_ip_address = inet_aton(server->host, &in); if(is_ip_address) { return in.s_addr; } else { /* its a hostname, they better have given us the callback */ if(hostname_cb == NULL) { service_filegetter_log(sfg, SFG_CRITICAL, "need to do an address lookup for %s, but the hostbyname callback is NULL", server->host); return INADDR_NONE; } return (*hostname_cb)(server->host); } }
static void service_filegetter_report(service_filegetter_tp sfg, enum service_filegetter_loglevel level, gchar* preamble, filegetter_filestats_tp stats, gint current_download, gint total_downloads) { if(preamble != NULL && stats != NULL) { GString* reportStringBuffer = g_string_new(""); g_string_printf(reportStringBuffer, "%s got first bytes in %lu.%.3d seconds and %zu of %zu bytes in %lu.%.3d seconds (download %i", preamble, stats->first_byte_time.tv_sec, (gint)(stats->first_byte_time.tv_nsec / 1000000), stats->body_bytes_downloaded, stats->body_bytes_expected, stats->download_time.tv_sec, (gint) (stats->download_time.tv_nsec / 1000000), current_download); if(total_downloads > 0) { g_string_append_printf(reportStringBuffer, " of %i)", total_downloads); } else { g_string_append_printf(reportStringBuffer, ")"); } service_filegetter_log(sfg, level, "%s", reportStringBuffer->str); g_string_free(reportStringBuffer, TRUE); } }
enum filegetter_code service_filegetter_activate(service_filegetter_tp sfg, gint sockd) { assert(sfg); start_over: if((sfg->state == SFG_THINKING || sfg->state == SFG_DOWNLOADING) && sfg->expire.tv_sec > 0) { /* they set a service expiration, check if we have expired */ struct timespec now; clock_gettime(CLOCK_REALTIME, &now); if(now.tv_sec > sfg->expire.tv_sec) { return service_filegetter_expire(sfg); } } if(sfg->state == SFG_THINKING) { struct timespec now; clock_gettime(CLOCK_REALTIME, &now); if(now.tv_sec >= sfg->wakeup.tv_sec) { /* time to wake up and download the next file */ service_filegetter_download_next(sfg); } else { return FG_ERR_WOULDBLOCK; } } if(sfg->state != SFG_DOWNLOADING || sfg->fg.sockd != sockd) { return FG_ERR_INVALID; } reactivate:; enum filegetter_code result = filegetter_activate(&sfg->fg); if(result == FG_ERR_FATAL || result == FG_ERR_SOCKSCONN) { /* it had to shut down */ service_filegetter_log(sfg, SFG_NOTICE, "filegetter shutdown due to error '%s'... retrying in 60 seconds", filegetter_codetoa(result)); filegetter_shutdown(&sfg->fg); filegetter_start(&sfg->fg, sfg->fg.epolld); /* set wakeup timer and call the sleep function */ sfg->state = SFG_THINKING; clock_gettime(CLOCK_REALTIME, &sfg->wakeup); sfg->wakeup.tv_sec += 60; (*sfg->sleep_cb)(sfg, 60); service_filegetter_log(sfg, SFG_NOTICE, "[fg-pause] pausing for 60 seconds"); return FG_ERR_WOULDBLOCK; } else if(result != FG_OK_200 && result != FG_ERR_WOULDBLOCK) { service_filegetter_log(sfg, SFG_CRITICAL, "filegetter shutdown due to protocol error '%s'...", filegetter_codetoa(result)); filegetter_shutdown(&sfg->fg); return result; } /* report progress */ filegetter_filestats_t stats; filegetter_stat_download(&sfg->fg, &stats); service_filegetter_report(sfg, SFG_INFO, "[fg-download-progress]", &stats, sfg->downloads_completed+1, sfg->downloads_requested); if(result == FG_OK_200) { /* completed a download */ sfg->downloads_completed++; sfg->state = SFG_THINKING; /* report completion stats */ service_filegetter_report(sfg, SFG_NOTICE, "[fg-download-complete]", &stats, sfg->downloads_completed, sfg->downloads_requested); if(sfg->downloads_requested > 0 && sfg->downloads_completed >= sfg->downloads_requested) { return service_filegetter_expire(sfg); } else { if(sfg->type == SFG_MULTI && sfg->think_times != NULL) { /* get think time and set wakeup timer */ gdouble percentile = (gdouble)(((gdouble)rand()) / ((gdouble)RAND_MAX)); guint sleeptime = (guint) (cdf_getValue(sfg->think_times, percentile) / 1000); clock_gettime(CLOCK_REALTIME, &sfg->wakeup); sfg->wakeup.tv_sec += sleeptime; /* dont sleep if it would put us beyond our expiration (if its set) */ if(sfg->expire.tv_sec > 0 && sfg->wakeup.tv_sec > sfg->expire.tv_sec) { return service_filegetter_expire(sfg); } /* call the sleep function, then check if we are done thinking */ (*sfg->sleep_cb)(sfg, sleeptime); goto start_over; } else { /* reset download file */ service_filegetter_download_next(sfg); goto reactivate; } } } return result; }