static int extract_authentication_data(struct curl_slist *headers, struct auth_data * data) {
int found_data = 0;
char *authorization_header = NULL;
int keysize = strlen("Authorization:");
if ((authorization_header = get_header(headers, "Authorization:", keysize))) {
regex_t regex;
regmatch_t matches[3];
if (regcomp(®ex, "AuthHMAC ([^:]+):([A-Za-z0-9+/=]+)$", REG_EXTENDED)) {
fatal("Error compiling regex");
return -1;
}
if (0 == regexec(®ex, authorization_header, 3, matches, 0)) {
char * access_id = extract_match(&matches[1], authorization_header);
char * signature = extract_match(&matches[2], authorization_header);
if (access_id && signature) {
data->access_id = access_id;
data->signature = signature;
found_data = 1;
}
}
regfree(®ex);
}
return found_data;
}
static word fetch_file(void)
{
// Returns 0=Success with relevant file open on fp_result
// Or !0 = failure and file is closed.
// DANGER: In failure case, fp_result is INVALID and may not be null.
char zs[256], filepathz[256], filepath[256], url[256];
time_t now, when;
struct tm * broken;
static char * weekdays[] = { "sun", "mon", "tue", "wed", "thu", "fri", "sat" };
stats[Fetches]++;
now = time(NULL);
if(!opt_filename)
{
static CURL * curlh;
struct curl_slist * slist;
if(!(curlh = curl_easy_init()))
{
_log(CRITICAL, "fetch_file(): Failed to obtain libcurl easy handle.");
return 1;
}
curl_easy_setopt(curlh, CURLOPT_WRITEFUNCTION, cif_write_data);
slist = NULL;
slist = curl_slist_append(slist, "Cache-Control: no-cache");
if(!slist)
{
_log(MAJOR,"fetch_file(): Failed to create slist.");
return 1;
}
// Build URL
when = now - 24*60*60;
broken = localtime(&when); // Note broken contains "yesterday"
if(opt_url)
{
strcpy(url, opt_url);
}
else
{
if(fetch_all)
{
sprintf(url, "https://datafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full");
}
else
{
sprintf(url, "https://datafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-%s", weekdays[broken->tm_wday]);
}
}
sprintf(zs, "Fetching \"%s\".", url);
_log(GENERAL, zs);
if(opt_url || debug)
{
sprintf(filepathz, "/tmp/tscdb-cif-fetch-%ld.gz", now);
sprintf(filepath, "/tmp/tscdb-cif-fetch-%ld", now);
}
else if(fetch_all)
{
sprintf(filepathz, "/tmp/tscdb-cif-all-%02d-%02d-%02d-%s.gz", broken->tm_year%100, broken->tm_mon + 1, broken->tm_mday, weekdays[broken->tm_wday]);
sprintf(filepath, "/tmp/tscdb-cif-all-%02d-%02d-%02d-%s", broken->tm_year%100, broken->tm_mon + 1, broken->tm_mday, weekdays[broken->tm_wday]);
}
else
{
sprintf(filepathz, "/tmp/tscdb-cif-%02d-%02d-%02d-%s.gz", broken->tm_year%100, broken->tm_mon + 1, broken->tm_mday, weekdays[broken->tm_wday]);
sprintf(filepath, "/tmp/tscdb-cif-%02d-%02d-%02d-%s", broken->tm_year%100, broken->tm_mon + 1, broken->tm_mday, weekdays[broken->tm_wday]);
}
if(!(fp_result = fopen(filepathz, "w")))
{
sprintf(zs, "Failed to open \"%s\" for writing.", filepathz);
_log(MAJOR, zs);
return 1;
}
curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, slist);
// Set timeouts
curl_easy_setopt(curlh, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curlh, CURLOPT_FTP_RESPONSE_TIMEOUT, 128L);
curl_easy_setopt(curlh, CURLOPT_TIMEOUT, 128L);
curl_easy_setopt(curlh, CURLOPT_CONNECTTIMEOUT, 128L);
// Debugging prints.
// curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L);
// URL and login
curl_easy_setopt(curlh, CURLOPT_URL, url);
sprintf(zs, "%s:%s", conf[conf_nr_user], conf[conf_nr_password]);
curl_easy_setopt(curlh, CURLOPT_USERPWD, zs);
curl_easy_setopt(curlh, CURLOPT_FOLLOWLOCATION, 1L); // On receiving a 3xx response, follow the redirect.
total_bytes = 0;
CURLcode result;
if((result = curl_easy_perform(curlh)))
{
_log(MAJOR, "fetch_file(): curl_easy_perform() returned error %d: %s.", result, curl_easy_strerror(result));
if(opt_insecure && (result == 51 || result == 60))
{
_log(MAJOR, "Retrying download in insecure mode.");
// SSH failure, retry without
curl_easy_setopt(curlh, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curlh, CURLOPT_SSL_VERIFYHOST, 0L);
used_insecure = true;
if((result = curl_easy_perform(curlh)))
{
_log(MAJOR, "fetch_file(): In insecure mode curl_easy_perform() returned error %d: %s.", result, curl_easy_strerror(result));
if(fp_result) fclose(fp_result);
fp_result = NULL;
return 1;
}
}
else
{
if(fp_result) fclose(fp_result);
fp_result = NULL;
return 1;
}
}
char * actual_url;
if(!curl_easy_getinfo(curlh, CURLINFO_EFFECTIVE_URL, &actual_url) && actual_url)
{
_log(GENERAL, "Download was redirected to \"%s\".", actual_url);
}
if(fp_result) fclose(fp_result);
fp_result = NULL;
if(curlh) curl_easy_cleanup(curlh);
curlh = NULL;
if(slist) curl_slist_free_all(slist);
slist = NULL;
sprintf(zs, "Received %s bytes of compressed CIF updates.", commas(total_bytes));
_log(GENERAL, zs);
if(total_bytes == 0) return 1;
_log(GENERAL, "Decompressing data...");
sprintf(zs, "/bin/gunzip -f %s", filepathz);
char * rc;
if((rc = system_call(zs)))
{
_log(MAJOR, "Failed to uncompress file: %s", rc);
if((fp_result = fopen(filepathz, "r")))
{
char error_message[2048];
size_t length;
if((length = fread(error_message, 1, 2047, fp_result)) && error_message[0] == '<')
{
error_message[length] = '\0';
_log(MAJOR, "Received message:\n%s", error_message);
}
fclose(fp_result);
}
return 1;
}
_log(GENERAL, "Decompressed.");
}
else
{
strcpy(filepath, opt_filename);
}
if(!(fp_result = fopen(filepath, "r")))
{
_log(MAJOR, "Failed to open \"%s\" for reading.", filepath);
return 1;
}
// Check if it's really an update
{
sprintf(zs, "grep -q \\\"Delete\\\" %s", filepath);
word not_update = system(zs);
if(not_update && (total_bytes < 32000000L)) not_update = false;
_log(test_mode?GENERAL:DEBUG, "File is an update assessment: %s.", not_update?"File is not an update":"File is an update");
if(fetch_all && !not_update)
{
_log(MAJOR, "Requested full timetable looks like an update.");
fclose(fp_result);
return 1;
}
if(!fetch_all && not_update)
{
_log(MAJOR, "Requested update looks like a full timetable.");
fclose(fp_result);
return 1;
}
}
// Read enough of the file to find the datestamp
char front[256];
regmatch_t matches[3];
if(!fread(front, 1, sizeof(front), fp_result))
{
fclose(fp_result);
return 1;
}
else
{
front[255] = '\0';
if(regexec(&match[0], front, 2, matches, 0))
{
// Failed
_log(MAJOR, "Failed to derive CIF file timestamp.");
fclose(fp_result);
return 1;
}
else
{
char zstamp[256];
extract_match(front, matches, 1, zstamp, sizeof(zstamp));
time_t stamp = atoi(zstamp);
_log(test_mode?GENERAL:DEBUG, "Recovered timestamp: %s", time_text(stamp, 0));
_log(test_mode?GENERAL:DEBUG, "Stored in file \"%s\"", filepath);
if(regexec(&match[1], front, 2, matches, 0))
{
_log(MINOR, "Failed to derive CIF file sequence number.");
}
else
{
extract_match(front, matches, 1, zstamp, sizeof(zstamp));
stats[Sequence] = atol(zstamp);
_log(test_mode?GENERAL:DEBUG, "Sequence number: %s", commas_q(stats[Sequence]));
}
if(!test_mode && !opt_url && !opt_filename && (now < stamp || now - stamp > 36*60*60))
{
_log(MAJOR, "Timestamp %s is incorrect. Received sequence number %d.", time_text(stamp, true), stats[Sequence]);
fclose(fp_result);
stats[Sequence] = 0;
return 1;
}
}
}
fseeko(fp_result, 0, SEEK_SET);
return 0;
}