strlist *readfile(char *filename) {
int fd, n, i;
char line[MAXLEN];
strlist *data = strlist_new();
strlist *ret = data;
INFO("Opening file %s\n", filename);
fd = open(filename, O_RDONLY);
if (fd < 0) {
ERROR("ERROR: could not open %s\n", filename);
IFERROR(perror(""));
return NULL;
}
i = 0; // TODO make sure this doesn't overflow line
while ((n = read(fd, &line[i], 1)) > 0) {
if (line[i] == '\n') {
line[i+1] = '\0';
data = strlist_add(data, line);
i = 0;
continue;
}
i++;
}
close(fd);
return ret;
}
response *make_404_response() {
char *path = "404.html", *html;
response *resp = response_new();
strlist *filedata = strlist_new();
response_set_status_code(resp, 404);
response_set_content_type(resp, "text/html");
html = "<html><head><title>Not Found</title></head><body> \
<h1>404!</h1><br>Sorry, the object you requested was not found. \
</body><html>";
filedata = readfile(path);
if (filedata) {
html = strlist_to_string(filedata);
}
//INFO("Read HTML:\n%s", html);
response_set_body(resp, html);
//INFO("Generated response:\n");
//IFINFO(response_write(resp, STDOUT_FILENO));
return resp;
}
static const char *wlist(const char *s)
{
const char *res = "FAIL";
struct StrList *sl = strlist_new(USUAL_ALLOC);
bool ok = parse_word_list(s, sl_add, sl);
if (ok) {
if (strlist_empty(sl))
res = "-";
else
res = lshow(sl);
}
strlist_free(sl);
return res;
}
void split(strlist_t **tokens, char *buf, char *sep,
char *(*tokenizer)(char *, const char*, char **)) {
char *strptr = NULL;
char *tokctx;
char *dupbuf = NULL;
char *tok;
dupbuf = strdup(buf);
strptr = dupbuf;
*tokens = strlist_new();
//printf("Split: '%s' on '%s'\n", buf, sep);
while ((tok = tokenizer(strptr, sep, &tokctx)) != NULL) {
strptr = NULL;
strlist_append(*tokens, tok);
}
free(dupbuf);
}
static void test_strlist(void *p)
{
struct StrList *sl = NULL;
const char *s;
sl = strlist_new(USUAL_ALLOC);
str_check(lshow(sl), "");
strlist_append(sl, "1");
str_check(lshow(sl), "1");
strlist_append(sl, "2");
str_check(lshow(sl), "1,2");
strlist_append(sl, "3");
str_check(lshow(sl), "1,2,3");
s = strlist_pop(sl);
str_check(s, "1");
free(s);
strlist_append(sl, NULL);
str_check(lshow(sl), "2,3,NULL");
strlist_free(sl);
end:;
}
char *extract(char *format, char *buf) {
char *sep = NULL;
char *buffer = strdup(buf);
int nbuffer = 0;
int buffer_size = 1024;
char *(*tokenizer)(char *, const char*, char **);
/* strdup() because string literals aren't writable */
sep = strdup(" ");
/* If first char is not a number or '{', use it to split instead of the
* default of space */
if (!isdigit(format[0]) && (format[0] != '{') && (format[0] != '-')) {
sep[0] = format[0];
format++;
}
//printf("extract: %s\n", format);
while (format[0] != '\0') {
strlist_t *tokens;
strlist_t *fields;
strlist_t *results;
char *fieldstr;
tokenizer = tokenizer_greedy;
results = strlist_new();
/* All of these cases will advance the format string position */
/* This if case is a reallly lame hack */
//printf("%s\n", format);
if (isdigit(format[0]) || (format[0] == '-' && isdigit(format[1]))) {
asprintf(&fieldstr, "%ld", strtol(format, &format, 10));
} else if (format[0] == '{') {
int fieldlen;
format++; /* Skip '{' */
if (format[0] == '?') {
//printf("Nongreedy\n");
format++;
tokenizer = tokenizer_nongreedy;
}
fieldlen = strcspn(format, "}") + 1;
if (format[fieldlen - 1] != '}') {
fprintf(stderr, "Could not find closing '}'. Bad format: %s\n",
(format - 1));
exit(1);
}
fieldstr = malloc(fieldlen * sizeof(char));
memset(fieldstr, 0, fieldlen * sizeof(char));
strncpy(fieldstr, format, fieldlen - 1);
format += fieldlen;
} else {
/* Error, this format is invalid? */
fprintf(stderr, "Invalid format... %s\n", format);
exit(1);
}
/* Split the input by sep using tokenizer */
split(&tokens, buffer, sep, tokenizer);
/* fieldstr is the field selector(s). ie; "1,3" in a{1,3} */
split(&fields, fieldstr, ",", tokenizer_greedy);
free(fieldstr);
int i = 0;
//printf("\n");
//printf("nfields selected: %d", fields->nitems);
for (i = 0; i < fields->nitems; i++) {
long start, end;
strlist_t *range;
char *field = fields->items[i];
split(&range, field, ":", tokenizer_greedy);
if (range->nitems == 1) {
/* Support {N} and {N:} */
start = end = strtol(range->items[0], NULL, 10);
/* Support {:N} */
if (field[strlen(field) - 1] == ':')
end = (start > 0) ? tokens->nitems : 0;
/* Support {N:} */
if (field[0] == ':')
start = 1;
} else if (*field == ':') {
/* Support {:} as whole all fields */
start = 0;
end = 0;
} else {
start = strtol(range->items[0], NULL, 10);
end = strtol(range->items[1], NULL, 10);
}
int j;
/* Add 1 here because field indexing starts at 1, not 0 */
if (start < 0) {
start += tokens->nitems + 1;
/* For sanity, negative indexing shouldn't result in <= 0 values. */
/* XXX: Throw an error for a bad index? */
if (start < 0)
start = 1;
}
if (end < 0) {
end += tokens->nitems + 1;
if (end < 0)
end = start;
}
//printf("s/e= %ld %ld\n", start, end);
/* If end is 0, and set end to start. End of 0 doesn't make sense */
if (end == 0)
end = start;
//printf("%ld %ld\n", start, end);
if (start > end) {
fprintf(stderr, "start > end is invalid: %ld > %ld\n", start, end);
exit(1);
}
if (((start == 0) && (end != 0))
|| ((start != 0) && (end == 0))) {
fprintf(stderr,
"Start or end cannot be 0 when the other is not 0: %ld "
"and %ld\n", start, end);
exit(1);
}
/* We start indexing at 1. */
if (start == 0) {
strlist_append(results, buffer);
} else {
start--;
end--;
for (j = start; j < tokens->nitems && j <= end; j++) {
strlist_append(results, tokens->items[j]);
}
}
strlist_free(range);
}
/* Free buffer then allocate a new one for the new string slice */
free(buffer);
buffer_size = 1024;
nbuffer = 0;
buffer = malloc(buffer_size);
memset(buffer, 0, buffer_size);
for (i = 0; i < results->nitems; i++) {
char *item = results->items[i];
int len = strlen(item);
if (len + nbuffer > buffer_size) {
buffer_size = buffer_size * 2 + len + 1;
buffer = realloc(buffer, buffer_size);
}
strncpy(buffer + nbuffer, item, len);
nbuffer += len;
if (i < results->nitems - 1) {
buffer[nbuffer] = *sep;
nbuffer++;
}
}
if (format[0] != '\0') {
sep[0] = format[0];
format++;
}
strlist_free(fields);
strlist_free(tokens);
strlist_free(results);
}
free(sep);
return buffer;
}
struct strlist *
pickup_random_files ()
{
char *url;
char *year;
char *yday;
char *station;
int i, n;
struct strlist *tmp, *result;
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/");
tmp = parse_apache_index (url);
free (url);
if (tmp->strings_count == 0)
return tmp;
n = (tmp->strings_count - 1) * ((float) rand () / (float) RAND_MAX);
station = xstrdup (tmp->strings_list[n]);
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/%s/individual/",
station);
tmp = parse_apache_index (url);
free (url);
if (tmp->strings_count == 0)
{
free (station);
return tmp;
}
n = (tmp->strings_count - 1) * ((float) rand () / (float) RAND_MAX);
year = xstrdup (tmp->strings_list[n]);
strlist_destroy (tmp);
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/%s/individual/%s/",
station, year);
tmp = parse_apache_index (url);
free (url);
if (tmp->strings_count == 0)
{
free (station);
free (year);
return tmp;
}
n = (tmp->strings_count - 1) * ((float) rand () / (float) RAND_MAX);
yday = xstrdup (tmp->strings_list[n]);
strlist_destroy (tmp);
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/%s/individual/%s/%s/scaled/",
station, year, yday);
free (yday);
free (year);
free (station);
tmp = parse_apache_index (url);
free (url);
result = strlist_new ();
for (i = 0; i < tmp->strings_count; i++)
{
if (strcmp (&tmp->strings_list[i][strlen (tmp->strings_list[i]) - 3], "SAO") == 0)
strlist_append_string (result, tmp->strings_list[i]); /* TODO: append_STRING? that's redundant */
}
strlist_destroy (tmp);
return result;
}
int
main (int argc, char **argv)
{
FILE *fp;
#ifdef USE_LIBFANN
struct fann *ann;
#else
struct mlp *mlp;
#endif
struct training_set *set;
struct ionogram *ionogram;
struct ionogram_filename fn;
struct strlist *names, *files;
struct station_info *info;
struct globe_data *globe;
struct ionogram_filetype *ft;
char c;
numeric_t best_mse, best_mse_before;
int i;
char *weightfile = default_weight_file;
int interrogate = 0;
int weightfile_flag = 0;
int best_flag = 0;
files = strlist_new ();
set = training_set_new ();
while ((c = getopt (argc, argv, ":n:e:w:s:I:ibh")) != -1)
{
switch (c)
{
case 'n':
if (!sscanf (optarg, "%i", &set->epoch_count))
{
fprintf (stderr, "%s: option -n expects a number\n", argv[0]);
return 1;
}
break;
case 'I':
if (!sscanf (optarg, "%i", &set->info_interval))
{
fprintf (stderr, "%s: option -I expects a number\n", argv[0]);
return 1;
}
if (set->info_interval < 1)
{
fprintf (stderr, "%s: interval must be strictly positive\n", argv[0]);
return 1;
}
break;
case 'b':
best_flag++;
break;
case 'e':
if (!sscanf (optarg, "%lg", &set->desired_mse))
{
fprintf (stderr, "%s: option -e expects a number\n", argv[0]);
return 1;
}
break;
case 's':
if (!sscanf (optarg, "%lg", &set->training_speed))
{
fprintf (stderr, "%s: option -s expects a number\n", argv[0]);
return 1;
}
break;
case 'w':
weightfile = optarg;
weightfile_flag++;
break;
case 'i':
interrogate++;
break;
case ':':
fprintf (stderr, "%s: option -%c requires an argument\n", argv[0], optopt);
help (argv);
exit (1);
break;
case '?':
fprintf (stderr, "%s: unrecognized option -- -%c\n", argv[0], optopt);
help (argv);
exit (1);
break;
case 'h':
help (argv);
exit (0);
break; /* Sure, sure */
}
}
for (i = optind; i < argc; i++)
strlist_append_string (files, argv[i]);
srand (time (NULL));
if (libsao_init () == -1)
return 1;
if (ionowatch_config_init () == -1)
return -1;
#ifdef USE_LIBFANN
ann = build_fann ();
best_mse = INFINITY;
best_mse_before = INFINITY;
#else
mlp = build_mlp ();
NOTICE ("trying to load weights from %s...\n", weightfile);
if (mlp_load_weights (mlp, weightfile) == -1)
NOTICE ("failed: %s\n", strerror (errno));
else
NOTICE ("done\n");
best_mse_before = mlp->best_mse;
#endif
printf ("Best MSE: %g\n", best_mse_before);
globe = globe_data_new (0, 0, 0, 0);
ft = ionogram_filetype_lookup ("SAO");
info = NULL;
names = NULL;
if (files->strings_count > 0)
{
if (files->strings_count > 1)
{
ERROR ("currently one file at a time supported\n");
return 1; /* This is because I'm too lazy to save pointers to stations */
}
for (i = 0; i < files->strings_count; i++)
{
if (ionogram_parse_filename (files->strings_list[i], &fn) == -1)
{
NOTICE ("%s: malformed filename, couldn't load\n", files->strings_list[i]);
continue;
}
if (fn.type != ft->type)
{
ERROR ("not a SAO file, only SAO files supported\n");
return 1;
}
if ((info = station_lookup (fn.station)) == NULL)
{
ERROR ("couldn't find station data for `%s'\n", fn.station);
strlist_destroy (names);
return 1; /* We're almost sure that there will be no other filename
refering to a different station in this directory */
}
names = get_day_ionograms (&fn);
}
}
else
{
NOTICE ("looking for a suitable station...\n");
for (;;)
{
for (;;)
{
names = pickup_random_files ();
if (names->strings_count == 0)
{
strlist_destroy (names);
continue;
}
break;
}
if (ionogram_parse_filename (names->strings_list[0], &fn) == -1)
{
ERROR ("%s: malformed filename, couldn't load\n", names->strings_list[0]);
strlist_destroy (names);
continue;
}
if ((info = station_lookup (fn.station)) == NULL)
{
ERROR ("couldn't find station data for `%s'\n", fn.station);
strlist_destroy (names);
continue; /* We're almost sure that there will be no other filename
refering to a different station in this directory */
}
break;
}
}
if (!names->strings_count)
{
ERROR ("no ionograms that day!\n");
return 1;
}
printf ("file: %s\n", names->strings_list[0]);
NOTICE ("configutarion is: %d epochs, taking %g as minimum MSE "
"warning every %d epochs\n", set->epoch_count, set->desired_mse, set->info_interval);
if (best_flag)
NOTICE ("weights will be saved ONLY if they achieve better results\n");
NOTICE ("selected station %s (%s, %s)\n",
fn.station, info->name_long, info->country);
NOTICE ("daily files are of the form %s\n", names->strings_list[0]);
NOTICE ("ionotrainer is going to parse %d files, please wait...\n",
names->strings_count);
for (i = 0; i < names->strings_count; i++)
{
NOTICE ("parsing files [%3d/%3d]... ",
i + 1, names->strings_count);
if (ionogram_parse_filename (names->strings_list[i], &fn) == -1)
{
NOTICE ("%s: malformed filename, couldn't load\n", names->strings_list[i]);
continue;
}
if ((fp = cache_get_ionogram (&fn)) != NULL)
{
ionogram = ionogram_new ();
if ((ft->parse_callback) (ionogram, fp) == 0)
{
ionogram->lat = RADADJUST (DEG2RAD (info->lat));
ionogram->lon = RADADJUST (DEG2RAD (-info->lon));
globe_data_set_time (globe, fn.time);
ionogram->sun_inclination = globe_data_get_sun_inclination (globe, ionogram->lat, ionogram->lon);
ionogram->solstice_offset = RADADJUST (globe->sol);
ionogram->sunspot_number = get_monthly_sunspot_number (fn.time);
training_set_add_ionogram (set, ionogram);
}
else
NOTICE ("%s: error parsing ionogram\n", names->strings_list[i]);
fclose (fp);
printf ("\n");
}
else
NOTICE ("%s: coudln't retrieve from cache\n", names->strings_list[i]);
/* Hang on, man. This is NOT a botnet. */
sleep (1);
}
NOTICE ("work done!\n");
NOTICE ("preparing training vectors...\n");
training_set_build (set);
NOTICE ("got %d training vectors\n", set->training_vector_count);
if (!set->training_vector_count)
{
fprintf (stderr, "no training vectors, exiting...\n");
return 0;
}
#ifdef USE_LIBFANN
best_mse = training_set_train_on_fann (set, ann);
#else
best_mse = training_set_train_on_mlp (set, mlp);
#endif
if (best_mse < best_mse_before)
fprintf (stderr, "training ended with better MSE at %d epochs (mse: %g)\n", set->passed_epochs, best_mse);
else
fprintf (stderr, "Training stopped, no better MSE found\n");
#ifdef USE_LIBFANN
fprintf (stderr, "Weights NOT saved\n");
#else
if (!best_flag || best_mse < best_mse_before)
{
if (interrogate)
{
fprintf (stderr, "do you want to save weights to %s? [y/N] ", weightfile);
c = getchar ();
}
if (c == 'y' || c == 'Y' || !interrogate)
{
if (mlp_save_weights (mlp, weightfile) == -1)
fprintf (stderr, "saving failed: %s\n", strerror (errno));
else
fprintf (stderr, "weights saved\n");
}
else
fprintf (stderr, "not saved\n");
}
#endif
return 0;
}
