static void expand_class_path(char *cp, struct strlist *dst) {
struct strlist wildcards;
size_t i;
strlist_init(&wildcards, 16);
split(cp, ':', &wildcards);
for (i = 0; i < wildcards.size; i++) {
expand_jars(wildcards.items[i], dst);
}
strlist_destroy(&wildcards);
}
static char* create_class_path_option(char *cp) {
struct strlist parts;
char *result;
strlist_init(&parts, 128);
strlist_add(&parts, "-Djava.class.path=");
expand_class_path(cp, &parts);
if (parts.size > 1) {
strlist_remove_last(&parts); /* last entry is a ':' */
}
result = strlist_concat(&parts);
strlist_destroy(&parts);
return result;
}
struct strlist *
pickup_random_files ()
{
char *url;
char *year;
char *yday;
char *station;
int i, n;
struct strlist *tmp, *result;
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/");
tmp = parse_apache_index (url);
free (url);
if (tmp->strings_count == 0)
return tmp;
n = (tmp->strings_count - 1) * ((float) rand () / (float) RAND_MAX);
station = xstrdup (tmp->strings_list[n]);
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/%s/individual/",
station);
tmp = parse_apache_index (url);
free (url);
if (tmp->strings_count == 0)
{
free (station);
return tmp;
}
n = (tmp->strings_count - 1) * ((float) rand () / (float) RAND_MAX);
year = xstrdup (tmp->strings_list[n]);
strlist_destroy (tmp);
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/%s/individual/%s/",
station, year);
tmp = parse_apache_index (url);
free (url);
if (tmp->strings_count == 0)
{
free (station);
free (year);
return tmp;
}
n = (tmp->strings_count - 1) * ((float) rand () / (float) RAND_MAX);
yday = xstrdup (tmp->strings_list[n]);
strlist_destroy (tmp);
url = strbuild ("http://ngdc.noaa.gov/ionosonde/MIDS/data/%s/individual/%s/%s/scaled/",
station, year, yday);
free (yday);
free (year);
free (station);
tmp = parse_apache_index (url);
free (url);
result = strlist_new ();
for (i = 0; i < tmp->strings_count; i++)
{
if (strcmp (&tmp->strings_list[i][strlen (tmp->strings_list[i]) - 3], "SAO") == 0)
strlist_append_string (result, tmp->strings_list[i]); /* TODO: append_STRING? that's redundant */
}
strlist_destroy (tmp);
return result;
}
int
main (int argc, char **argv, char **envp)
{
FILE *list, *fp;
int i, j, count;
int lines = 0;
struct ionogram *ionogram;
struct ionogram_filename fn;
struct strlist *names;
struct station_info *info;
struct globe_data *globe;
struct ionogram_filetype *ft;
struct layer_info *this;
if (argc != 2)
{
fprintf (stderr, "Usage: %s <pattern.lst>\n", argv[0]);
fprintf (stderr, "\n%s picks random ionogram from the NOAA server and builds\n", argv[0]);
fprintf (stderr, "a training set with the existence state of each ionosphere layer.\n");
fprintf (stderr, "\n");
fprintf (stderr, "The training set file is in ID3 format.\n");
return 1;
}
signal (SIGINT, sigint_handler);
if (access (argv[1], F_OK) != -1)
fprintf (stderr, "%s: appending to existing file %s\n", argv[0], argv[1]);
if ((list = fopen (argv[1], "a")) == NULL)
{
fprintf (stderr, "%s: couldn't append to file %s: %s\n", argv[0], argv[1], strerror (errno));
return 1;
}
srand (time (NULL));
if (libsao_init () == -1)
return 1;
if (ionowatch_config_init () == -1)
return -1;
globe = globe_data_new (0, 0, 100, 100);
ft = ionogram_filetype_lookup ("SAO");
printf ("%s staring (press Ctrl+C to stop)\n", argv[0]);
for (; training;)
{
printf ("Please wait while %s looks for a suitable station...\n", argv[0]);
for (;;)
{
names = pickup_random_files ();
if (names->strings_count == 0)
{
strlist_destroy (names);
continue;
}
break;
}
if (ionogram_parse_filename (names->strings_list[0], &fn) == -1)
{
ERROR ("%s: malformed filename, couldn't load\n", names->strings_list[i]);
strlist_destroy (names);
continue;
}
if ((info = station_lookup (fn.station)) == NULL)
{
ERROR ("couldn't find station data for `%s'\n", fn.station);
strlist_destroy (names);
continue; /* We're almost sure that there will be no other filename
refering to a different station in this directory */
}
for (i = 0; i < names->strings_count && training; i++)
{
NOTICE ("parsing files [%3d/%3d]... ",
i + 1, names->strings_count);
fflush (stdout);
if (ionogram_parse_filename (names->strings_list[i], &fn) == -1)
{
NOTICE ("%s: malformed filename, couldn't load\n", names->strings_list[i]);
continue;
}
if ((fp = cache_get_ionogram (&fn)) != NULL)
{
ionogram = ionogram_new ();
if ((ft->parse_callback) (ionogram, fp) == 0)
{
globe_data_set_time (globe, fn.time);
count = 0;
for (j = 0; j < IONOGRAM_MAX_LAYERS && training; j++)
{
fprintf (list, "%lg %lg %lg %lg %lg %d ",
info->lat, info->lon,
globe_data_get_sun_inclination (globe, RADADJUST (DEG2RAD (info->lat)), RADADJUST (DEG2RAD (-info->lon))),
RADADJUST (globe->sol),
get_monthly_sunspot_number (fn.time),
j);
if ((this = ionogram->layers[j]) != NULL)
{
count++;
fprintf (list, " 1\n");
}
else
fprintf (list, " 0\n");
lines++;
}
printf ("%d layers present\n", count);
}
else
printf ("%s: error parsing ionogram\n", names->strings_list[i]);
ionogram_destroy (ionogram);
fclose (fp);
}
else
NOTICE ("%s: coudln't retrieve form cache\n", names->strings_list[i]);
}
strlist_destroy (names);
}
printf ("\n%s: fetching stopped, %d samples saved to %s\n",
argv[0], lines, argv[1]);
fclose (list);
return 0;
}
static int
write_includes(GenCodeInfo genCodeInfo, ABObj project)
{
File codeFile = genCodeInfo->code_file;
STRING *p = NULL;
AB_TRAVERSAL trav;
ABObj module = NULL;
StringList includes = strlist_create();
char buf[MAXPATHLEN+1];
int i;
strlist_set_is_unique(includes, TRUE);
*buf = 0;
/* standard system includes */
for (p = Includes; *p; p++)
{
strlist_add_str(includes, *p, NULL);
}
/*
* Includes for sessioning.
* These include files are needed only if sessioning
* is used.
*/
if (abmfP_proj_needs_session_save(project) ||
abmfP_proj_needs_session_restore(project))
{
for (p = Session_Includes; *p; p++)
{
strlist_add_str(includes, *p, NULL);
}
}
/*
* Includes for i18n.
* These include files are needed only if i18n
* is enabled.
*/
if (genCodeInfo->i18n_method == ABMF_I18N_XPG4_API)
{
for (p = I18n_Includes; *p; p++)
{
strlist_add_str(includes, *p, NULL);
}
}
/* module includes */
for (trav_open(&trav, project, AB_TRAV_MODULES);
(module = trav_next(&trav)) != NULL; )
{
if (!obj_is_defined(module))
{
continue;
}
sprintf(buf, "\"%s\"", abmfP_get_ui_header_file_name(module));
strlist_add_str(includes, buf, NULL);
}
trav_close(&trav);
/* project include */
sprintf(buf, "\"%s\"", abmfP_get_project_header_file_name(project));
strlist_add_str(includes, buf, NULL);
sprintf(buf, "\"%s\"", abmfP_get_utils_header_file_name(project));
strlist_add_str(includes, buf, NULL);
abmfP_get_connect_includes(includes, project);
/* Write the includes */
for (i = 0; i < strlist_get_num_strs(includes); ++i)
{
abio_printf(codeFile, "#include %s\n",
strlist_get_str(includes, i, NULL));
}
abio_puts(codeFile, nlstr);
strlist_destroy(includes);
return 0;
}
int extfs_mkfs(cdico *d, char *partition, int extfstype, char *fsoptions)
{
cstrlist strfeatures;
u64 features_tab[3];
u64 fsextrevision;
int origextfstype;
char buffer[2048];
char command[2048];
char options[2048];
char temp[1024];
char progname[64];
u64 e2fstoolsver;
int compat_type;
u64 temp64;
int exitst;
int ret=0;
int res;
int i;
// init
memset(options, 0, sizeof(options));
snprintf(progname, sizeof(progname), "mke2fs");
strlist_init(&strfeatures);
// ---- check that mkfs is installed and get its version
if (exec_command(command, sizeof(command), NULL, NULL, 0, NULL, 0, "%s -V", progname)!=0)
{ errprintf("%s not found. please install a recent e2fsprogs on your system or check the PATH.\n", progname);
ret=-1;
goto extfs_mkfs_cleanup;
}
e2fstoolsver=check_prog_version(progname);
// ---- filesystem revision (good-old-rev or dynamic)
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTREVISION, &fsextrevision)!=0)
fsextrevision=EXT2_DYNAMIC_REV; // don't fail (case of fs conversion to extfs)
// "mke2fs -q" prevents problems in exec_command when too many output details printed
strlcatf(options, sizeof(options), " -q ");
// filesystem revision: good-old-rev or dynamic
strlcatf(options, sizeof(options), " -r %d ", (int)fsextrevision);
strlcatf(options, sizeof(options), " %s ", fsoptions);
// ---- set the advanced filesystem settings from the dico
if (dico_get_string(d, 0, FSYSHEADKEY_FSLABEL, buffer, sizeof(buffer))==0 && strlen(buffer)>0)
strlcatf(options, sizeof(options), " -L '%.16s' ", buffer);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTBLOCKSIZE, &temp64)==0)
strlcatf(options, sizeof(options), " -b %ld ", (long)temp64);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSINODESIZE, &temp64)==0)
strlcatf(options, sizeof(options), " -I %ld ", (long)temp64);
// ---- get original filesystem features (if the original filesystem was an ext{2,3,4})
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFEATURECOMPAT, &features_tab[E2P_FEATURE_COMPAT])!=0 ||
dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFEATUREINCOMPAT, &features_tab[E2P_FEATURE_INCOMPAT])!=0 ||
dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFEATUREROCOMPAT, &features_tab[E2P_FEATURE_RO_INCOMPAT])!=0)
{ // dont fail the original filesystem may not be ext{2,3,4}. in that case set defaults features
features_tab[E2P_FEATURE_COMPAT]=EXT2_FEATURE_COMPAT_RESIZE_INODE|EXT2_FEATURE_COMPAT_DIR_INDEX;
features_tab[E2P_FEATURE_INCOMPAT]=EXT2_FEATURE_INCOMPAT_FILETYPE;
features_tab[E2P_FEATURE_RO_INCOMPAT]=EXT2_FEATURE_RO_COMPAT_SPARSE_SUPER;
}
// ---- check that fsarchiver is aware of all the filesystem features used on that filesystem
if (extfs_check_compatibility(features_tab[E2P_FEATURE_COMPAT], features_tab[E2P_FEATURE_INCOMPAT], features_tab[E2P_FEATURE_RO_INCOMPAT])!=0)
{ errprintf("this filesystem has ext{2,3,4} features which are not supported by this fsarchiver version.\n");
return -1;
}
// ---- get original filesystem type
origextfstype=extfs_get_fstype_from_compat_flags(features_tab[E2P_FEATURE_COMPAT],
features_tab[E2P_FEATURE_INCOMPAT], features_tab[E2P_FEATURE_RO_INCOMPAT]);
msgprintf(MSG_VERB2, "the filesystem type determined by the original filesystem features is [%s]\n", format_fstype(origextfstype));
// remove all the features not supported by the filesystem to create (conversion = downgrade fs)
for (i=0; mkfeatures[i].name; i++)
{
compat_type=mkfeatures[i].compat;
if (mkfeatures[i].firstfs > extfstype)
features_tab[compat_type] &= ~mkfeatures[i].mask;
}
// add new features if the filesystem to create is newer than the filesystem type that was backed up
// eg: user did a "savefs" of an ext3 and does a "restfs mkfs=ext4" --> add features to force ext4
// it's a bit more difficult because we only want to add such a feature if no feature of the new
// filesystem is currently enabled.
msgprintf(MSG_VERB2, "the filesystem type to create considering the command options is [%s]\n", format_fstype(extfstype));
if (origextfstype==EXTFSTYPE_EXT2 && extfstype>EXTFSTYPE_EXT2) // upgrade ext2 to ext{3,4}
{ fsextrevision=EXT2_DYNAMIC_REV;
features_tab[E2P_FEATURE_COMPAT]|=EXT3_FEATURE_COMPAT_HAS_JOURNAL;
}
if (origextfstype<EXTFSTYPE_EXT4 && extfstype>=EXTFSTYPE_EXT4) // upgrade ext{2,3} to ext4
{ fsextrevision=EXT2_DYNAMIC_REV;
features_tab[E2P_FEATURE_INCOMPAT]|=EXT3_FEATURE_INCOMPAT_EXTENTS;
}
// convert int features to string to be passed to mkfs
for (i=0; mkfeatures[i].name; i++)
{
if (mkfeatures[i].firste2p<=e2fstoolsver) // don't pass an option to a program that does not support it
{
compat_type=mkfeatures[i].compat;
if (features_tab[compat_type] & mkfeatures[i].mask)
{ msgprintf(MSG_VERB2, "--> feature [%s]=YES\n", mkfeatures[i].name);
strlist_add(&strfeatures, mkfeatures[i].name);
}
else
{ msgprintf(MSG_VERB2, "--> feature [%s]=NO\n", mkfeatures[i].name);
snprintf(temp, sizeof(temp), "^%s", mkfeatures[i].name); // exclude feature
strlist_add(&strfeatures, temp);
}
}
}
// if extfs revision is dynamic and there are features in the list
if (fsextrevision!=EXT2_GOOD_OLD_REV && strlist_count(&strfeatures)>0)
{ strlist_merge(&strfeatures, temp, sizeof(temp), ',');
strlcatf(options, sizeof(options), " -O %s ", temp);
msgprintf(MSG_VERB2, "features: mkfs_options+=[-O %s]\n", temp);
}
// ---- check mke2fs version requirement
msgprintf(MSG_VERB2, "mke2fs version detected: %s\n", format_prog_version(e2fstoolsver, temp, sizeof(temp)));
msgprintf(MSG_VERB2, "mke2fs version required: %s\n", format_prog_version(e2fsprogs_minver[extfstype], temp, sizeof(temp)));
if (e2fstoolsver < e2fsprogs_minver[extfstype])
{ errprintf("mke2fs was found but is too old, please upgrade to a version %s or more recent.\n",
format_prog_version(e2fsprogs_minver[extfstype], temp, sizeof(temp)));
ret=-1;
goto extfs_mkfs_cleanup;
}
// ---- extended options
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTEOPTRAIDSTRIDE, &temp64)==0)
strlcatf(options, sizeof(options), " -E stride=%ld ", (long)temp64);
if ((dico_get_u64(d, 0, FSYSHEADKEY_FSEXTEOPTRAIDSTRIPEWIDTH, &temp64)==0) && e2fstoolsver>=PROGVER(1,40,7))
strlcatf(options, sizeof(options), " -E stripe-width=%ld ", (long)temp64);
// ---- execute mke2fs
msgprintf(MSG_VERB2, "exec: %s\n", command);
if (exec_command(command, sizeof(command), &exitst, NULL, 0, NULL, 0, "%s %s %s", progname, partition, options)!=0 || exitst!=0)
{ errprintf("command [%s] failed with return status=%d\n", command, exitst);
ret=-1;
goto extfs_mkfs_cleanup;
}
// ---- use tune2fs to set the other advanced options
memset(options, 0, sizeof(options));
if (dico_get_string(d, 0, FSYSHEADKEY_FSUUID, buffer, sizeof(buffer))==0 && strlen(buffer)==36)
strlcatf(options, sizeof(options), " -U %s ", buffer);
if (dico_get_string(d, 0, FSYSHEADKEY_FSEXTDEFMNTOPT, buffer, sizeof(buffer))==0 && strlen(buffer)>0)
strlcatf(options, sizeof(options), " -o %s ", buffer);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFSCKMAXMNTCOUNT, &temp64)==0)
strlcatf(options, sizeof(options), " -c %ld ", (long)temp64);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFSCKCHECKINTERVAL, &temp64)==0)
strlcatf(options, sizeof(options), " -i %ldd ", (long)(temp64/86400L));
if (options[0])
{
if (exec_command(command, sizeof(command), &exitst, NULL, 0, NULL, 0, "tune2fs %s %s", partition, options)!=0 || exitst!=0)
{ errprintf("command [%s] failed with return status=%d\n", command, exitst);
ret=-1;
goto extfs_mkfs_cleanup;
}
// run e2fsck to workaround an tune2fs bug in e2fsprogs < 1.41.4 on ext4
// http://article.gmane.org/gmane.comp.file-systems.ext4/11181
if (extfstype==EXTFSTYPE_EXT4 && e2fstoolsver<PROGVER(1,41,4))
{
if ( ((res=exec_command(command, sizeof(command), &exitst, NULL, 0, NULL, 0, "e2fsck -fy %s", partition))!=0) || ((exitst!=0) && (exitst!=1)) )
{ errprintf("command [%s] failed with return status=%d\n", command, exitst);
ret=-1;
goto extfs_mkfs_cleanup;
}
}
}
extfs_mkfs_cleanup:
strlist_destroy(&strfeatures);
return ret;
}
int
main (int argc, char **argv)
{
FILE *fp;
#ifdef USE_LIBFANN
struct fann *ann;
#else
struct mlp *mlp;
#endif
struct training_set *set;
struct ionogram *ionogram;
struct ionogram_filename fn;
struct strlist *names, *files;
struct station_info *info;
struct globe_data *globe;
struct ionogram_filetype *ft;
char c;
numeric_t best_mse, best_mse_before;
int i;
char *weightfile = default_weight_file;
int interrogate = 0;
int weightfile_flag = 0;
int best_flag = 0;
files = strlist_new ();
set = training_set_new ();
while ((c = getopt (argc, argv, ":n:e:w:s:I:ibh")) != -1)
{
switch (c)
{
case 'n':
if (!sscanf (optarg, "%i", &set->epoch_count))
{
fprintf (stderr, "%s: option -n expects a number\n", argv[0]);
return 1;
}
break;
case 'I':
if (!sscanf (optarg, "%i", &set->info_interval))
{
fprintf (stderr, "%s: option -I expects a number\n", argv[0]);
return 1;
}
if (set->info_interval < 1)
{
fprintf (stderr, "%s: interval must be strictly positive\n", argv[0]);
return 1;
}
break;
case 'b':
best_flag++;
break;
case 'e':
if (!sscanf (optarg, "%lg", &set->desired_mse))
{
fprintf (stderr, "%s: option -e expects a number\n", argv[0]);
return 1;
}
break;
case 's':
if (!sscanf (optarg, "%lg", &set->training_speed))
{
fprintf (stderr, "%s: option -s expects a number\n", argv[0]);
return 1;
}
break;
case 'w':
weightfile = optarg;
weightfile_flag++;
break;
case 'i':
interrogate++;
break;
case ':':
fprintf (stderr, "%s: option -%c requires an argument\n", argv[0], optopt);
help (argv);
exit (1);
break;
case '?':
fprintf (stderr, "%s: unrecognized option -- -%c\n", argv[0], optopt);
help (argv);
exit (1);
break;
case 'h':
help (argv);
exit (0);
break; /* Sure, sure */
}
}
for (i = optind; i < argc; i++)
strlist_append_string (files, argv[i]);
srand (time (NULL));
if (libsao_init () == -1)
return 1;
if (ionowatch_config_init () == -1)
return -1;
#ifdef USE_LIBFANN
ann = build_fann ();
best_mse = INFINITY;
best_mse_before = INFINITY;
#else
mlp = build_mlp ();
NOTICE ("trying to load weights from %s...\n", weightfile);
if (mlp_load_weights (mlp, weightfile) == -1)
NOTICE ("failed: %s\n", strerror (errno));
else
NOTICE ("done\n");
best_mse_before = mlp->best_mse;
#endif
printf ("Best MSE: %g\n", best_mse_before);
globe = globe_data_new (0, 0, 0, 0);
ft = ionogram_filetype_lookup ("SAO");
info = NULL;
names = NULL;
if (files->strings_count > 0)
{
if (files->strings_count > 1)
{
ERROR ("currently one file at a time supported\n");
return 1; /* This is because I'm too lazy to save pointers to stations */
}
for (i = 0; i < files->strings_count; i++)
{
if (ionogram_parse_filename (files->strings_list[i], &fn) == -1)
{
NOTICE ("%s: malformed filename, couldn't load\n", files->strings_list[i]);
continue;
}
if (fn.type != ft->type)
{
ERROR ("not a SAO file, only SAO files supported\n");
return 1;
}
if ((info = station_lookup (fn.station)) == NULL)
{
ERROR ("couldn't find station data for `%s'\n", fn.station);
strlist_destroy (names);
return 1; /* We're almost sure that there will be no other filename
refering to a different station in this directory */
}
names = get_day_ionograms (&fn);
}
}
else
{
NOTICE ("looking for a suitable station...\n");
for (;;)
{
for (;;)
{
names = pickup_random_files ();
if (names->strings_count == 0)
{
strlist_destroy (names);
continue;
}
break;
}
if (ionogram_parse_filename (names->strings_list[0], &fn) == -1)
{
ERROR ("%s: malformed filename, couldn't load\n", names->strings_list[0]);
strlist_destroy (names);
continue;
}
if ((info = station_lookup (fn.station)) == NULL)
{
ERROR ("couldn't find station data for `%s'\n", fn.station);
strlist_destroy (names);
continue; /* We're almost sure that there will be no other filename
refering to a different station in this directory */
}
break;
}
}
if (!names->strings_count)
{
ERROR ("no ionograms that day!\n");
return 1;
}
printf ("file: %s\n", names->strings_list[0]);
NOTICE ("configutarion is: %d epochs, taking %g as minimum MSE "
"warning every %d epochs\n", set->epoch_count, set->desired_mse, set->info_interval);
if (best_flag)
NOTICE ("weights will be saved ONLY if they achieve better results\n");
NOTICE ("selected station %s (%s, %s)\n",
fn.station, info->name_long, info->country);
NOTICE ("daily files are of the form %s\n", names->strings_list[0]);
NOTICE ("ionotrainer is going to parse %d files, please wait...\n",
names->strings_count);
for (i = 0; i < names->strings_count; i++)
{
NOTICE ("parsing files [%3d/%3d]... ",
i + 1, names->strings_count);
if (ionogram_parse_filename (names->strings_list[i], &fn) == -1)
{
NOTICE ("%s: malformed filename, couldn't load\n", names->strings_list[i]);
continue;
}
if ((fp = cache_get_ionogram (&fn)) != NULL)
{
ionogram = ionogram_new ();
if ((ft->parse_callback) (ionogram, fp) == 0)
{
ionogram->lat = RADADJUST (DEG2RAD (info->lat));
ionogram->lon = RADADJUST (DEG2RAD (-info->lon));
globe_data_set_time (globe, fn.time);
ionogram->sun_inclination = globe_data_get_sun_inclination (globe, ionogram->lat, ionogram->lon);
ionogram->solstice_offset = RADADJUST (globe->sol);
ionogram->sunspot_number = get_monthly_sunspot_number (fn.time);
training_set_add_ionogram (set, ionogram);
}
else
NOTICE ("%s: error parsing ionogram\n", names->strings_list[i]);
fclose (fp);
printf ("\n");
}
else
NOTICE ("%s: coudln't retrieve from cache\n", names->strings_list[i]);
/* Hang on, man. This is NOT a botnet. */
sleep (1);
}
NOTICE ("work done!\n");
NOTICE ("preparing training vectors...\n");
training_set_build (set);
NOTICE ("got %d training vectors\n", set->training_vector_count);
if (!set->training_vector_count)
{
fprintf (stderr, "no training vectors, exiting...\n");
return 0;
}
#ifdef USE_LIBFANN
best_mse = training_set_train_on_fann (set, ann);
#else
best_mse = training_set_train_on_mlp (set, mlp);
#endif
if (best_mse < best_mse_before)
fprintf (stderr, "training ended with better MSE at %d epochs (mse: %g)\n", set->passed_epochs, best_mse);
else
fprintf (stderr, "Training stopped, no better MSE found\n");
#ifdef USE_LIBFANN
fprintf (stderr, "Weights NOT saved\n");
#else
if (!best_flag || best_mse < best_mse_before)
{
if (interrogate)
{
fprintf (stderr, "do you want to save weights to %s? [y/N] ", weightfile);
c = getchar ();
}
if (c == 'y' || c == 'Y' || !interrogate)
{
if (mlp_save_weights (mlp, weightfile) == -1)
fprintf (stderr, "saving failed: %s\n", strerror (errno));
else
fprintf (stderr, "weights saved\n");
}
else
fprintf (stderr, "not saved\n");
}
#endif
return 0;
}
void *
trainer_thread_entry (void *unused)
{
struct training_set *set;
struct strlist *names;
struct ionogram_filename fn;
struct station_info *info;
struct ionogram *ionogram;
struct ionogram_filetype *ft;
struct globe_data *globe;
FILE *fp;
double average;
int i;
if ((ft = ionogram_filetype_lookup ("SAO")) == NULL)
{
ERROR ("no SAO filetype registered\n");
return NULL;
}
h2pf_mlp = h2pf_build_mlp ();
predictor_mlp = predictor_build_mlp ();
plasma_mlp = plasma_build_mlp ();
(void) load_weights_from_config (h2pf_mlp, "h2pf.mlp");
(void) load_weights_from_config (plasma_mlp, "plasma.mlp");
(void) load_weights_from_config (predictor_mlp, "predictor.mlp");
globe = globe_data_new (0, 0, 0, 0);
set = training_set_new ();
for (;;)
{
getloadavg (&average, 1);
DEBUG ("load average: %lg\n", average);
if (average >= autotrain_max_load)
{
DEBUG ("load average (%lg) too high to train, 15 minutes to restart\n", average);
sleep (900);
continue;
}
DEBUG ("system load is ok to train, (%lg <= %lg) starting...\n", average, autotrain_max_load);
for (;;)
{
for (;;)
{
names = pickup_random_files ();
if (names->strings_count == 0)
{
strlist_destroy (names);
continue;
}
break;
}
if (ionogram_parse_filename (names->strings_list[0], &fn) == -1)
{
DEBUG ("%s: malformed filename, couldn't load\n", names->strings_list[0]);
strlist_destroy (names);
continue;
}
if ((info = station_lookup (fn.station)) == NULL)
{
DEBUG ("couldn't find station data for `%s'\n", fn.station);
strlist_destroy (names);
continue;
}
break;
}
if (!names->strings_count)
{
DEBUG ("no ionograms this day\n");
strlist_destroy (names);
continue;
}
sleep_until_low_load ();
for (i = 0; i < names->strings_count; i++)
{
DEBUG ("parsing files [%3d/%3d]...\n",
i + 1, names->strings_count);
if (ionogram_parse_filename (names->strings_list[i], &fn) == -1)
{
DEBUG ("%s: malformed filename, couldn't load\n", names->strings_list[i]);
continue;
}
if ((fp = cache_get_ionogram (&fn)) != NULL)
{
ionogram = ionogram_new ();
if ((ft->parse_callback) (ionogram, fp) == 0)
{
ionogram->lat = RADADJUST (DEG2RAD (info->lat));
ionogram->lon = RADADJUST (DEG2RAD (-info->lon));
globe_data_set_time (globe, fn.time);
ionogram->sun_inclination = globe_data_get_sun_inclination (globe, ionogram->lat, ionogram->lon);
ionogram->solstice_offset = RADADJUST (globe->sol);
ionogram->sunspot_number = get_monthly_sunspot_number (fn.time);
training_set_add_ionogram (set, ionogram);
}
else
DEBUG ("%s: error parsing ionogram\n", names->strings_list[i]);
fclose (fp);
}
else
DEBUG ("%s: coudln't retrieve form cache\n", names->strings_list[i]);
/* Hang on, hang on. */
sleep (1);
}
sleep_until_low_load ();
training_set_set_epoch_count (set, 1000);
training_set_set_info_interval (set, 0);
training_set_set_training_speed (set, 1e-6);
DEBUG ("Start layer MLP training...\n");
/* Build training set for layers */
predictor_training_set_build (set);
training_set_train_on_mlp (set, predictor_mlp);
training_set_destroy_vectors (set);
save_weights (predictor_mlp, "predictor.mlp");
sleep_until_low_load ();
training_set_set_training_speed (set, 1e-5);
DEBUG ("Start plasma MLP training...\n");
/* Build training set for layers */
plasma_training_set_build (set);
training_set_train_on_mlp (set, plasma_mlp);
training_set_destroy_vectors (set);
save_weights (plasma_mlp, "plasma.mlp");
sleep_until_low_load ();
training_set_set_training_speed (set, 1e-7);
DEBUG ("Start height to plasma MLP training...\n");
/* Build training set for layers */
h2pf_training_set_build (set);
training_set_train_on_mlp (set, h2pf_mlp);
training_set_destroy_vectors (set);
save_weights (h2pf_mlp, "h2pf.mlp");
training_set_destroy (set);
set = training_set_new ();
}
}
int archwriter_destroy(carchwriter *ai)
{
assert(ai);
strlist_destroy(&ai->vollist);
return 0;
}
