void computeMorseGraph ( MorseGraph & morsegraph,
boost::shared_ptr<const Map> map,
const int SINGLECMG_INIT_PHASE_SUBDIVISIONS,
const int SINGLECMG_MIN_PHASE_SUBDIVISIONS,
const int SINGLECMG_MAX_PHASE_SUBDIVISIONS,
const int SINGLECMG_COMPLEXITY_LIMIT,
const char * outputfile ) {
#ifdef CMG_VERBOSE
std::cout << "SingleCMG: computeMorseGraph.\n";
#endif
boost::shared_ptr < Grid > phase_space = morsegraph . phaseSpace ();
clock_t start_time = clock ();
Compute_Morse_Graph ( & morsegraph,
phase_space,
map,
SINGLECMG_INIT_PHASE_SUBDIVISIONS,
SINGLECMG_MIN_PHASE_SUBDIVISIONS,
SINGLECMG_MAX_PHASE_SUBDIVISIONS,
SINGLECMG_COMPLEXITY_LIMIT );
clock_t stop_time = clock ();
if ( outputfile != NULL ) {
morsegraph . save ( outputfile );
}
std::ofstream stats_file ( "SingleCMG_statistics.txt" );
stats_file << "Morse Graph calculation resource usage statistics.\n";
stats_file << "The final grid has " << phase_space -> size () << " grid elements.\n";
stats_file << "The computation took " << ((double)(stop_time-start_time)/(double)CLOCKS_PER_SEC)
<< " seconds.\n";
stats_file << "All memory figures are in bytes:\n";
stats_file << "grid_memory_use = " << phase_space -> memory () << "\n";
stats_file << "max_graph_memory = " << max_graph_memory << "\n";
stats_file << "max_scc_memory_internal = " << max_scc_memory_internal << "\n";
stats_file << "max_scc_memory_external = " << max_scc_memory_external << "\n";
stats_file . close ();
}
int main(int argc, char* argv[]) {
StatsCollector2 stats;
std::ifstream stats_file(argv[1]);
int count = 0;
while (true) {
Session session;
if (count % 10000 == 0) std::cout << "Stats: " << count << std::endl;
if (!Session::readSession(stats_file, session, false)) break;
stats.processSession(session);
++count;
}
std::ifstream train_file(argv[2]);
std::ofstream feats_file(argv[3]);
stats.writeFeaturesHeader(feats_file);
count = 0;
while (true) {
Session session;
if (count % 10000 == 0) std::cout << "Generating feats: " << count << std::endl;
if (!Session::readSession(train_file, session, false)) break;
stats.writeFeatures(feats_file, session);
++count;
}
}
/* XXX MDL20161018 should only be invoked after all instances of EventHandlers are
* destroyed. This is the expected case and the user should not have to
* do anything special */
void onExit()
{
/* For SynchroTraceSim thread scheduling
* and CPI calculations */
std::string pthread_metadata(EventHandlers::output_directory + "/sigil.pthread.out");
std::string stats_metadata(EventHandlers::output_directory + "/sigil.stats.out");
std::ofstream pthread_file(pthread_metadata, std::ios::trunc | std::ios::out);
std::ofstream stats_file(stats_metadata, std::ios::trunc | std::ios::out);
if (pthread_file.fail() == true)
{
SigiLog::fatal("Failed to open: " + pthread_metadata);
}
else if (stats_file.fail() == true)
{
SigiLog::fatal("Failed to open: " + stats_metadata);
}
spdlog::set_sync_mode();
auto pthread_sink = std::make_shared<spdlog::sinks::ostream_sink_st>(pthread_file);
auto stats_sink = std::make_shared<spdlog::sinks::ostream_sink_st>(stats_file);
auto pthread_logger = spdlog::create(pthread_metadata, {pthread_sink});
auto stats_logger = spdlog::create(stats_metadata, {stats_sink});
pthread_logger->set_pattern("%v");
stats_logger->set_pattern("%v");
/*********************************************************************/
SigiLog::info("Flushing thread metadata to: " + pthread_metadata);
/* The order the threads were seen SHOULD match to
* the order of thread_t values of the pthread_create
* calls. For example, with the valgrind frontend,
* the --fair-sched=yes option should make sure each
* thread is switched to in the order they were created */
assert(thread_spawns.size() == thread_creates.size());
int create_idx = 1; //skip the first idx, which is the initial thread
for (auto &pair : thread_spawns)
{
/* SynchroTraceSim only supports threads
* that were spawned from the original thread */
if (pair.first == 1)
{
pthread_logger->info("##" + std::to_string(pair.second) +
"," + std::to_string(thread_creates[create_idx]));
}
++create_idx; //Skip past thread spawns that happened in other threads
}
/* TODO Confirm with KS and SN how barriers are processed */
/* Iterate through each unique barrier_t address and
* aggregate all the associated, participating threads */
for (auto &pair : barrier_participants)
{
std::ostringstream ss;
ss << "**" << pair.first;
for (auto &tid : pair.second)
{
ss << "," << tid;
}
pthread_logger->info(ss.str());
}
pthread_logger->flush();
/*********************************************************************/
/*********************************************************************/
SigiLog::info("Flushing statistics to: " + stats_metadata);
StatCounter total_instr_count{0};
for (auto &p : PerThreadStats::per_thread_counts)
{
stats_logger->info("Thread Stats: " + std::to_string(p.first));
stats_logger->info("\tIOPS : " +
std::to_string(std::get<PerThreadStats::Type::IOP>(p.second)));
stats_logger->info("\tFLOPS : " +
std::to_string(std::get<PerThreadStats::Type::FLOP>(p.second)));
stats_logger->info("\tReads : " +
std::to_string(std::get<PerThreadStats::Type::Read>(p.second)));
stats_logger->info("\tWrites: " +
std::to_string(std::get<PerThreadStats::Type::Write>(p.second)));
total_instr_count += std::get<PerThreadStats::Type::Instr>(p.second);
}
stats_logger->info("Total instructions: " + std::to_string(total_instr_count));
stats_logger->flush();
/*********************************************************************/
}
/* returns:
* 0 ok, remove file from list
* -1 failure
*/
static int getfile(const rfile *fi, unsigned int opt,
const char *output, const char *destname)
{
struct stat sb;
char* dest = NULL;
getmode_t how = getNormal;
bool mkunique = false;
int r, ret = -1;
if((get_glob_mask && fnmatch(get_glob_mask, base_name_ptr(fi->path),
FNM_EXTMATCH) == FNM_NOMATCH)
#ifdef HAVE_REGEX
|| (get_rx_mask_set && regexec(&get_rx_mask, base_name_ptr(fi->path),
0, 0, 0) == REG_NOMATCH)
#endif
)
{
return 0;
}
if(!output)
output = ".";
if(test(opt, GET_PARENTS)) {
char *apath = base_dir_xptr(fi->path);
if (asprintf(&dest, "%s%s/%s", output, apath, destname) == -1)
{
free(apath);
fprintf(stderr, _("Failed to allocate memory.\n"));
return -1;
}
free(apath);
} else {
/* check if -o option is given, if GET_OUTPUT_FILE is set, we only
* transfer one file and output is set to the filename. However, if
* the destination already exists and is a directory, we assume
* that the user meant a directory */
int dest_is_file = test(opt, GET_OUTPUT_FILE);
if(stat(output, &sb) == 0) {
if(S_ISDIR(sb.st_mode)) {
dest_is_file = false;
}
}
if(dest_is_file)
dest = xstrdup(output);
else
if (asprintf(&dest, "%s/%s", output, destname) == -1)
{
fprintf(stderr, _("Failed to allocate memory.\n"));
return -1;
}
}
/* make sure destination directory exists */
{
char *destdir = base_dir_xptr(dest);
if(destdir) {
bool r = make_path(destdir);
if(!r) {
if (errno == EEXIST)
ftp_err("`%s' exists but is not a directory\n", destdir);
else
ftp_err("%s: %s\n", destdir, strerror(errno));
transfer_mail_msg(_("Couldn't create directory: %s\n"),
destdir);
free(destdir);
return -1;
}
/* change permission and group, if requested */
if(test(opt, GET_CHMOD)) {
if(stat(destdir, &sb) == 0) {
mode_t m = sb.st_mode;
m = mode_adjust(m, cmod);
if(chmod(destdir, m) != 0)
perror(destdir);
} }
if(test(opt, GET_CHGRP)) {
if(chown(destdir, -1, group_change) != 0)
perror(dest);
}
free(destdir);
}
}
/* check if destination file exists */
if(stat(dest, &sb) == 0) {
if(test(opt, GET_SKIP_EXISTING)) {
if(test(opt, GET_VERBOSE)) {
char* sp = shortpath(dest, 42, gvLocalHomeDir);
fprintf(stderr, _("Local file '%s' exists, skipping...\n"),
sp);
stats_file(STATS_SKIP, 0);
free(sp);
}
return 0;
}
if(test(opt, GET_UNIQUE))
mkunique = true;
else if(test(opt, GET_APPEND))
how = getAppend;
else if(test(opt, GET_NEWER)) {
struct tm *fan = gmtime(&sb.st_mtime);
time_t ft = ftp_filetime(fi->path, test(opt, GET_FORCE_NEWER));
sb.st_mtime = gmt_mktime(fan);
ftp_trace("get -n: remote file: %s", ctime(&ft));
ftp_trace("get -n: local file: %s\n", ctime(&sb.st_mtime));
if(sb.st_mtime >= ft && ft != (time_t)-1) {
if(test(opt, GET_VERBOSE)) {
char* sp = shortpath(dest, 30, gvLocalHomeDir);
ftp_err(_(
"Local file '%s' is newer than remote, skipping...\n"),
sp);
stats_file(STATS_SKIP, 0);
free(sp);
}
return 0;
}
} else if(!test(opt, GET_RESUME)) {
if(!get_owbatch && !gvSighupReceived) {
struct tm *fan = gmtime(&sb.st_mtime);
time_t ft = ftp_filetime(fi->path, test(opt, GET_FORCE_NEWER));
int a;
char *e;
sb.st_mtime = gmt_mktime(fan);
e = xstrdup(ctime(&sb.st_mtime));
char* sp = shortpath(dest, 42, gvLocalHomeDir);
a = ask(ASKYES|ASKNO|ASKUNIQUE|ASKCANCEL|ASKALL|ASKRESUME,
ASKRESUME,
_("Local file '%s' exists\nLocal: %lld bytes, %sRemote: %lld bytes, %sOverwrite?"),
sp,
(unsigned long long) sb.st_size, e ? e : "unknown date\n",
ftp_filesize(fi->path), ctime(&ft));
free(sp);
free(e);
if(a == ASKCANCEL) {
get_quit = true;
return 0;
}
else if(a == ASKNO)
return 0;
else if(a == ASKUNIQUE)
mkunique = true;
else if(a == ASKALL) {
get_owbatch = true;
}
else if(a == ASKRESUME)
opt |= GET_RESUME; /* for this file only */
/* else a == ASKYES */
}
}
if(test(opt, GET_RESUME))
how = getResume;
}
if(mkunique) {
char* newdest = make_unique_filename(dest);
free(dest);
dest = newdest;
}
/* the file doesn't exist or we choosed to overwrite it, or changed dest */
if(rislink(fi) && test(opt, GET_NO_DEREFERENCE)) {
/* remove any existing destination */
unlink(dest);
ftp_err(_("symlinking '%s' to '%s'\n"), dest, fi->link);
if(symlink(fi->link, dest) != 0)
perror(dest);
ret = 0;
}
else {
r = do_the_get(fi->path, dest, how, opt);
if(r == 0) {
stats_file(STATS_SUCCESS, ftp->ti.total_size);
ret = 0;
if(test(opt, GET_PRESERVE))
get_preserve_attribs(fi, dest);
if(test(opt, GET_CHMOD)) {
mode_t m = rfile_getmode(fi);
m = mode_adjust(m, cmod);
if(chmod(dest, m) != 0)
perror(dest);
}
if(test(opt, GET_CHGRP)) {
if(chown(dest, -1, group_change) != 0)
perror(dest);
}
if(test(opt, GET_DELETE_AFTER)) {
bool dodel = false;
char* sp = shortpath(fi->path, 42, ftp->homedir);
if(!test(opt, GET_FORCE)
&& !get_delbatch && !gvSighupReceived)
{
int a = ask(ASKYES|ASKNO|ASKCANCEL|ASKALL, ASKYES,
_("Delete remote file '%s'?"), sp);
if(a == ASKALL) {
get_delbatch = true;
dodel = true;
}
else if(a == ASKCANCEL)
get_quit = true;
else if(a != ASKNO)
dodel = true;
} else
dodel = true;
if(dodel) {
ftp_unlink(fi->path);
if(ftp->code == ctComplete)
fprintf(stderr, _("%s: deleted\n"), sp);
else
fprintf(stderr, _("error deleting '%s': %s\n"),
sp, ftp_getreply(false));
}
free(sp);
}
} else {
stats_file(STATS_FAIL, 0);
ret = -1;
}
}
free(dest);
return ret;
}
