string getHostname() {
string hostname;
ifstream statsFile("/etc/hostname");
if (getline(statsFile, hostname)) {
return hostname;
}
return "general_hostname";
}
// ######################################################################
void SimulationViewerStats::saveAGMaskStats(const Image<float> &img,
const std::string caller,
const std::string suffix)
{
ushort minx = 0, miny = 0, maxx = 0, maxy = 0;
float min, max, avg, std;
uint N;
// get a whole bunch of stats about this output image
getMinMaxAvgEtc(img, min, max, avg, std, minx, miny, maxx, maxy, N);
std::string fileName;
std::string txt = suffix;
fileName = itsGetSingleChannelStatsFile.getVal() + txt;
LINFO("SAVING AG STATS TO %s",fileName.c_str());
std::ofstream statsFile(fileName.c_str(), std::ios::app);
statsFile << "AGstats" << "\t";
statsFile << (itsFrameNumber - 1) << "\t";
// differentiate between scale image stats and the combined max norm
// for compatability with single channel stats
statsFile << "COMBINED\t" << caller << "\t";
//itsFrameIdx++;
statsFile << "final" << "\t" << "FINAL" << "\t";
statsFile << itsTotalCount << "\t"
<< itsMaskCount << "\t"
<< itsLamCount << "\t"
<< itsOverlapCount << "\t";
statsFile << min << "\t" << max << "\t" << avg << "\t" << std << "\t"
<< minx << "\t" << miny << "\t" << maxx << "\t" << maxy << "\t"
<< N << "\n";
statsFile.close();
}
// ######################################################################
void SimulationViewerStats::saveCompat(const Image<float>& img,
const std::string suffix,
const int frameOffset)
{
unsigned int frameIdx = (unsigned int)((int)itsFrameNumber + frameOffset);
std::string fileName;
std::string txt = suffix;
fileName = itsGetSingleChannelStatsFile.getVal() + txt;
ushort minx = 0, miny = 0, maxx = 0, maxy = 0;
float min, max, avg, std;
uint N;
// get a whole bunch of stats about this output image
getMinMaxAvgEtc(img, min, max, avg, std, minx, miny, maxx, maxy, N);
LINFO("SAVING COMPAT STATS TO %s",fileName.c_str());
std::ofstream statsFile(fileName.c_str(), std::ios::app);
statsFile << "final" << "\t";
statsFile << frameIdx << "\t";
// differentiate between scale image stats and the combined max norm
// for compatability with single channel stats
statsFile << "COMBINED\t-1\t";
//itsFrameIdx++;
statsFile << "final" << "\t" << "FINAL" << "\t";
statsFile << min << "\t" << max << "\t" << avg << "\t" << std << "\t"
<< minx << "\t" << miny << "\t" << maxx << "\t" << maxy << "\t"
<< N << "\n";
statsFile.close();
if(itsSaveStatsPerChannelFreq.getVal())
{
std::string txt = ".final.freq.txt";
fileName = itsGetSingleChannelStatsFile.getVal() + txt;
FFTWWrapper fft(img.getWidth(), img.getHeight());
double dimg[img.getHeight() * img.getWidth()];
Image<float>::const_iterator itr = img.begin();
double *ditr = &dimg[0];
while(itr != img.end()) *ditr++ = double(*itr++);
fft.init(dimg);
double mag[img.getHeight() * (img.getWidth()/2 + 1)];
fft.compute(mag);
std::ofstream freqFile(fileName.c_str(), std::ios::app);
freqFile << "final" << "\t";
freqFile << frameIdx << "\t";
freqFile << "COMBINED\t-1\t";
freqFile << "SIZE\t" << img.getWidth() <<"\t"<< img.getHeight() << "\n";
for(int i = 0; i < img.getHeight(); i++)
{
for(int j = 0; j < (img.getWidth()/2 + 1); j++)
freqFile << mag[i * (img.getWidth()/2 + 1) + j] << "\t";
freqFile << "\n";
}
freqFile.close();
}
}
int main(int argc, char** argv)
{
const std::string program_name = "hts_AdapterTrimmer";
std::string app_description =
"Adapter Trimmer, trims off adapters by first overlapping paired-end reads and\n";
app_description += " trimming off overhangs which by definition are adapter sequence in standard\n";
app_description += " libraries. SE Reads are trimmed by overlapping the adapter-sequence and trimming off the overlap.";
try
{
/** Define and parse the program options
*/
namespace po = boost::program_options;
po::options_description standard = setStandardOptions();
// version|v ; help|h ; notes|N ; stats-file|L ; append-stats-file|A
po::options_description input = setInputOptions();
// read1-input|1 ; read2-input|2 ; singleend-input|U
// tab-input|T ; interleaved-input|I ; from-stdin|S
po::options_description output = setOutputOptions(program_name);
// force|F ; prefix|p ; gzip-output,g ; fastq-output|f
// tab-output|t ; interleaved-output|i ; unmapped-output|u ; to-stdout,O
po::options_description desc("Application Specific Options");
setDefaultParamsCutting(desc);
// no-orphans|n ; stranded|s ; min-length|m
setDefaultParamsOverlapping(desc);
// kmer|k ; kmer-offset|r ; max-mismatch-errorDensity|x
// check-lengths|c ; min-overlap|o
desc.add_options()
("no-fixbases,X", po::bool_switch()->default_value(false), "after trimming adapter, DO NOT use consensus sequence of paired reads, only trims adapter sequence");
desc.add_options()
("adapter-sequence,a", po::value<std::string>()->default_value("AGATCGGAAGAGCACACGTCTGAACTCCAGTCA"), "Primer sequence to trim in SE adapter trimming, default is truseq ht primer sequence");
po::options_description cmdline_options;
cmdline_options.add(standard).add(input).add(output).add(desc);
po::variables_map vm;
try
{
po::store(po::parse_command_line(argc, argv, cmdline_options), vm); // can throw
version_or_help(program_name, app_description, cmdline_options, vm);
po::notify(vm); // throws on error, so do after help in case
std::string statsFile(vm["stats-file"].as<std::string>());
std::string prefix(vm["prefix"].as<std::string>());
AdapterCounters counters(statsFile, vm["append-stats-file"].as<bool>(), program_name, vm["notes"].as<std::string>());
std::shared_ptr<OutputWriter> pe = nullptr;
std::shared_ptr<OutputWriter> se = nullptr;
outputWriters(pe, se, vm["fastq-output"].as<bool>(), vm["tab-output"].as<bool>(), vm["interleaved-output"].as<bool>(), vm["unmapped-output"].as<bool>(), vm["force"].as<bool>(), vm["gzip-output"].as<bool>(), vm["to-stdout"].as<bool>(), prefix );
if(vm.count("read1-input")) { // paired-end reads
if (vm["read1-input"].as<std::vector<std::string> >().size() != vm["read1-input"].as<std::vector<std::string> >().size()) {
throw std::runtime_error("must have same number of input files for read1 and read2");
}
InputReader<PairedEndRead, PairedEndReadFastqImpl> ifr(vm["read1-input"].as<std::vector<std::string> >(), vm["read2-input"].as<std::vector<std::string> >());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if (vm.count("interleaved-input")) { // interleaved pairs
InputReader<PairedEndRead, InterReadImpl> ifr(vm["interleaved-input"].as<std::vector<std::string > >());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if(vm.count("singleend-input")) { // single-end reads
InputReader<SingleEndRead, SingleEndReadFastqImpl> ifr(vm["singleend-input"].as<std::vector<std::string> >());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if(vm.count("tab-input")) { // tab_input
InputReader<ReadBase, TabReadImpl> ifr(vm["tab-input"].as<std::vector<std::string> > ());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if(vm["from-stdin"].as<bool>()) { // stdin
bi::stream<bi::file_descriptor_source> tabin {fileno(stdin), bi::close_handle};
InputReader<ReadBase, TabReadImpl> ifr(tabin);
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
// no input specified on the command line
if (!vm.count("read1-input") && !vm.count("interleaved-input") && !vm.count("singleend-input") && !vm.count("tab-input") && !vm["from-stdin"].as<bool>()) {
std::cerr << "ERROR: " << "Input file type absent from command line" << std::endl << std::endl;
version_or_help(program_name, app_description, cmdline_options, vm, true);
exit(ERROR_IN_COMMAND_LINE); //success
}
counters.write_out();
}
catch(po::error& e)
{
std::cerr << "ERROR: " << e.what() << std::endl << std::endl;
return ERROR_IN_COMMAND_LINE;
}
}
catch(std::exception& e)
{
std::cerr << "\n\tUnhandled Exception: "
<< e.what() << std::endl;
return ERROR_UNHANDLED_EXCEPTION;
}
return SUCCESS;
}
int main(int argc, char** argv) {
// Declare the supported options.
knob::options_description desc("Allowed options");
desc.add_options()
("help", "produce help message")
(KnobStatsFile, knob::value<string>()->default_value("rcdcsim-stats.py"), "stats file to generate")
(KnobToSimulatorFifo, knob::value<string>(), "named fifo used to get events from the front-end")
// these are used to tag the output file, but don't affect the simulation at all
(KnobScheme, knob::value<string>()->default_value("<none/>"), "text describing this simulation setup")
(KnobWorkload, knob::value<string>()->default_value("<none/>"), "workload being run")
(KnobInput, knob::value<string>()->default_value("<none/>"), "input being used")
(KnobThreads, knob::value<unsigned>()->default_value(0), "number of threads in the workload")
// everything below are simulator parameters
(KnobCores, knob::value<unsigned>()->default_value(8), "number of cores to simulate")
(KnobIgnoreStackRefs, "Ignore stack accesses." )
// cache parameters
(KnobBlockSize, knob::value<unsigned>()->default_value(64), "Block size for all caches")
(KnobL1Size, knob::value<unsigned>()->default_value(1<<15/*32KB*/), "Size (in bytes) of each private L1 cache")
(KnobL1Assoc, knob::value<unsigned>()->default_value(8), "Associativity of each private L1 cache")
(KnobUseL2, "Model a private L2 for each core")
(KnobL2Size, knob::value<unsigned>()->default_value(1<<18/*256KB*/), "Size (in bytes) of the private L2 cache")
(KnobL2Assoc, knob::value<unsigned>()->default_value(8), "Associativity of the private L2 cache")
(KnobUseL3, "Model an L3 cache shared amongst all cores")
(KnobL3Size, knob::value<unsigned>()->default_value(1<<23/*8MB*/), "Size (in bytes) of the shared L3 cache")
(KnobL3Assoc, knob::value<unsigned>()->default_value(16), "Associativity of the shared L3 cache")
// RCDC
(KnobTSO, "Enable simulation of Det-TSO. Mutually exclusive with other Det-X schemes." )
(KnobHB, "Enable simulation of Det-HB. Mutually exclusive with other Det-X schemes." )
(KnobNondet, "Enable simulation of Nondet. Mutually exclusive with other Det-X schemes." )
(KnobQuantumSize, knob::value<unsigned>()->default_value(1000), "Quantum size (insns)")
(KnobSmartQuantumBuilding, "Use store buffer hit/miss information to deterministically estimate runtime when possible." )
;
knob::store( knob::parse_command_line(argc, argv, desc), s_knobs );
knob::notify(s_knobs);
if (s_knobs.count("help")) {
cout << desc << "\n";
return 1;
}
// can only simulate one execution strategy at a time
assert( s_knobs.count(KnobTSO) + s_knobs.count(KnobHB) + s_knobs.count(KnobNondet) == 1 );
time_t startTime = time( NULL );
MultiCacheSimulator<RCDCLine, uint64_t>* sim;
CacheConfiguration<RCDCLine> l1config, l2config, l3config;
l1config.blockSize = l2config.blockSize = l3config.blockSize = s_knobs[KnobBlockSize].as<unsigned>();
l1config.callbacks = l2config.callbacks = l3config.callbacks = NULL;
l1config.assoc = s_knobs[KnobL1Assoc].as<unsigned>();
l1config.cacheSize = s_knobs[KnobL1Size].as<unsigned>();
l2config.assoc = s_knobs[KnobL2Assoc].as<unsigned>();
l2config.cacheSize = s_knobs[KnobL2Size].as<unsigned>();
l3config.assoc = s_knobs[KnobL3Assoc].as<unsigned>();
l3config.cacheSize = s_knobs[KnobL3Size].as<unsigned>();
sim = new MultiCacheSimulator<RCDCLine, uint64_t> (
s_knobs[KnobCores].as<unsigned>(),
l1config,
s_knobs.count(KnobUseL2), l2config,
s_knobs.count(KnobUseL3), l3config );
// pass knobs values through to the caches
SMPCache<RCDCLine, uint64_t>::cache_iter_t it;
sim->m_simulateHB = s_knobs.count(KnobHB);
sim->m_simulateTSO = s_knobs.count(KnobTSO);
sim->m_quantumSize = s_knobs[KnobQuantumSize].as<unsigned>();
sim->m_smartQuantumBuilding = s_knobs.count(KnobSmartQuantumBuilding);
for ( it = sim->m_allCaches.begin(); it != sim->m_allCaches.end(); it++ ) {
// per-cache initialization goes here
(*it)->useDetStoreBuffers = (sim->m_simulateHB || sim->m_simulateTSO);
}
ifstream eventFifo;
eventFifo.open( s_knobs[KnobToSimulatorFifo].as<string>().c_str(), ios::binary );
assert( eventFifo.good() );
// main event loop
processEvents( sim, eventFifo );
cerr << "[rcdcsim] generating stats for " << nameOfDetStrategy() << endl;
eventFifo.close();
// each stat is dumped as a Python dictionary object
stringstream prefix;
/* prefix << "{'RCDCStat':True, ";
prefix << "'determinismStrategy': '" << nameOfDetStrategy() << "', ";
prefix << "'quantumSize': '" << (s_knobs[KnobQuantumSize].as<unsigned>()/1000) << "k', ";
prefix << "'smartQuantumBuilding': " << pyOfBool( s_knobs.count(KnobSmartQuantumBuilding) ) << ", ";
prefix << "'threads': " << s_knobs[KnobThreads].as<unsigned>() << ", ";
prefix << "'workload': '" << s_knobs[KnobWorkload].as<string>() << "', ";
prefix << "'input': '" << s_knobs[KnobInput].as<string>() << "', ";
prefix << "'scheme': '" << s_knobs[KnobScheme].as<string>() << "', ";
prefix << "'cores': '" << s_knobs[KnobCores].as<unsigned>() << "p', ";
prefix << "'ignoreStackRefs': " << pyOfBool( s_knobs.count(KnobIgnoreStackRefs) ) << ", ";
prefix << "'BlockSize': " << s_knobs[KnobBlockSize].as<unsigned>() << ", ";
prefix << "'l1Assoc': " << s_knobs[KnobL1Assoc].as<unsigned>() << ", ";
prefix << "'l1Size': " << s_knobs[KnobL1Size].as<unsigned>() << ", ";
prefix << "'useL2': " << pyOfBool( s_knobs.count(KnobUseL2) ) << ", ";
prefix << "'l2Assoc': " << s_knobs[KnobL2Assoc].as<unsigned>() << ", ";
prefix << "'l2Size': " << s_knobs[KnobL2Size].as<unsigned>() << ", ";
prefix << "'useL3': " << pyOfBool( s_knobs.count(KnobUseL3) ) << ", ";
prefix << "'l3Assoc': " << s_knobs[KnobL3Assoc].as<unsigned>() << ", ";
prefix << "'l3Size': " << s_knobs[KnobL3Size].as<unsigned>() << ", ";*/
// actual stat value goes here
string suffix = "}\n";
// check for filename collisions and rename around them
string statsFilename = s_knobs["statsfile"].as<string>();
while ( fileExists( statsFilename ) ) {
statsFilename += ".1";
}
ofstream statsFile( statsFilename.c_str(), ios_base::trunc );
// dump stats from the caches
sim->dumpStats( statsFile, prefix.str(), suffix );
// dump "global" stats
double minutes = difftime( time( NULL ), startTime ) / 60.0;
statsFile << prefix.str() << "'SimulationRunningTimeMinutes': " << minutes << suffix;
statsFile << prefix.str() << "'maxLiveThreads': " << s_maxLiveThreads << suffix;
statsFile << prefix.str() << "'numSpawnedThreads': " << s_numSpawnedThreads << suffix;
statsFile << prefix.str() << "'numStackAccesses': " << s_stackAccesses << suffix;
statsFile << prefix.str() << "'numTotalInstructions': " << s_insnsExecuted << suffix;
statsFile << prefix.str() << "'causalityInducedEventDelays': " << s_causalityDelays << suffix;
statsFile << prefix.str() << "'unprocessedEvents': " << s_unprocessedEvents << suffix;
statsFile << prefix.str() << "'forcedCommits': " << s_forcedCommits << suffix;
statsFile.close();
cerr << "[rcdcsim] finished generating stats for " << nameOfDetStrategy() << endl;
delete sim;
cerr << "[rcdcsim] simulation process exiting" << endl;
return 0;
}
/////////////////////////////////////////////////////////////////////////////
/// main looop
void DeviceMonitor::Main( ) {
assert( dev_monitor_ );
const int fd_mon = udev_monitor_get_fd( dev_monitor_ );
const int nfds = fd_mon + 1;
int queue_tok = 8;
sigset_t sigempty;
sigemptyset( &sigempty );
struct timespec timeout;
if( activity )
{
timeout.tv_sec = 0;
//timeout.tv_sec = 1;
timeout.tv_nsec = 100000000;
//timeout.tv_nsec = 0;
}
else
{
timeout.tv_sec = 999;
timeout.tv_nsec = 0;
}
while ( true ) {
fd_set fds_read;
FD_ZERO( &fds_read );
FD_SET( fd_mon, &fds_read );
// block for something interesting to happen
int res = pselect( nfds, &fds_read, 0, 0, &timeout, &sigempty );
if ( res < 0 ) {
if ( EINTR != errno ) throw ErrnoException( "select" );
std::cout << "Exiting on signal\n";
return; // signalled
}
// udev monitor notification?
if ( FD_ISSET( fd_mon, &fds_read ) ) {
std::tr1::shared_ptr< udev_device > device( udev_monitor_receive_device( dev_monitor_ ), &udev_device_unref );
// make sure this is a device we want to monitor
if (!acceptDevice_(device.get())) continue;
const char* str = udev_device_get_action( device.get() );
if ( !str ) {
} else if ( 0 == strcasecmp( str, "add" ) ) {
deviceAdded_( device.get() );
} else if ( 0 == strcasecmp( str, "remove" ) ) {
deviceRemove_( device.get() );
} else {
if ( debug ) {
std::cout << "action: " << str << '\n';
std::cout << ' ' << udev_device_get_syspath(device.get()) << "' (" << udev_device_get_subsystem(device.get()) << ")\n";
}
}
}
if( activity )
{
for( int i = 0; i < num_disks_; ++i )
{
std::ifstream stats;
stats.open( statsFile(i).c_str() );
char buf[256];
stats.getline( &(buf[0]), 255 );
std::string s(&(buf[0]));
// tokenize the string
std::string::size_type last = s.find_first_not_of( " ", 0 );
std::string::size_type pos = s.find_first_of( " ", last );
int tok = 0;
int queue_length = 0;
while( std::string::npos != pos || std::string::npos != last )
{
last = s.find_first_not_of(" ", pos );
pos = s.find_first_of( " ", last );
++tok;
if( tok == queue_tok )
{
queue_length = atoi(s.substr( last, pos - last ).c_str());
if( debug )
std::cout << " " << i << " " << queue_length;
break;
}
}
int led_idx = ledIndex(i);
if( led_enabled_[led_idx] && leds_ )
{
if( queue_length > 0 )
{
leds_->Set( LED_BLUE, led_idx, true );
leds_->Set( LED_RED, led_idx, true );
}
else
{
leds_->Set( LED_BLUE, led_idx, true );
leds_->Set( LED_RED, led_idx, false );
}
}
}
}
if( debug )
std::cout << "\n";
}
}
