/** Executes the algorithm
*@param localworkspace :: the input workspace
*@param indices :: set of indices to sum up
*/
void SumSpectra::execEvent(EventWorkspace_const_sptr localworkspace,
std::set<int> &indices) {
auto outputWorkspace = create<EventWorkspace>(*localworkspace, 1);
Progress progress(this, 0, 1, indices.size());
// Get the pointer to the output event list
EventList &outEL = outputWorkspace->getSpectrum(0);
outEL.setSpectrumNo(m_outSpecNum);
outEL.clearDetectorIDs();
const auto &spectrumInfo = localworkspace->spectrumInfo();
// Loop over spectra
size_t numSpectra(0);
size_t numMasked(0);
size_t numZeros(0);
for (const auto i : indices) {
// Don't go outside the range.
if ((i >= m_numberOfSpectra) || (i < 0)) {
g_log.error() << "Invalid index " << i
<< " was specified. Sum was aborted.\n";
break;
}
if (spectrumInfo.hasDetectors(i)) {
// Skip monitors, if the property is set to do so
if (!m_keepMonitors && spectrumInfo.isMonitor(i))
continue;
// Skip masked detectors
if (spectrumInfo.isMasked(i)) {
numMasked++;
continue;
}
}
numSpectra++;
// Add the event lists with the operator
const EventList &tOutEL = localworkspace->getSpectrum(i);
if (tOutEL.empty()) {
++numZeros;
}
outEL += tOutEL;
progress.report();
}
outputWorkspace->mutableRun().addProperty("NumAllSpectra", int(numSpectra),
"", true);
outputWorkspace->mutableRun().addProperty("NumMaskSpectra", int(numMasked),
"", true);
outputWorkspace->mutableRun().addProperty("NumZeroSpectra", int(numZeros), "",
true);
// Assign it to the output workspace property
setProperty("OutputWorkspace", std::move(outputWorkspace));
}
int main(int argc, char* argv[])
{
std::ofstream logFile("dump_log.txt");
/*
// output test spectra.
double mioSpectraComparePerSecond;
double mioSpectraAdaptionPerSecond;
SpectraBaseHelpers::testSpectraPerformance( mioSpectraComparePerSecond, mioSpectraAdaptionPerSecond );
Helpers::print( Helpers::numberToString<double>(mioSpectraComparePerSecond) +std::string(" million spectra compares per second.\n"), &logFile );
Helpers::print( Helpers::numberToString<double>(mioSpectraAdaptionPerSecond) +std::string(" million spectra adaption per second.\n"), &logFile );
return;
*/
Helpers::print("Welcome to SDSS Dump "+sstrSDSSVersionString+" !\n\n\n", &logFile);
Helpers::print("Dump can do the following:\n", &logFile);
Helpers::print("(1) Read SDSS spectra as FITS files from a given directory (and subdirectories) and dumps the data to a single binary file. Use -d\n", &logFile);
Helpers::print(" DR12 spectra can be downloaded here: http://data.sdss3.org/sas/dr12/sdss/spectro/redux/26/spectra \n\n", &logFile);
Helpers::print("(2) Generation of n sine test spectra with increasing frequency. Use -t\n\n", &logFile);
Helpers::print("(3) Reads binary dump files and extracts text tables out of it. Use -i\n\n", &logFile);
Helpers::print("(4) Uploads spectra from binary dump files to ASPECT-FPGA-Accelerator (AFA). Use -a\n\n", &logFile);
std::string sstrDataDir = FileHelpers::getCurrentDirectory()+DATADIR;
std::string sstrDumpFile = DUMPFILE;
int sineTestSpectra = 0;
unsigned int spectraFilter = SPT_DEFAULTFILTER;
std::string sstrInputDumpFile("");
std::string sstrAfaDumpFile("");
std::string sstrSelectionListFilename("");
try {
std::string sstrExamples("examples:\n");
sstrExamples += std::string("Write FITS files to binary dump file: \n dump.exe -d F:/SDSS_ANALYZE/fits/spectro/data/* -o allSpectra.bin -f 25 -s selectionlist.txt\n");
sstrExamples += std::string("Write 1000 sine test spectra: \n dump.exe -t 1000 -o allSpectra.bin\n");
sstrExamples += std::string("Outputs a linear list of the network: \n dump.exe -i sofmnet.bin\n");
sstrExamples += std::string("Uploads spectra dump to ASPECT-FPGA-Accelerator (AFA) \n dump.exe -a allSpectra.bin\n");
TCLAP::CmdLine cmd(sstrExamples, ' ', sstrSDSSVersionString);
std::string sstrFilterDesc = std::string("where <filter> is any added combination of:\n");
sstrFilterDesc += std::string( " SPEC_UNKNOWN = 2\n");
sstrFilterDesc += std::string( " SPEC_STAR = 4\n");
sstrFilterDesc += std::string( " SPEC_GALAXY = 8\n");
sstrFilterDesc += std::string( " SPEC_QSO = 16\n");
TCLAP::ValueArg<std::string> dataDirArg("d", "datadir", "example: F:/SDSS_ANALYZE/fits/spectro/data/*", false, sstrDataDir, "datadir/*");
TCLAP::ValueArg<unsigned int> sineTestArg("t", "sinetest", "generate n sine test spectra", false, 0, "Number of sine test spectra.");
TCLAP::ValueArg<std::string> outputFilenameArg("o", "outputdumpfile", "example: allSpectra.bin", false, sstrDumpFile, "outputfilename.bin");
TCLAP::ValueArg<unsigned int> filterArg("f", "filter", sstrFilterDesc, false, spectraFilter, "Dump only FITS files with the given filter type.");
TCLAP::ValueArg<std::string> inputFilenameArg("i", "inputdumpfile", "example: sofmnet.bin. If input dump file is specified, then all other arguments are ignored. Outputs a linear list of the network.", false, sstrInputDumpFile, "Dumpfile for reverse reads.");
TCLAP::ValueArg<std::string> selectionListFilenameArg("s", "selection", "Optional selection list of FITS files to dump a small subset of input spectra. File should contain plate-mjd-fiber pairs, e.g. 3586 55181 0001. First line in the file is the header and is ignored.", false, sstrSelectionListFilename, "selectionlist.txt");
TCLAP::ValueArg<std::string> afaFilenameArg("a", "afaupload", "example: allSpectra.bin. If afa upload is specified, then upload spectra dump to ASPECT-FPGA-Accelerator (AFA). All other arguments are ignored.", false, sstrAfaDumpFile, "Dumpfile for AFA upload.");
cmd.add( dataDirArg );
cmd.add( outputFilenameArg );
cmd.add( sineTestArg );
cmd.add( filterArg );
cmd.add( inputFilenameArg );
cmd.add( selectionListFilenameArg );
cmd.add( afaFilenameArg );
cmd.parse( argc, argv );
sstrDataDir = dataDirArg.getValue();
sstrDumpFile = outputFilenameArg.getValue();
sineTestSpectra = sineTestArg.getValue();
spectraFilter = filterArg.getValue();
sstrInputDumpFile = inputFilenameArg.getValue();
sstrAfaDumpFile = afaFilenameArg.getValue();
sstrSelectionListFilename = selectionListFilenameArg.getValue();
}
catch (TCLAP::ArgException &e)
{
Helpers::print( "error: "+e.error()+" for argument "+e.argId()+"\n", &logFile );
}
const bool bExtractFilenames = !sstrInputDumpFile.empty();
const bool bAfaUpload = !sstrAfaDumpFile.empty();
if ( sineTestSpectra > 0 )
{
// generate sine test spectra.
///////////////////////////////////////////////////////////////////////////////////
Helpers::print( "Generating : "+Helpers::numberToString<int>(sineTestSpectra)+" sine test spectra.\n", &logFile );
Helpers::print( "dumpfile: "+sstrInputDumpFile+"\n", &logFile );
SpectraVFS::write( sineTestSpectra, 0.0f, sstrDumpFile );
Helpers::print( "fin.\n", &logFile );
return 0;
}
else if ( bExtractFilenames )
{
// extract FITS filenames from a given binary dump.
///////////////////////////////////////////////////////////////////////////////////
Helpers::print( "Filename extraction from dumpfile with following parameters:\n", &logFile);
Helpers::print( "dumpfile: "+sstrInputDumpFile+"\n", &logFile );
SpectraVFS vfs(sstrInputDumpFile, true);
size_t numSpectra( vfs.getNumSpectra() );
if ( numSpectra == 0)
{
return 1;
}
std::string sstrOutFileName(FileHelpers::getFileNameMinusExtension(sstrDumpFile));
sstrOutFileName += ".txt";
std::ofstream fon(sstrOutFileName.c_str());
for (size_t i=0;i<numSpectra;i++)
{
Spectra *a = vfs.beginRead(i);
fon << a->getFileName() + std::string("\n");
vfs.endRead(i);
}
Helpers::print( "fin.\n", &logFile );
return 0;
}
else if ( bAfaUpload )
{
// upload spectra dump file to AFA
///////////////////////////////////////////////////////////////////////////////////
Helpers::print( "Uploading spectra to ASPECT-FPGA-Accelerator (AFA) with following parameters:\n", &logFile);
Helpers::print( "dumpfile: "+sstrAfaDumpFile+"\n", &logFile );
SpectraVFS vfs(sstrAfaDumpFile, true);
size_t numSpectra( vfs.getNumSpectra() );
if ( numSpectra == 0)
{
return 1;
}
AfaConnector afaConnector;
if ( !afaConnector.isAFADeviceAvailable() )
{
Helpers::print( "Could not find ASPECT-FPGA-Accelerator:\n", &logFile );
Helpers::print( afaConnector.getErrorMsg()+"\n", &logFile );
return 1;
}
if ( !afaConnector.writeSpectra(vfs) )
{
Helpers::print( "Error transferring data to ASPECT-FPGA-Accelerator:\n", &logFile );
Helpers::print( afaConnector.getErrorMsg(), &logFile );
return 1;
}
Helpers::print( "Finished transfer of "+ Helpers::numberToString<int>(numSpectra) +" spectra to ASPECT-FPGA-Accelerator.\n", &logFile );
return 0;
}
else
{
// load spectra and pack all necessary information into a single binary file.
///////////////////////////////////////////////////////////////////////////////////
Helpers::print( "Creating binary dump with following parameters:\n", &logFile);
Helpers::print( "datadir: "+sstrDataDir+"\n", &logFile );
Helpers::print( "dumpfile: "+sstrDumpFile+"\n", &logFile );
Helpers::print( "filter: "+Spectra::spectraFilterToString(spectraFilter)+"\n", &logFile );
Helpers::print( "selectionlist: "+sstrSelectionListFilename+"\n", &logFile );
Helpers::print( "each spectrum contains " + Helpers::numberToString<size_t>(sizeof(Spectra)) + " bytes.\n\n", &logFile );
std::set<std::string> FITSFilenameSet;
if ( !sstrSelectionListFilename.empty() )
{
bool bSuccess = SpectraHelpers::readSelectionList(sstrSelectionListFilename, FITSFilenameSet);
if ( !bSuccess || FITSFilenameSet.empty() )
{
Helpers::print( "Selection list not found or empty.\n", &logFile );
}
}
Helpers::print( "starting dump...\n", &logFile );
size_t writtenSpectra = SpectraVFS::write( sstrDataDir, sstrDumpFile, spectraFilter, &logFile, &FITSFilenameSet );
Helpers::print( "...finished writing "+ Helpers::numberToString<size_t>(writtenSpectra) +" spectra.\n", &logFile );
}
return 0;
}
/** Executes the algorithm
*@param localworkspace :: the input workspace
*@param indices :: set of indices to sum up
*/
void SumSpectra::execEvent(EventWorkspace_const_sptr localworkspace,
std::set<int> &indices) {
// Make a brand new EventWorkspace
EventWorkspace_sptr outputWorkspace =
boost::dynamic_pointer_cast<EventWorkspace>(
API::WorkspaceFactory::Instance().create("EventWorkspace", 1, 2, 1));
// Copy geometry over.
API::WorkspaceFactory::Instance().initializeFromParent(localworkspace,
outputWorkspace, true);
Progress progress(this, 0, 1, indices.size());
// Get the pointer to the output event list
EventList &outEL = outputWorkspace->getEventList(0);
outEL.setSpectrumNo(m_outSpecId);
outEL.clearDetectorIDs();
// Loop over spectra
std::set<int>::iterator it;
size_t numSpectra(0);
size_t numMasked(0);
size_t numZeros(0);
// for (int i = m_minSpec; i <= m_maxSpec; ++i)
for (it = indices.begin(); it != indices.end(); ++it) {
int i = *it;
// Don't go outside the range.
if ((i >= m_numberOfSpectra) || (i < 0)) {
g_log.error() << "Invalid index " << i
<< " was specified. Sum was aborted.\n";
break;
}
try {
// Get the detector object for this spectrum
Geometry::IDetector_const_sptr det = localworkspace->getDetector(i);
// Skip monitors, if the property is set to do so
if (!m_keepMonitors && det->isMonitor())
continue;
// Skip masked detectors
if (det->isMasked()) {
numMasked++;
continue;
}
} catch (...) {
// if the detector not found just carry on
}
numSpectra++;
// Add the event lists with the operator
const EventList &tOutEL = localworkspace->getEventList(i);
if (tOutEL.empty()) {
++numZeros;
}
outEL += tOutEL;
progress.report();
}
// Set all X bins on the output
cow_ptr<MantidVec> XValues;
XValues.access() = localworkspace->readX(0);
outputWorkspace->setAllX(XValues);
outputWorkspace->mutableRun().addProperty("NumAllSpectra", int(numSpectra),
"", true);
outputWorkspace->mutableRun().addProperty("NumMaskSpectra", int(numMasked),
"", true);
outputWorkspace->mutableRun().addProperty("NumZeroSpectra", int(numZeros), "",
true);
// Assign it to the output workspace property
setProperty("OutputWorkspace",
boost::dynamic_pointer_cast<MatrixWorkspace>(outputWorkspace));
}
/** Executes the algorithm
*
*/
void SumSpectra::exec()
{
// Try and retrieve the optional properties
m_MinSpec = getProperty("StartWorkspaceIndex");
m_MaxSpec = getProperty("EndWorkspaceIndex");
const std::vector<int> indices_list = getProperty("ListOfWorkspaceIndices");
keepMonitors = getProperty("IncludeMonitors");
// Get the input workspace
MatrixWorkspace_const_sptr localworkspace = getProperty("InputWorkspace");
numberOfSpectra = static_cast<int>(localworkspace->getNumberHistograms());
this->yLength = static_cast<int>(localworkspace->blocksize());
// Check 'StartSpectrum' is in range 0-numberOfSpectra
if ( m_MinSpec > numberOfSpectra )
{
g_log.warning("StartWorkspaceIndex out of range! Set to 0.");
m_MinSpec = 0;
}
if (indices_list.empty())
{
//If no list was given and no max, just do all.
if ( isEmpty(m_MaxSpec) ) m_MaxSpec = numberOfSpectra-1;
}
//Something for m_MaxSpec was given but it is out of range?
if (!isEmpty(m_MaxSpec) && ( m_MaxSpec > numberOfSpectra-1 || m_MaxSpec < m_MinSpec ))
{
g_log.warning("EndWorkspaceIndex out of range! Set to max Workspace Index");
m_MaxSpec = numberOfSpectra;
}
//Make the set of indices to sum up from the list
this->indices.insert(indices_list.begin(), indices_list.end());
//And add the range too, if any
if (!isEmpty(m_MaxSpec))
{
for (int i = m_MinSpec; i <= m_MaxSpec; i++)
this->indices.insert(i);
}
//determine the output spectrum id
m_outSpecId = this->getOutputSpecId(localworkspace);
g_log.information() << "Spectra remapping gives single spectra with spectra number: "
<< m_outSpecId << "\n";
m_CalculateWeightedSum = getProperty("WeightedSum");
EventWorkspace_const_sptr eventW = boost::dynamic_pointer_cast<const EventWorkspace>(localworkspace);
if (eventW)
{
m_CalculateWeightedSum = false;
this->execEvent(eventW, this->indices);
}
else
{
//-------Workspace 2D mode -----
// Create the 2D workspace for the output
MatrixWorkspace_sptr outputWorkspace = API::WorkspaceFactory::Instance().create(localworkspace,
1,localworkspace->readX(0).size(),this->yLength);
size_t numSpectra(0); // total number of processed spectra
size_t numMasked(0); // total number of the masked and skipped spectra
size_t numZeros(0); // number of spectra which have 0 value in the first column (used in special cases of evaluating how good Puasonian statistics is)
Progress progress(this, 0, 1, this->indices.size());
// This is the (only) output spectrum
ISpectrum * outSpec = outputWorkspace->getSpectrum(0);
// Copy over the bin boundaries
outSpec->dataX() = localworkspace->readX(0);
//Build a new spectra map
outSpec->setSpectrumNo(m_outSpecId);
outSpec->clearDetectorIDs();
if (localworkspace->id() == "RebinnedOutput")
{
this->doRebinnedOutput(outputWorkspace, progress,numSpectra,numMasked,numZeros);
}
else
{
this->doWorkspace2D(localworkspace, outSpec, progress,numSpectra,numMasked,numZeros);
}
// Pointer to sqrt function
MantidVec& YError = outSpec->dataE();
typedef double (*uf)(double);
uf rs=std::sqrt;
//take the square root of all the accumulated squared errors - Assumes Gaussian errors
std::transform(YError.begin(), YError.end(), YError.begin(), rs);
outputWorkspace->generateSpectraMap();
// set up the summing statistics
outputWorkspace->mutableRun().addProperty("NumAllSpectra",int(numSpectra),"",true);
outputWorkspace->mutableRun().addProperty("NumMaskSpectra",int(numMasked),"",true);
outputWorkspace->mutableRun().addProperty("NumZeroSpectra",int(numZeros),"",true);
// Assign it to the output workspace property
setProperty("OutputWorkspace", outputWorkspace);
}
}
/**
* Reads the data from the file. It is assumed that the provided file stream has its position
* set such that the first call to getline will be give the first line of data
* @param file :: A reference to a file stream
* @returns A pointer to a new workspace
*/
API::Workspace_sptr LoadAscii::readData(std::ifstream & file) const
{
// Get the first line and find the number of spectra from the number of columns
std::string line;
getline(file,line);
boost::trim(line);
std::list<std::string> columns;
const int numCols = splitIntoColumns(columns, line);
if( numCols < 2 )
{
g_log.error() << "Invalid data format found in file \"" << getPropertyValue("Filename") << "\"\n";
throw std::runtime_error("Invalid data format. Fewer than 2 columns found.");
}
size_t numSpectra(0);
bool haveErrors(false);
bool haveXErrors(false);
// Assume single data set with no errors
if( numCols == 2 )
{
numSpectra = numCols/2;
}
// Data with errors
else if( (numCols-1) % 2 == 0 )
{
numSpectra = (numCols - 1)/2;
haveErrors = true;
}
// Data with errors on both X and Y (4-column file)
else if( numCols == 4 )
{
numSpectra = 1;
haveErrors = true;
haveXErrors = true;
}
else
{
g_log.error() << "Invalid data format found in file \"" << getPropertyValue("Filename") << "\"\n";
g_log.error() << "LoadAscii requires the number of columns to be an even multiple of either 2 or 3.";
throw std::runtime_error("Invalid data format.");
}
// A quick check at the number of lines won't be accurate enough as potentially there
// could be blank lines and comment lines
int numBins(0), lineNo(0);
std::vector<DataObjects::Histogram1D> spectra(numSpectra);
std::vector<double> values(numCols, 0.);
do
{
++lineNo;
boost::trim(line);
if( this->skipLine(line) ) continue;
columns.clear();
int lineCols = this->splitIntoColumns(columns, line);
if( lineCols != numCols )
{
std::ostringstream ostr;
ostr << "Number of columns changed at line " << lineNo;
throw std::runtime_error(ostr.str());
}
try
{
fillInputValues(values, columns); //ignores nans and replaces them with 0
}
catch(boost::bad_lexical_cast&)
{
g_log.error() << "Invalid value on line " << lineNo << " of \""
<< getPropertyValue("Filename") << "\"\n";
throw std::runtime_error("Invalid value encountered.");
}
for (size_t i = 0; i < numSpectra; ++i)
{
spectra[i].dataX().push_back(values[0]);
spectra[i].dataY().push_back(values[i*2+1]);
if( haveErrors )
{
spectra[i].dataE().push_back(values[i*2+2]);
}
else
{
spectra[i].dataE().push_back(0.0);
}
if( haveXErrors )
{
// Note: we only have X errors with 4-column files.
// We are only here when i=0.
spectra[i].dataDx().push_back(values[3]);
}
else
{
spectra[i].dataDx().push_back(0.0);
}
}
++numBins;
}
while(getline(file,line));
MatrixWorkspace_sptr localWorkspace = boost::dynamic_pointer_cast<MatrixWorkspace>
(WorkspaceFactory::Instance().create("Workspace2D",numSpectra,numBins,numBins));
try
{
localWorkspace->getAxis(0)->unit() = UnitFactory::Instance().create(getProperty("Unit"));
}
catch (Exception::NotFoundError&)
{
// Asked for dimensionless workspace (obviously not in unit factory)
}
for (size_t i = 0; i < numSpectra; ++i)
{
localWorkspace->dataX(i) = spectra[i].dataX();
localWorkspace->dataY(i) = spectra[i].dataY();
localWorkspace->dataE(i) = spectra[i].dataE();
// Just have spectrum number start at 1 and count up
localWorkspace->getAxis(1)->spectraNo(i) = static_cast<specid_t>(i+1);
}
return localWorkspace;
}
