// >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
int Inventory::getAllStations(StationList& stationList,
const Core::Time& t) {
if ( _inventory == NULL )
return 0;
std::set<std::string> networkMap;
for ( size_t i = 0; i < _inventory->networkCount(); ++i ) {
DataModel::Network* network = _inventory->network(i);
if ( networkMap.find(network->code()) != networkMap.end() ) continue;
try {
if ( network->end() < t ) continue;
}
catch (...) {}
if ( network->start() > t ) continue;
networkMap.insert(network->code());
std::set<std::string> stationMap;
for ( size_t j = 0; j < network->stationCount(); ++j ) {
DataModel::Station* station = network->station(j);
if ( stationMap.find(station->code()) != stationMap.end() ) continue;
try {
if ( station->end() < t ) continue;
}
catch (...) {}
if ( station->start() > t ) continue;
stationList.push_back(station);
stationMap.insert(station->code());
}
}
return stationList.size();
}
bool syncInventory() {
DataModel::Inventory *targetInv = Client::Inventory::Instance()->inventory();
bool createNotifier = commandline().hasOption("create-notifier");
bool testMode = commandline().hasOption("test");
if ( testMode || _output.empty() ) createNotifier = true;
if ( targetInv == NULL ) {
cerr << "No inventory to sync" << endl;
return false;
}
// Disable object registration
DataModel::PublicObject::SetRegistrationEnabled(false);
// Disable notifier check
DataModel::Notifier::SetCheckEnabled(false);
vector<string> files;
collectFiles(files);
if ( _filebase.empty() && files.empty() ) {
cerr << "Nothing to merge, no files given" << endl;
return false;
}
DataModel::InventoryPtr mergedInventory = new DataModel::Inventory();
Merge merger(mergedInventory.get());
merger.setLogHandler(this);
_continueOperation = true;
_currentTask = &merger;
DataModel::Notifier::SetEnabled(false);
for ( size_t i = 0; i < files.size(); ++i ) {
if ( _exitRequested ) break;
IO::XMLArchive ar;
if ( !ar.open(files[i].c_str()) ) {
cerr << "Could not open file (ignored): " << files[i] << endl;
continue;
}
DataModel::InventoryPtr inv;
cerr << "Parsing " << files[i] << " ... " << flush;
ar >> inv;
cerr << "done" << endl;
if ( !inv ) {
cerr << "No inventory found (ignored): " << files[i] << endl;
continue;
}
_inventorySources[inv.get()] = files[i];
// Pushing the inventory into the merger cleans it
// completely. The ownership of all childs goes to
// the merger
merger.push(inv.get());
}
if ( _exitRequested ) {
cerr << "Exit requested: abort" << endl;
return false;
}
cerr << "Merging inventory ... " << flush;
if ( merger.merge(false) )
cerr << "done" << endl;
else
cerr << "failed" << endl;
printLogs();
if ( !_continueOperation ) {
cerr << "Unresolvable errors ... aborting" << endl;
return false;
}
if ( _exitRequested ) {
cerr << "Exit requested: abort" << endl;
return false;
}
// Activate registration again
DataModel::PublicObject::SetRegistrationEnabled(true);
Sync syncTask(targetInv);
_currentTask = &syncTask;
if ( createNotifier ) DataModel::Notifier::SetEnabled(true);
cerr << "Synchronising inventory ... " << flush;
if ( syncTask.push(mergedInventory.get()) )
cerr << "done";
else
cerr << "failed";
cerr << endl;
if ( _exitRequested ) {
cerr << "Exit requested: abort" << endl;
return false;
}
if ( createNotifier ) DataModel::Notifier::SetEnabled(true);
cerr << "Removing remaining objects ... " << flush;
syncTask.cleanUp();
cerr << "done" << endl;
// --- Check key files
// Collect all station key files
map<string,string> keyFiles;
try {
fs::path directory = SC_FS_PATH(_keydir);
fs::directory_iterator it(directory);
fs::directory_iterator dirEnd;
for ( ; it != dirEnd; ++it ) {
if ( fs::is_directory(SC_FS_IT_PATH(it)) ) continue;
string name = SC_FS_IT_LEAF(it);
if ( name.compare(0, 8, "station_") == 0 )
keyFiles[SC_FS_IT_LEAF(it)] = SC_FS_IT_STR(it);
}
}
catch ( ... ) {}
for ( size_t n = 0; n < mergedInventory->networkCount(); ++n ) {
DataModel::Network *net = mergedInventory->network(n);
for ( size_t s = 0; s < net->stationCount(); ++s ) {
DataModel::Station *sta = net->station(s);
string id = net->code() + "_" + sta->code();
string filename = "station_" + id;
// Remove filename from keyFiles
map<string,string>::iterator it = keyFiles.find(filename);
if ( it != keyFiles.end() ) keyFiles.erase(it);
}
}
// Warn about existing key files without a corresponding station
// in inventory
if ( !keyFiles.empty() ) {
SEISCOMP_WARNING("Found %d key file%s without a corresponding "
"station in inventory:",
(int)keyFiles.size(), keyFiles.size() > 1?"s":"");
for ( map<string,string>::iterator it = keyFiles.begin();
it != keyFiles.end(); ++it )
SEISCOMP_WARNING(" %s", it->second.c_str());
}
DataModel::Notifier::SetEnabled(false);
_currentTask = NULL;
if ( _exitRequested ) {
cerr << "Exit requested: abort" << endl;
return false;
}
bool doSyncKeys = false;
if ( createNotifier ) {
DataModel::NotifierMessagePtr nmsg;
size_t notifierCount = DataModel::Notifier::Size();
if ( notifierCount > 0 ) {
cerr << notifierCount << " notifiers available" << endl;
if ( !_output.empty() ) {
IO::XMLArchive ar;
if ( !ar.create(_output.c_str()) ) {
cerr << "Failed to create output file: " << _output << endl;
DataModel::Notifier::Clear();
return false;
}
cerr << "Generating output ... " << flush;
ar.setFormattedOutput(true);
nmsg = DataModel::Notifier::GetMessage(true);
ar << nmsg;
ar.close();
cerr << "done" << endl;
}
else if ( !testMode ) {
// Notify about start of synchronization
DataModel::InventorySyncMessagePtr ismsg = new DataModel::InventorySyncMessage(false);
ismsg->setCreationInfo(DataModel::CreationInfo());
ismsg->creationInfo().setCreationTime(Core::Time::GMT());
ismsg->creationInfo().setAuthor(author());
ismsg->creationInfo().setAgencyID(agencyID());
connection()->send(Communication::Protocol::STATUS_GROUP, ismsg.get());
// Send an inital sync command to also wake-up the messaging
sync();
// Send notifier
DataModel::NotifierMessagePtr tmp = new DataModel::NotifierMessage();
DataModel::NotifierMessage::iterator it;
int count = 0;
// Fetch each single notifier message. Fetching all notifiers
// in one message can take a long time if a huge amount of
// notifiers is in the queue. Due to memory fragmentation
// most of the time spent is in malloc.
while ( (nmsg = DataModel::Notifier::GetMessage(false)) != NULL ) {
if ( _exitRequested ) break;
for ( it = nmsg->begin(); it != nmsg->end(); ++it ) {
DataModel::Notifier* n = DataModel::Notifier::Cast(*it);
if ( !n ) continue;
tmp->attach(n);
++count;
if ( count % 100 == 0 ) {
cerr << "\rSending notifiers: " << (int)(count*100/notifierCount) << "%" << flush;
if ( !send(tmp.get()) ) {
SEISCOMP_ERROR("Failed to send message, abort");
return false;
}
tmp->clear();
sync();
}
}
}
if ( !_exitRequested && !tmp->empty() ) {
if ( !send(tmp.get()) ) {
SEISCOMP_ERROR("Failed to send message, abort");
return false;
}
cerr << "\rSending notifiers: " << (int)(count*100/notifierCount) << "%" << flush;
}
cerr << endl;
sync();
// Notify about end of synchronization
ismsg->creationInfo().setCreationTime(Core::Time::GMT());
ismsg->isFinished = true;
connection()->send(Communication::Protocol::STATUS_GROUP, ismsg.get());
doSyncKeys = true;
}
else {
/*
DataModel::NotifierMessage::iterator it;
while ( (nmsg = DataModel::Notifier::GetMessage(false)) != NULL ) {
for ( it = nmsg->begin(); it != nmsg->end(); ++it ) {
DataModel::Notifier* n = DataModel::Notifier::Cast(*it);
if ( !n ) continue;
switch ( n->operation() ) {
case DataModel::OP_ADD:
cout << "+";
break;
case DataModel::OP_REMOVE:
cout << "-";
break;
case DataModel::OP_UPDATE:
cout << "M";
break;
default:
cout << "?";
break;
}
cout << " " << n->object()->className() << endl;
}
}
*/
DataModel::Notifier::Clear();
cout << "OK - synchronization test passed" << endl;
}
}
else
cerr << "Inventory is synchronised already, nothing to do" << endl;
DataModel::Notifier::Clear();
}
else {
if ( _output.empty() ) _output = "-";
IO::XMLArchive ar;
if ( !ar.create(_output.c_str()) ) {
cerr << "Failed to create output file: " << _output << endl;
return false;
}
cerr << "Generating output ... " << flush;
ar.setFormattedOutput(true);
ar << targetInv;
ar.close();
cerr << "done" << endl;
}
if ( doSyncKeys ) {
if ( !commandline().hasOption("no-rc") ) {
if ( !syncRCFiles(targetInv) ) return false;
}
if ( !commandline().hasOption("no-keys") ) {
bool syncKeys = true;
try { syncKeys = configGetBool("syncKeys"); } catch (...) {}
if ( syncKeys && !syncKeyFiles(targetInv) ) return false;
}
}
return true;
}
bool listNetworks() {
// Disable object registration
DataModel::PublicObject::SetRegistrationEnabled(false);
vector<string> files;
collectFiles(files);
if ( files.empty() ) {
cerr << "Nothing to merge, no files given" << endl;
return false;
}
DataModel::InventoryPtr finalInventory = new DataModel::Inventory();
Merge merger(finalInventory.get());
_currentTask = &merger;
for ( size_t i = 0; i < files.size(); ++i ) {
if ( _exitRequested ) break;
IO::XMLArchive ar;
if ( !ar.open(files[i].c_str()) ) {
cerr << "Could not open file (ignored): " << files[i] << endl;
continue;
}
DataModel::InventoryPtr inv;
cerr << "Parsing " << files[i] << " ... " << flush;
ar >> inv;
cerr << "done" << endl;
if ( !inv ) {
cerr << "No inventory found (ignored): " << files[i] << endl;
continue;
}
_inventorySources[inv.get()] = files[i];
// Pushing the inventory into the merger cleans it
// completely. The ownership of all childs went to
// the merger
merger.push(inv.get());
}
_currentTask = NULL;
if ( _exitRequested ) {
cerr << "Exit requested: abort" << endl;
return false;
}
cerr << "Merging inventory ... " << flush;
merger.merge(false);
cerr << "done" << endl;
bool compact = commandline().hasOption("compact");
int level = 2;
if ( _level == "net" )
level = 0;
else if ( _level == "sta" )
level = 1;
else if ( _level == "cha" )
level = 2;
else if ( _level == "resp" )
level = 3;
std::vector<DataModel::Network*> nets;
for ( size_t n = 0; n < finalInventory->networkCount(); ++n )
nets.push_back(finalInventory->network(n));
sort(nets.begin(), nets.end(), lessID<DataModel::Network>);
for ( size_t n = 0; n < nets.size(); ++n ) {
DataModel::Network *net = nets[n];
if ( compact )
cout << net->code() << "\t" << epochToStr(net) << endl;
else {
cout << " network " << net->code();
if ( !net->description().empty() ) {
cout << setfill(' ') << setw(8-net->code().size()) << ' ';
cout << " " << net->description();
}
cout << endl;
cout << " epoch " << epochToStr(net) << endl;
}
std::vector<DataModel::Station*> stas;
if ( level > 0 ) {
for ( size_t s = 0; s < net->stationCount(); ++s )
stas.push_back(net->station(s));
}
sort(stas.begin(), stas.end(), lessID<DataModel::Station>);
for ( size_t s = 0; s < stas.size(); ++s ) {
DataModel::Station *sta = stas[s];
if ( compact )
cout << " " << sta->code() << "\t" << epochToStr(sta) << endl;
else {
cout << " station " << sta->code();
if ( !sta->description().empty() ) {
cout << setfill(' ') << setw(6-sta->code().size()) << ' ';
cout << " " << sta->description();
}
cout << endl;
cout << " epoch " << epochToStr(sta) << endl;
}
std::vector<DataModel::SensorLocation*> locs;
if ( level > 1 ) {
for ( size_t l = 0; l < sta->sensorLocationCount(); ++l )
locs.push_back(sta->sensorLocation(l));
}
sort(locs.begin(), locs.end(), lessID<DataModel::SensorLocation>);
for ( size_t l = 0; l < locs.size(); ++l ) {
DataModel::SensorLocation *loc = locs[l];
if ( compact ) {
cout << " ";
if ( loc->code().empty() )
cout << "__";
else
cout << loc->code();
cout << "\t" << epochToStr(loc) << endl;
}
else {
cout << " location ";
if ( loc->code().empty() )
cout << "__";
else
cout << loc->code();
cout << endl;
cout << " epoch " << epochToStr(loc) << endl;
}
std::vector<DataModel::Stream*> streams;
for ( size_t s = 0; s < loc->streamCount(); ++s )
streams.push_back(loc->stream(s));
sort(streams.begin(), streams.end(), lessID<DataModel::Stream>);
for ( size_t s = 0; s < streams.size(); ++s ) {
DataModel::Stream *str = streams[s];
if ( compact )
cout << " " << str->code() << "\t" << epochToStr(str) << endl;
else {
cout << " channel ";
cout << str->code() << endl;
cout << " epoch " << epochToStr(str) << endl;
}
if ( level >= 3 ) {
const DataModel::Sensor *sens;
const DataModel::Datalogger *dl;
const DataModel::ResponsePAZ *paz;
const DataModel::ResponsePolynomial *poly;
const DataModel::ResponseFAP *fap;
const DataModel::ResponseIIR *iir;
try {
int sr_num = str->sampleRateNumerator();
int sr_den = str->sampleRateDenominator();
cout << " rate " << sr_num << "/" << sr_den << " sps" << endl;
}
catch ( ... ) {}
try {
double gain = str->gain();
cout << " gain " << gain << endl;
}
catch ( ... ) {}
try {
double freq = str->gainFrequency();
cout << " freq " << freq << "Hz" << endl;
}
catch ( ... ) {}
if ( !str->gainUnit().empty() )
cout << " unit " << str->gainUnit() << endl;
sens = merger.findSensor(str->sensor());
if ( sens ) {
cout << " sens " << sens->description() << endl;
paz = merger.findPAZ(sens->response());
if ( paz ) {
cout << " resp PAZ" << endl;
cout << tabular(paz, 16) << endl;
}
else {
poly = merger.findPoly(sens->response());
if ( poly ) {
cout << " resp polynomial" << endl;
cout << tabular(poly, 16) << endl;
}
else {
fap = merger.findFAP(sens->response());
if ( fap ) {
cout << " resp fap" << endl;
cout << tabular(fap, 16) << endl;
}
else {
iir = merger.findIIR(sens->response());
if ( iir ) {
cout << " resp iir" << endl;
cout << tabular(iir, 16) << endl;
}
}
}
}
}
dl = merger.findDatalogger(str->datalogger());
if ( dl ) {
if ( !dl->description().empty() )
cout << " dl " << dl->description() << endl;
else
cout << " dl -" << endl;
try {
double gain = dl->gain();
cout << " gain " << gain << endl;
}
catch ( ... ) {}
try {
DataModel::Decimation *deci = dl->decimation(DataModel::DecimationIndex(str->sampleRateNumerator(), str->sampleRateDenominator()));
if ( deci ) {
cout << " dec " << str->sampleRateNumerator() << "/" << str->sampleRateDenominator() << " sps" << endl;
cout << tabular(&merger, deci, 16) << endl;
}
}
catch ( ... ) {}
}
}
}
}
}
}
return true;
}
bool syncKeyFiles(DataModel::Inventory *inv) {
if ( !Util::pathExists(_keydir) ) {
if ( !Util::createPath(_keydir) ) {
cerr << "ERROR: Unable to create key output path "
<< _keydir << endl;
return false;
}
}
bool allowPurge = !commandline().hasOption("no-purge-keys");
if ( !commandline().hasOption("no-purge-keys") && !commandline().hasOption("purge-keys") ) {
try { allowPurge = configGetBool("purgeKeys"); } catch (...) {}
}
// Collect all station key files
map<string, string> oldFiles;
try {
fs::path dir = SC_FS_PATH(_keydir);
fs::directory_iterator it(dir);
fs::directory_iterator dirEnd;
for ( ; it != dirEnd; ++it ) {
if ( fs::is_directory(*it) ) continue;
string name = SC_FS_IT_LEAF(it);
if ( name.compare(0, 8, "station_") == 0 )
oldFiles[SC_FS_IT_LEAF(it)] = SC_FS_IT_STR(it);
}
}
catch ( ... ) {}
int added = 0, removed = 0;
// Create station key files
for ( size_t n = 0; n < inv->networkCount(); ++n ) {
DataModel::Network *net = inv->network(n);
for ( size_t s = 0; s < net->stationCount(); ++s ) {
DataModel::Station *sta = net->station(s);
string id = net->code() + "_" + sta->code();
string filename = "station_" + id;
// Remove filename from oldFiles and prevent deletion
map<string, string>::iterator it = oldFiles.find(filename);
if ( it != oldFiles.end() ) oldFiles.erase(it);
fs::path fp = SC_FS_PATH(_keydir) / SC_FS_PATH(filename);
// Ignore existing files
if ( fs::exists(fp) ) continue;
ofstream ofs(fp.string().c_str());
++added;
}
}
if ( allowPurge ) {
// Delete remaining files
map<string, string>::iterator it;
for ( it = oldFiles.begin(); it != oldFiles.end(); ++it ) {
unlink(it->second.c_str());
++removed;
}
}
if ( added > 0 )
cerr << "Added " << added << " new key file(s)";
else
cerr << "No new key file added";
cerr << " and ";
if ( removed > 0 )
cerr << "removed " << removed << " old key file(s)";
else
cerr << "no old key file removed";
cerr << endl;
return true;
}
bool syncRCFiles(DataModel::Inventory *inv) {
if ( !Util::pathExists(_rcdir) ) {
if ( !Util::createPath(_rcdir) ) {
cerr << "ERROR: Unable to create rc output path "
<< _rcdir << endl;
return false;
}
}
typedef pair<Core::Time, string> Item;
typedef map<string, Item> DescMap;
DescMap descs;
for ( size_t n = 0; n < inv->networkCount(); ++n ) {
DataModel::Network *net = inv->network(n);
for ( size_t s = 0; s < net->stationCount(); ++s ) {
DataModel::Station *sta = net->station(s);
string id = net->code() + "_" + sta->code();
Core::Time endTime;
try { endTime = sta->end(); }
catch ( ... ) {}
DescMap::iterator it = descs.find(id);
if ( it == descs.end() )
descs[id] = Item(endTime, sta->description());
else {
if ( (it->second.first.valid() && it->second.first < endTime) ||
!endTime.valid() )
it->second.second = sta->description();
}
}
}
// Collect all station files
vector<string> oldFiles;
try {
fs::path directory = SC_FS_PATH(_rcdir);
fs::directory_iterator it(directory);
fs::directory_iterator dirEnd;
for ( ; it != dirEnd; ++it ) {
if ( fs::is_directory(*it) ) continue;
string name = SC_FS_IT_LEAF(it);
if ( name.compare(0, 8, "station_") == 0 )
oldFiles.push_back(SC_FS_IT_STR(it));
}
}
catch ( ... ) {}
// Delete them
for ( size_t i = 0; i < oldFiles.size(); ++i )
unlink(oldFiles[i].c_str());
for ( DescMap::iterator it = descs.begin(); it != descs.end(); ++it ) {
fs::path fp = SC_FS_PATH(_rcdir) / SC_FS_PATH((string("station_") + it->first));
Config::Config cfg;
cfg.setString("description", it->second.second);
if (! cfg.writeConfig(fp.string()) ) {
cerr << "ERROR: " << fp.string() << ": unable to write file" << endl;
return false;
}
}
return true;
}