bool syncRCFiles(DataModel::Inventory *inv) { if ( !Util::pathExists(_rcdir) ) { if ( !Util::createPath(_rcdir) ) { cerr << "ERROR: Unable to create rc output path " << _rcdir << endl; return false; } } typedef pair<Core::Time, string> Item; typedef map<string, Item> DescMap; DescMap descs; for ( size_t n = 0; n < inv->networkCount(); ++n ) { DataModel::Network *net = inv->network(n); for ( size_t s = 0; s < net->stationCount(); ++s ) { DataModel::Station *sta = net->station(s); string id = net->code() + "_" + sta->code(); Core::Time endTime; try { endTime = sta->end(); } catch ( ... ) {} DescMap::iterator it = descs.find(id); if ( it == descs.end() ) descs[id] = Item(endTime, sta->description()); else { if ( (it->second.first.valid() && it->second.first < endTime) || !endTime.valid() ) it->second.second = sta->description(); } } } // Collect all station files vector<string> oldFiles; try { fs::path directory = SC_FS_PATH(_rcdir); fs::directory_iterator it(directory); fs::directory_iterator dirEnd; for ( ; it != dirEnd; ++it ) { if ( fs::is_directory(*it) ) continue; string name = SC_FS_IT_LEAF(it); if ( name.compare(0, 8, "station_") == 0 ) oldFiles.push_back(SC_FS_IT_STR(it)); } } catch ( ... ) {} // Delete them for ( size_t i = 0; i < oldFiles.size(); ++i ) unlink(oldFiles[i].c_str()); for ( DescMap::iterator it = descs.begin(); it != descs.end(); ++it ) { fs::path fp = SC_FS_PATH(_rcdir) / SC_FS_PATH((string("station_") + it->first)); Config::Config cfg; cfg.setString("description", it->second.second); if (! cfg.writeConfig(fp.string()) ) { cerr << "ERROR: " << fp.string() << ": unable to write file" << endl; return false; } } return true; }
bool listNetworks() { // Disable object registration DataModel::PublicObject::SetRegistrationEnabled(false); vector<string> files; collectFiles(files); if ( files.empty() ) { cerr << "Nothing to merge, no files given" << endl; return false; } DataModel::InventoryPtr finalInventory = new DataModel::Inventory(); Merge merger(finalInventory.get()); _currentTask = &merger; for ( size_t i = 0; i < files.size(); ++i ) { if ( _exitRequested ) break; IO::XMLArchive ar; if ( !ar.open(files[i].c_str()) ) { cerr << "Could not open file (ignored): " << files[i] << endl; continue; } DataModel::InventoryPtr inv; cerr << "Parsing " << files[i] << " ... " << flush; ar >> inv; cerr << "done" << endl; if ( !inv ) { cerr << "No inventory found (ignored): " << files[i] << endl; continue; } // Pushing the inventory into the merger cleans it // completely. The ownership of all childs went to // the merger merger.push(inv.get()); } _currentTask = NULL; if ( _exitRequested ) { cerr << "Exit requested: abort" << endl; return false; } cerr << "Merging inventory ... " << flush; merger.merge(false); cerr << "done" << endl; bool compact = commandline().hasOption("compact"); int level = 2; if ( _level == "net" ) level = 0; else if ( _level == "sta" ) level = 1; else if ( _level == "cha" ) level = 2; else if ( _level == "resp" ) level = 3; std::vector<DataModel::Network*> nets; for ( size_t n = 0; n < finalInventory->networkCount(); ++n ) nets.push_back(finalInventory->network(n)); sort(nets.begin(), nets.end(), lessID<DataModel::Network>); for ( size_t n = 0; n < nets.size(); ++n ) { DataModel::Network *net = nets[n]; if ( compact ) cout << net->code() << "\t" << epochToStr(net) << endl; else { cout << " network " << net->code(); if ( !net->description().empty() ) { cout << setfill(' ') << setw(8-net->code().size()) << ' '; cout << " " << net->description(); } cout << endl; cout << " epoch " << epochToStr(net) << endl; } std::vector<DataModel::Station*> stas; if ( level > 0 ) { for ( size_t s = 0; s < net->stationCount(); ++s ) stas.push_back(net->station(s)); } sort(stas.begin(), stas.end(), lessID<DataModel::Station>); for ( size_t s = 0; s < stas.size(); ++s ) { DataModel::Station *sta = stas[s]; if ( compact ) cout << " " << sta->code() << "\t" << epochToStr(sta) << endl; else { cout << " station " << sta->code(); if ( !sta->description().empty() ) { cout << setfill(' ') << setw(6-sta->code().size()) << ' '; cout << " " << sta->description(); } cout << endl; cout << " epoch " << epochToStr(sta) << endl; } std::vector<DataModel::SensorLocation*> locs; if ( level > 1 ) { for ( size_t l = 0; l < sta->sensorLocationCount(); ++l ) locs.push_back(sta->sensorLocation(l)); } sort(locs.begin(), locs.end(), lessID<DataModel::SensorLocation>); for ( size_t l = 0; l < locs.size(); ++l ) { DataModel::SensorLocation *loc = locs[l]; if ( compact ) { cout << " "; if ( loc->code().empty() ) cout << "__"; else cout << loc->code(); cout << "\t" << epochToStr(loc) << endl; } else { cout << " location "; if ( loc->code().empty() ) cout << "__"; else cout << loc->code(); cout << endl; cout << " epoch " << epochToStr(loc) << endl; } std::vector<DataModel::Stream*> streams; for ( size_t s = 0; s < loc->streamCount(); ++s ) streams.push_back(loc->stream(s)); sort(streams.begin(), streams.end(), lessID<DataModel::Stream>); for ( size_t s = 0; s < streams.size(); ++s ) { DataModel::Stream *str = streams[s]; if ( compact ) cout << " " << str->code() << "\t" << epochToStr(str) << endl; else { cout << " channel "; cout << str->code() << endl; cout << " epoch " << epochToStr(str) << endl; } if ( level >= 3 ) { const DataModel::Sensor *sens; const DataModel::Datalogger *dl; const DataModel::ResponsePAZ *paz; const DataModel::ResponsePolynomial *poly; const DataModel::ResponseFAP *fap; try { int sr_num = str->sampleRateNumerator(); int sr_den = str->sampleRateDenominator(); cout << " rate " << sr_num << "/" << sr_den << " sps" << endl; } catch ( ... ) {} try { double gain = str->gain(); cout << " gain " << gain << endl; } catch ( ... ) {} try { double freq = str->gainFrequency(); cout << " freq " << freq << "Hz" << endl; } catch ( ... ) {} if ( !str->gainUnit().empty() ) cout << " unit " << str->gainUnit() << endl; sens = merger.findSensor(str->sensor()); if ( sens ) { cout << " sens " << sens->description() << endl; paz = merger.findPAZ(sens->response()); if ( paz ) { cout << " resp PAZ" << endl; cout << tabular(paz, 16) << endl; } else { poly = merger.findPoly(sens->response()); if ( poly ) { cout << " resp polynomial" << endl; cout << tabular(poly, 16) << endl; } else { fap = merger.findFAP(sens->response()); if ( fap ) { cout << " resp fap" << endl; cout << tabular(fap, 16) << endl; } } } } dl = merger.findDatalogger(str->datalogger()); if ( dl ) { if ( !dl->description().empty() ) cout << " dl " << dl->description() << endl; else cout << " dl -" << endl; try { double gain = dl->gain(); cout << " gain " << gain << endl; } catch ( ... ) {} try { DataModel::Decimation *deci = dl->decimation(DataModel::DecimationIndex(str->sampleRateNumerator(), str->sampleRateDenominator())); if ( deci ) { cout << " dec " << str->sampleRateNumerator() << "/" << str->sampleRateDenominator() << " sps" << endl; cout << tabular(&merger, deci, 16) << endl; } } catch ( ... ) {} } } } } } } return true; }
bool syncInventory() { DataModel::Inventory *targetInv = Client::Inventory::Instance()->inventory(); bool createNotifier = commandline().hasOption("create-notifier"); bool testMode = commandline().hasOption("test"); if ( testMode || _output.empty() ) createNotifier = true; if ( targetInv == NULL ) { cerr << "No inventory to sync" << endl; return false; } // Disable object registration DataModel::PublicObject::SetRegistrationEnabled(false); // Disable notifier check DataModel::Notifier::SetCheckEnabled(false); vector<string> files; collectFiles(files); if ( _filebase.empty() && files.empty() ) { cerr << "Nothing to merge, no files given" << endl; return false; } DataModel::InventoryPtr mergedInventory = new DataModel::Inventory(); Merge merger(mergedInventory.get()); merger.setLogHandler(this); _continueOperation = true; _currentTask = &merger; DataModel::Notifier::SetEnabled(false); for ( size_t i = 0; i < files.size(); ++i ) { if ( _exitRequested ) break; IO::XMLArchive ar; if ( !ar.open(files[i].c_str()) ) { cerr << "Could not open file (ignored): " << files[i] << endl; continue; } DataModel::InventoryPtr inv; cerr << "Parsing " << files[i] << " ... " << flush; ar >> inv; cerr << "done" << endl; if ( !inv ) { cerr << "No inventory found (ignored): " << files[i] << endl; continue; } // Pushing the inventory into the merger cleans it // completely. The ownership of all childs goes to // the merger merger.push(inv.get()); } if ( _exitRequested ) { cerr << "Exit requested: abort" << endl; return false; } cerr << "Merging inventory ... " << flush; if ( merger.merge(false) ) cerr << "done" << endl; else cerr << "failed" << endl; printLogs(); if ( !_continueOperation ) { cerr << "Unresolvable errors ... aborting" << endl; return false; } if ( _exitRequested ) { cerr << "Exit requested: abort" << endl; return false; } // Activate registration again DataModel::PublicObject::SetRegistrationEnabled(true); Sync syncTask(targetInv); _currentTask = &syncTask; if ( createNotifier ) DataModel::Notifier::SetEnabled(true); cerr << "Synchronising inventory ... " << flush; if ( syncTask.push(mergedInventory.get()) ) cerr << "done"; else cerr << "failed"; cerr << endl; if ( _exitRequested ) { cerr << "Exit requested: abort" << endl; return false; } if ( createNotifier ) DataModel::Notifier::SetEnabled(true); cerr << "Removing remaining objects ... " << flush; syncTask.cleanUp(); cerr << "done" << endl; // --- Check key files // Collect all station key files map<string,string> keyFiles; try { fs::path directory = SC_FS_PATH(_keydir); fs::directory_iterator it(directory); fs::directory_iterator dirEnd; for ( ; it != dirEnd; ++it ) { if ( fs::is_directory(SC_FS_IT_PATH(it)) ) continue; string name = SC_FS_IT_LEAF(it); if ( name.compare(0, 8, "station_") == 0 ) keyFiles[SC_FS_IT_LEAF(it)] = SC_FS_IT_STR(it); } } catch ( ... ) {} for ( size_t n = 0; n < mergedInventory->networkCount(); ++n ) { DataModel::Network *net = mergedInventory->network(n); for ( size_t s = 0; s < net->stationCount(); ++s ) { DataModel::Station *sta = net->station(s); string id = net->code() + "_" + sta->code(); string filename = "station_" + id; // Remove filename from keyFiles map<string,string>::iterator it = keyFiles.find(filename); if ( it != keyFiles.end() ) keyFiles.erase(it); } } // Warn about existing key files without a corresponding station // in inventory if ( !keyFiles.empty() ) { SEISCOMP_WARNING("Found %d key file%s without a corresponding " "station in inventory:", (int)keyFiles.size(), keyFiles.size() > 1?"s":""); for ( map<string,string>::iterator it = keyFiles.begin(); it != keyFiles.end(); ++it ) SEISCOMP_WARNING(" %s", it->second.c_str()); } DataModel::Notifier::SetEnabled(false); _currentTask = NULL; if ( _exitRequested ) { cerr << "Exit requested: abort" << endl; return false; } bool doSyncKeys = false; if ( createNotifier ) { DataModel::NotifierMessagePtr nmsg; size_t notifierCount = DataModel::Notifier::Size(); if ( notifierCount > 0 ) { cerr << notifierCount << " notifiers available" << endl; if ( !_output.empty() ) { IO::XMLArchive ar; if ( !ar.create(_output.c_str()) ) { cerr << "Failed to create output file: " << _output << endl; DataModel::Notifier::Clear(); return false; } cerr << "Generating output ... " << flush; ar.setFormattedOutput(true); nmsg = DataModel::Notifier::GetMessage(true); ar << nmsg; ar.close(); cerr << "done" << endl; } else if ( !testMode ) { // Send an inital sync command to also wake-up the messaging sync(); // Send notifier DataModel::NotifierMessagePtr tmp = new DataModel::NotifierMessage(); DataModel::NotifierMessage::iterator it; int count = 0; // Fetch each single notifier message. Fetching all notifiers // in one message can take a long time if a huge amount of // notifiers is in the queue. Due to memory fragmentation // most of the time spent is in malloc. while ( (nmsg = DataModel::Notifier::GetMessage(false)) != NULL ) { if ( _exitRequested ) break; for ( it = nmsg->begin(); it != nmsg->end(); ++it ) { DataModel::Notifier* n = DataModel::Notifier::Cast(*it); if ( !n ) continue; tmp->attach(n); ++count; if ( count % 100 == 0 ) { cerr << "\rSending notifiers: " << (int)(count*100/notifierCount) << "%" << flush; if ( !send(tmp.get()) ) { SEISCOMP_ERROR("Failed to send message, abort"); return false; } tmp->clear(); sync(); } } } if ( !_exitRequested && !tmp->empty() ) { if ( !send(tmp.get()) ) { SEISCOMP_ERROR("Failed to send message, abort"); return false; } cerr << "\rSending notifiers: " << (int)(count*100/notifierCount) << "%" << flush; } cerr << endl; sync(); doSyncKeys = true; } else { /* DataModel::NotifierMessage::iterator it; while ( (nmsg = DataModel::Notifier::GetMessage(false)) != NULL ) { for ( it = nmsg->begin(); it != nmsg->end(); ++it ) { DataModel::Notifier* n = DataModel::Notifier::Cast(*it); if ( !n ) continue; switch ( n->operation() ) { case DataModel::OP_ADD: cout << "+"; break; case DataModel::OP_REMOVE: cout << "-"; break; case DataModel::OP_UPDATE: cout << "M"; break; default: cout << "?"; break; } cout << " " << n->object()->className() << endl; } } */ DataModel::Notifier::Clear(); cout << "OK - synchronization test passed" << endl; } } else cerr << "Inventory is synchronised already, nothing to do" << endl; DataModel::Notifier::Clear(); } else { if ( _output.empty() ) _output = "-"; IO::XMLArchive ar; if ( !ar.create(_output.c_str()) ) { cerr << "Failed to create output file: " << _output << endl; return false; } cerr << "Generating output ... " << flush; ar.setFormattedOutput(true); ar << targetInv; ar.close(); cerr << "done" << endl; } if ( doSyncKeys ) { if ( !commandline().hasOption("no-rc") ) { if ( !syncRCFiles(targetInv) ) return false; } if ( !commandline().hasOption("no-keys") ) { bool syncKeys = true; try { syncKeys = configGetBool("syncKeys"); } catch (...) {} if ( syncKeys && !syncKeyFiles(targetInv) ) return false; } } return true; }
bool syncKeyFiles(DataModel::Inventory *inv) { if ( !Util::pathExists(_keydir) ) { if ( !Util::createPath(_keydir) ) { cerr << "ERROR: Unable to create key output path " << _keydir << endl; return false; } } bool allowPurge = !commandline().hasOption("no-purge-keys"); if ( !commandline().hasOption("no-purge-keys") && !commandline().hasOption("purge-keys") ) { try { allowPurge = configGetBool("purgeKeys"); } catch (...) {} } // Collect all station key files map<string, string> oldFiles; try { fs::path dir = SC_FS_PATH(_keydir); fs::directory_iterator it(dir); fs::directory_iterator dirEnd; for ( ; it != dirEnd; ++it ) { if ( fs::is_directory(*it) ) continue; string name = SC_FS_IT_LEAF(it); if ( name.compare(0, 8, "station_") == 0 ) oldFiles[SC_FS_IT_LEAF(it)] = SC_FS_IT_STR(it); } } catch ( ... ) {} int added = 0, removed = 0; // Create station key files for ( size_t n = 0; n < inv->networkCount(); ++n ) { DataModel::Network *net = inv->network(n); for ( size_t s = 0; s < net->stationCount(); ++s ) { DataModel::Station *sta = net->station(s); string id = net->code() + "_" + sta->code(); string filename = "station_" + id; // Remove filename from oldFiles and prevent deletion map<string, string>::iterator it = oldFiles.find(filename); if ( it != oldFiles.end() ) oldFiles.erase(it); fs::path fp = SC_FS_PATH(_keydir) / SC_FS_PATH(filename); // Ignore existing files if ( fs::exists(fp) ) continue; ofstream ofs(fp.string().c_str()); ++added; } } if ( allowPurge ) { // Delete remaining files map<string, string>::iterator it; for ( it = oldFiles.begin(); it != oldFiles.end(); ++it ) { unlink(it->second.c_str()); ++removed; } } if ( added > 0 ) cerr << "Added " << added << " new key file(s)"; else cerr << "No new key file added"; cerr << " and "; if ( removed > 0 ) cerr << "removed " << removed << " old key file(s)"; else cerr << "no old key file removed"; cerr << endl; return true; }