Esempio n. 1
0
		bool syncRCFiles(DataModel::Inventory *inv) {
			if ( !Util::pathExists(_rcdir) ) {
				if ( !Util::createPath(_rcdir) ) {
					cerr << "ERROR: Unable to create rc output path "
					     << _rcdir << endl;
					return false;
				}
			}

			typedef pair<Core::Time, string> Item;
			typedef map<string, Item> DescMap;
			DescMap descs;

			for ( size_t n = 0; n < inv->networkCount(); ++n ) {
				DataModel::Network *net = inv->network(n);
				for ( size_t s = 0; s < net->stationCount(); ++s ) {
					DataModel::Station *sta = net->station(s);
					string id = net->code() + "_" + sta->code();
					Core::Time endTime;

					try { endTime = sta->end(); }
					catch ( ... ) {}

					DescMap::iterator it = descs.find(id);
					if ( it == descs.end() )
						descs[id] = Item(endTime, sta->description());
					else {
						if ( (it->second.first.valid() && it->second.first < endTime) ||
						     !endTime.valid() )
							it->second.second = sta->description();
					}
				}
			}

			// Collect all station files
			vector<string> oldFiles;
			try {
				fs::path directory = SC_FS_PATH(_rcdir);
				fs::directory_iterator it(directory);
				fs::directory_iterator dirEnd;

				for ( ; it != dirEnd; ++it ) {
					if ( fs::is_directory(*it) ) continue;
					string name = SC_FS_IT_LEAF(it);
					if ( name.compare(0, 8, "station_") == 0 )
						oldFiles.push_back(SC_FS_IT_STR(it));
				}
			}
			catch ( ... ) {}

			// Delete them
			for ( size_t i = 0; i < oldFiles.size(); ++i )
				unlink(oldFiles[i].c_str());

			for ( DescMap::iterator it = descs.begin(); it != descs.end(); ++it ) {
				fs::path fp = SC_FS_PATH(_rcdir) / SC_FS_PATH((string("station_") + it->first));
				Config::Config cfg;
				cfg.setString("description", it->second.second);
				if (! cfg.writeConfig(fp.string()) ) {
					cerr << "ERROR: " << fp.string() << ": unable to write file" << endl;
					return false;
				}
			}

			return true;
		}
Esempio n. 2
0
		bool listNetworks() {
			// Disable object registration
			DataModel::PublicObject::SetRegistrationEnabled(false);

			vector<string> files;
			collectFiles(files);

			if ( files.empty() ) {
				cerr << "Nothing to merge, no files given" << endl;
				return false;
			}

			DataModel::InventoryPtr finalInventory = new DataModel::Inventory();
			Merge merger(finalInventory.get());

			_currentTask = &merger;

			for ( size_t i = 0; i < files.size(); ++i ) {
				if ( _exitRequested ) break;

				IO::XMLArchive ar;
				if ( !ar.open(files[i].c_str()) ) {
					cerr << "Could not open file (ignored): " << files[i] << endl;
					continue;
				}

				DataModel::InventoryPtr inv;
				cerr << "Parsing " << files[i] << " ... " << flush;
				ar >> inv;
				cerr << "done" << endl;
				if ( !inv ) {
					cerr << "No inventory found (ignored): " << files[i] << endl;
					continue;
				}

				// Pushing the inventory into the merger cleans it
				// completely. The ownership of all childs went to
				// the merger
				merger.push(inv.get());
			}

			_currentTask = NULL;

			if ( _exitRequested ) {
				cerr << "Exit requested: abort" << endl;
				return false;
			}

			cerr << "Merging inventory ... " << flush;
			merger.merge(false);
			cerr << "done" << endl;

			bool compact = commandline().hasOption("compact");
			int level = 2;
			if ( _level == "net" )
				level = 0;
			else if ( _level == "sta" )
				level = 1;
			else if ( _level == "cha" )
				level = 2;
			else if ( _level == "resp" )
				level = 3;

			std::vector<DataModel::Network*> nets;

			for ( size_t n = 0; n < finalInventory->networkCount(); ++n )
				nets.push_back(finalInventory->network(n));

			sort(nets.begin(), nets.end(), lessID<DataModel::Network>);

			for ( size_t n = 0; n < nets.size(); ++n ) {
				DataModel::Network *net = nets[n];
				if ( compact )
					cout << net->code() << "\t" << epochToStr(net) << endl;
				else {
					cout << "  network " << net->code();
					if ( !net->description().empty() ) {
						cout << setfill(' ') << setw(8-net->code().size()) << ' ';
						cout << " " << net->description();
					}
					cout << endl;
					cout << "    epoch " << epochToStr(net) << endl;
				}

				std::vector<DataModel::Station*> stas;

				if ( level > 0 ) {
					for ( size_t s = 0; s < net->stationCount(); ++s )
						stas.push_back(net->station(s));
				}

				sort(stas.begin(), stas.end(), lessID<DataModel::Station>);

				for ( size_t s = 0; s < stas.size(); ++s ) {
					DataModel::Station *sta = stas[s];
					if ( compact )
						cout << " " << sta->code() << "\t" << epochToStr(sta) << endl;
					else {
						cout << "    station " << sta->code();
						if ( !sta->description().empty() ) {
							cout << setfill(' ') << setw(6-sta->code().size()) << ' ';
							cout << " " << sta->description();
						}
						cout << endl;
						cout << "      epoch " << epochToStr(sta) << endl;
					}

					std::vector<DataModel::SensorLocation*> locs;

					if ( level > 1 ) {
						for ( size_t l = 0; l < sta->sensorLocationCount(); ++l )
							locs.push_back(sta->sensorLocation(l));
					}

					sort(locs.begin(), locs.end(), lessID<DataModel::SensorLocation>);

					for ( size_t l = 0; l < locs.size(); ++l ) {
						DataModel::SensorLocation *loc = locs[l];
						if ( compact ) {
							cout << "  ";
							if ( loc->code().empty() )
								cout << "__";
							else
								cout << loc->code();
							cout << "\t" << epochToStr(loc) << endl;
						}
						else {
							cout << "      location ";
							if ( loc->code().empty() )
								cout << "__";
							else
								cout << loc->code();
							cout << endl;

							cout << "        epoch " << epochToStr(loc) << endl;
						}

						std::vector<DataModel::Stream*> streams;

						for ( size_t s = 0; s < loc->streamCount(); ++s )
							streams.push_back(loc->stream(s));

						sort(streams.begin(), streams.end(), lessID<DataModel::Stream>);

						for ( size_t s = 0; s < streams.size(); ++s ) {
							DataModel::Stream *str = streams[s];
							if ( compact )
								cout << "   " << str->code() << "\t" << epochToStr(str) << endl;
							else {
								cout << "        channel ";
								cout << str->code() << endl;

								cout << "          epoch " << epochToStr(str) << endl;
							}

							if ( level >= 3 ) {
								const DataModel::Sensor *sens;
								const DataModel::Datalogger *dl;
								const DataModel::ResponsePAZ *paz;
								const DataModel::ResponsePolynomial *poly;
								const DataModel::ResponseFAP *fap;

								try {
									int sr_num = str->sampleRateNumerator();
									int sr_den = str->sampleRateDenominator();
									cout << "          rate  " << sr_num << "/" << sr_den << " sps" << endl;
								}
								catch ( ... ) {}

								try {
									double gain = str->gain();
									cout << "          gain  " << gain << endl;
								}
								catch ( ... ) {}
								try {
									double freq = str->gainFrequency();
									cout << "          freq  " << freq << "Hz" << endl;
								}
								catch ( ... ) {}

								if ( !str->gainUnit().empty() )
									cout << "          unit  " << str->gainUnit() << endl;

								sens = merger.findSensor(str->sensor());
								if ( sens ) {
									cout << "          sens  " << sens->description() << endl;
									paz = merger.findPAZ(sens->response());
									if ( paz ) {
										cout << "          resp  PAZ" << endl;
										cout << tabular(paz, 16) << endl;
									}
									else {
										poly = merger.findPoly(sens->response());
										if ( poly ) {
											cout << "          resp  polynomial" << endl;
											cout << tabular(poly, 16) << endl;
										}
										else {
											fap = merger.findFAP(sens->response());
											if ( fap ) {
												cout << "          resp  fap" << endl;
												cout << tabular(fap, 16) << endl;
											}
										}
									}
								}

								dl = merger.findDatalogger(str->datalogger());
								if ( dl ) {
									if ( !dl->description().empty() )
										cout << "          dl    " << dl->description() << endl;
									else
										cout << "          dl    -" << endl;

									try {
										double gain = dl->gain();
										cout << "                gain         " << gain << endl;
									}
									catch ( ... ) {}

									try {
										DataModel::Decimation *deci = dl->decimation(DataModel::DecimationIndex(str->sampleRateNumerator(), str->sampleRateDenominator()));
										if ( deci ) {
											cout << "          dec   " << str->sampleRateNumerator() << "/" << str->sampleRateDenominator() << " sps" << endl;
											cout << tabular(&merger, deci, 16) << endl;
										}
									}
									catch ( ... ) {}
								}
							}
						}
					}
				}
			}

			return true;
		}
Esempio n. 3
0
		bool syncInventory() {
			DataModel::Inventory *targetInv = Client::Inventory::Instance()->inventory();
			bool createNotifier = commandline().hasOption("create-notifier");
			bool testMode = commandline().hasOption("test");

			if ( testMode || _output.empty() ) createNotifier = true;

			if ( targetInv == NULL ) {
				cerr << "No inventory to sync" << endl;
				return false;
			}

			// Disable object registration
			DataModel::PublicObject::SetRegistrationEnabled(false);
			// Disable notifier check
			DataModel::Notifier::SetCheckEnabled(false);

			vector<string> files;
			collectFiles(files);

			if ( _filebase.empty() && files.empty() ) {
				cerr << "Nothing to merge, no files given" << endl;
				return false;
			}

			DataModel::InventoryPtr mergedInventory = new DataModel::Inventory();
			Merge merger(mergedInventory.get());
			merger.setLogHandler(this);
			_continueOperation = true;
			_currentTask = &merger;

			DataModel::Notifier::SetEnabled(false);

			for ( size_t i = 0; i < files.size(); ++i ) {
				if ( _exitRequested ) break;

				IO::XMLArchive ar;
				if ( !ar.open(files[i].c_str()) ) {
					cerr << "Could not open file (ignored): " << files[i] << endl;
					continue;
				}

				DataModel::InventoryPtr inv;
				cerr << "Parsing " << files[i] << " ... " << flush;
				ar >> inv;
				cerr << "done" << endl;
				if ( !inv ) {
					cerr << "No inventory found (ignored): " << files[i] << endl;
					continue;
				}

				// Pushing the inventory into the merger cleans it
				// completely. The ownership of all childs goes to
				// the merger
				merger.push(inv.get());
			}

			if ( _exitRequested ) {
				cerr << "Exit requested: abort" << endl;
				return false;
			}

			cerr << "Merging inventory ... " << flush;
			if ( merger.merge(false) )
				cerr << "done" << endl;
			else
				cerr << "failed" << endl;

			printLogs();

			if ( !_continueOperation ) {
				cerr << "Unresolvable errors ... aborting" << endl;
				return false;
			}

			if ( _exitRequested ) {
				cerr << "Exit requested: abort" << endl;
				return false;
			}

			// Activate registration again
			DataModel::PublicObject::SetRegistrationEnabled(true);

			Sync syncTask(targetInv);
			_currentTask = &syncTask;

			if ( createNotifier ) DataModel::Notifier::SetEnabled(true);

			cerr << "Synchronising inventory ... " << flush;
			if ( syncTask.push(mergedInventory.get()) )
				cerr << "done";
			else
				cerr << "failed";
			cerr << endl;

			if ( _exitRequested ) {
				cerr << "Exit requested: abort" << endl;
				return false;
			}

			if ( createNotifier ) DataModel::Notifier::SetEnabled(true);

			cerr << "Removing remaining objects ... " << flush;
			syncTask.cleanUp();
			cerr << "done" << endl;

			// --- Check key files
			// Collect all station key files
			map<string,string> keyFiles;
			try {
				fs::path directory = SC_FS_PATH(_keydir);
				fs::directory_iterator it(directory);
				fs::directory_iterator dirEnd;

				for ( ; it != dirEnd; ++it ) {
					if ( fs::is_directory(SC_FS_IT_PATH(it)) ) continue;
					string name = SC_FS_IT_LEAF(it);
					if ( name.compare(0, 8, "station_") == 0 )
						keyFiles[SC_FS_IT_LEAF(it)] = SC_FS_IT_STR(it);
				}
			}
			catch ( ... ) {}

			for ( size_t n = 0; n < mergedInventory->networkCount(); ++n ) {
				DataModel::Network *net = mergedInventory->network(n);
				for ( size_t s = 0; s < net->stationCount(); ++s ) {
					DataModel::Station *sta = net->station(s);
					string id = net->code() + "_" + sta->code();
					string filename = "station_" + id;

					// Remove filename from keyFiles
					map<string,string>::iterator it = keyFiles.find(filename);
					if ( it != keyFiles.end() ) keyFiles.erase(it);
				}
			}

			// Warn about existing key files without a corresponding station
			// in inventory
			if ( !keyFiles.empty() ) {
				SEISCOMP_WARNING("Found %d key file%s without a corresponding "
				                 "station in inventory:",
				                 (int)keyFiles.size(), keyFiles.size() > 1?"s":"");
				for ( map<string,string>::iterator it = keyFiles.begin();
					  it != keyFiles.end(); ++it )
					SEISCOMP_WARNING("  %s", it->second.c_str());
			}

			DataModel::Notifier::SetEnabled(false);

			_currentTask = NULL;

			if ( _exitRequested ) {
				cerr << "Exit requested: abort" << endl;
				return false;
			}

			bool doSyncKeys = false;

			if ( createNotifier ) {
				DataModel::NotifierMessagePtr nmsg;
				size_t notifierCount = DataModel::Notifier::Size();

				if ( notifierCount > 0 ) {
					cerr << notifierCount << " notifiers available" << endl;

					if ( !_output.empty() ) {
						IO::XMLArchive ar;
						if ( !ar.create(_output.c_str()) ) {
							cerr << "Failed to create output file: " << _output << endl;
							DataModel::Notifier::Clear();
							return false;
						}

						cerr << "Generating output ... "  << flush;
						ar.setFormattedOutput(true);

						nmsg = DataModel::Notifier::GetMessage(true);
						ar << nmsg;

						ar.close();
						cerr << "done" << endl;
					}
					else if ( !testMode ) {
						// Send an inital sync command to also wake-up the messaging
						sync();

						// Send notifier
						DataModel::NotifierMessagePtr tmp = new DataModel::NotifierMessage();
						DataModel::NotifierMessage::iterator it;
						int count = 0;

						// Fetch each single notifier message. Fetching all notifiers
						// in one message can take a long time if a huge amount of
						// notifiers is in the queue. Due to memory fragmentation
						// most of the time spent is in malloc.
						while ( (nmsg = DataModel::Notifier::GetMessage(false)) != NULL ) {
							if ( _exitRequested ) break;
							for ( it = nmsg->begin(); it != nmsg->end(); ++it ) {
								DataModel::Notifier* n = DataModel::Notifier::Cast(*it);
								if ( !n ) continue;

								tmp->attach(n);
								++count;

								if ( count % 100 == 0 ) {
									cerr << "\rSending notifiers: " << (int)(count*100/notifierCount) << "%" << flush;

									if ( !send(tmp.get()) ) {
										SEISCOMP_ERROR("Failed to send message, abort");
										return false;
									}

									tmp->clear();
									sync();
								}
							}
						}

						if ( !_exitRequested && !tmp->empty() ) {
							if ( !send(tmp.get()) ) {
								SEISCOMP_ERROR("Failed to send message, abort");
								return false;
							}

							cerr << "\rSending notifiers: " << (int)(count*100/notifierCount) << "%" << flush;
						}

						cerr << endl;
						sync();

						doSyncKeys = true;
					}
					else {
						/*
						DataModel::NotifierMessage::iterator it;

						while ( (nmsg = DataModel::Notifier::GetMessage(false)) != NULL ) {
							for ( it = nmsg->begin(); it != nmsg->end(); ++it ) {
								DataModel::Notifier* n = DataModel::Notifier::Cast(*it);
								if ( !n ) continue;

								switch ( n->operation() ) {
									case DataModel::OP_ADD:
										cout << "+";
										break;
									case DataModel::OP_REMOVE:
										cout << "-";
										break;
									case DataModel::OP_UPDATE:
										cout << "M";
										break;
									default:
										cout << "?";
										break;
								}

								cout << " " << n->object()->className() << endl;
							}
						}
						*/

						DataModel::Notifier::Clear();
						cout << "OK - synchronization test passed" << endl;
					}
				}
				else
					cerr << "Inventory is synchronised already, nothing to do" << endl;

				DataModel::Notifier::Clear();
			}
			else {
				if ( _output.empty() ) _output = "-";

				IO::XMLArchive ar;
				if ( !ar.create(_output.c_str()) ) {
					cerr << "Failed to create output file: " << _output << endl;
					return false;
				}

				cerr << "Generating output ... "  << flush;
				ar.setFormattedOutput(true);

				ar << targetInv;

				ar.close();
				cerr << "done" << endl;
			}

			if ( doSyncKeys ) {
				if ( !commandline().hasOption("no-rc") ) {
					if ( !syncRCFiles(targetInv) ) return false;
				}

				if ( !commandline().hasOption("no-keys") ) {
					bool syncKeys = true;
					try { syncKeys = configGetBool("syncKeys"); } catch (...) {}
					if ( syncKeys && !syncKeyFiles(targetInv) ) return false;
				}
			}

			return true;
		}
Esempio n. 4
0
		bool syncKeyFiles(DataModel::Inventory *inv) {
			if ( !Util::pathExists(_keydir) ) {
				if ( !Util::createPath(_keydir) ) {
					cerr << "ERROR: Unable to create key output path "
					     << _keydir << endl;
					return false;
				}
			}

			bool allowPurge = !commandline().hasOption("no-purge-keys");
			if ( !commandline().hasOption("no-purge-keys") && !commandline().hasOption("purge-keys") ) {
				try { allowPurge = configGetBool("purgeKeys"); } catch (...) {}
			}

			// Collect all station key files
			map<string, string> oldFiles;
			try {
				fs::path dir = SC_FS_PATH(_keydir);
				fs::directory_iterator it(dir);
				fs::directory_iterator dirEnd;

				for ( ; it != dirEnd; ++it ) {
					if ( fs::is_directory(*it) ) continue;
					string name = SC_FS_IT_LEAF(it);
					if ( name.compare(0, 8, "station_") == 0 )
						oldFiles[SC_FS_IT_LEAF(it)] = SC_FS_IT_STR(it);
				}
			}
			catch ( ... ) {}

			int added = 0, removed = 0;

			// Create station key files
			for ( size_t n = 0; n < inv->networkCount(); ++n ) {
				DataModel::Network *net = inv->network(n);
				for ( size_t s = 0; s < net->stationCount(); ++s ) {
					DataModel::Station *sta = net->station(s);
					string id = net->code() + "_" + sta->code();
					string filename = "station_" + id;

					// Remove filename from oldFiles and prevent deletion
					map<string, string>::iterator it = oldFiles.find(filename);
					if ( it != oldFiles.end() ) oldFiles.erase(it);

					fs::path fp = SC_FS_PATH(_keydir) / SC_FS_PATH(filename);
					// Ignore existing files
					if ( fs::exists(fp) ) continue;

					ofstream ofs(fp.string().c_str());
					++added;
				}
			}

			if ( allowPurge ) {
				// Delete remaining files
				map<string, string>::iterator it;
				for ( it = oldFiles.begin(); it != oldFiles.end(); ++it ) {
					unlink(it->second.c_str());
					++removed;
				}
			}

			if ( added > 0 )
				cerr << "Added " << added << " new key file(s)";
			else
				cerr << "No new key file added";
			cerr << " and ";
			if ( removed > 0 )
				cerr << "removed " << removed << " old key file(s)";
			else
				cerr << "no old key file removed";
			cerr << endl;

			return true;
		}