bool RobotBrainServiceService::start(int argc, char* argv[])
{
char adapterStr[255];
LINFO("Creating Topic!");
//Create the topics
// SimEventsUtils::createTopic(communicator(), "RetinaMessageTopic");
//Create the adapter
int port = RobotBrainObjects::RobotBrainPort;
bool connected = false;
LDEBUG("Opening Connection");
while(!connected)
{
try
{
LINFO("Trying Port:%d", port);
sprintf(adapterStr, "default -p %i", port);
itsAdapter = communicator()->createObjectAdapterWithEndpoints("Retina",
adapterStr);
connected = true;
}
catch(Ice::SocketException)
{
port++;
}
}
//Create the manager and its objects
itsMgr = new ModelManager("RetinaService");
LINFO("Starting Retina");
nub::ref<RetinaI> ret(new RetinaI(0, *itsMgr, "Retina1", "Retina2"));
LINFO("Retina Created");
itsMgr->addSubComponent(ret);
LINFO("Retina Added As Sub Component");
ret->init(communicator(), itsAdapter);
LINFO("Retina Inited");
itsMgr->parseCommandLine((const int)argc, (const char**)argv, "", 0, 0);
itsAdapter->activate();
itsMgr->start();
return true;
}
communicator communicator::split(int color, int key) const
{
MPI_Comm newcomm;
BOOST_MPI_CHECK_RESULT(MPI_Comm_split,
(MPI_Comm(*this), color, key, &newcomm));
return communicator(newcomm, comm_take_ownership);
}
int HelloServer::run(int argc, char* argv[])
{
if (argc > 1)
{
std::cerr << appName() << ": too many arguments" << std::endl;
return EXIT_FAILURE;
}
Ice::ObjectAdapterPtr adapter = communicator()->createObjectAdapter("Hello");
Demo::HelloPtr hello = new HelloI;
adapter->add(hello, communicator()->stringToIdentity("hello"));
adapter->activate();
communicator()->waitForShutdown();
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
int const port = 8888;
QApplication app(argc, argv);
QString configPath = "./";
if (app.arguments().contains("-c")) {
int const index = app.arguments().indexOf("-c");
if (app.arguments().count() <= index + 1) {
qDebug() << "Usage: ./trikServer [-c <config file path>]";
return 1;
}
configPath = app.arguments()[index + 1];
if (configPath.right(1) != "/") {
configPath += "/";
}
}
qDebug() << "Running TrikServer on port" << port;
trikCommunicator::TrikCommunicator communicator(configPath);
communicator.listen(port);
return app.exec();
}
void MPIPortsCommunication:: requestConnection
(
const std::string& nameAcceptor,
const std::string& nameRequester,
int requesterProcessRank,
int requesterCommunicatorSize )
{
preciceTrace2 ( "requestConnection()", nameAcceptor, nameRequester );
assertion ( not _isConnection );
int argc = 1;
char* arg = new char[8];
strcpy(arg, "precice");
char** argv = &arg;
utils::Parallel::initialize ( &argc, &argv, nameRequester );
delete[] arg;
std::string portFilename(_publishingDirectory + "." + nameRequester + "-portname");
std::ifstream inFile;
do {
inFile.open ( portFilename.c_str(), std::ios::in );
usleep(100000);
} while ( not inFile );
inFile.getline ( _portname, MPI_MAX_PORT_NAME );
inFile.close();
preciceDebug ( "Read connection info \"" + std::string(_portname) +
"\" from file " + portFilename );
preciceDebug("Calling MPI_Comm_connect() with portname = " << _portname);
MPI_Comm localComm = utils::Parallel::getLocalCommunicator();
MPI_Comm_connect ( _portname, MPI_INFO_NULL, 0, localComm, &communicator() );
_isConnection = true;
}
int main(int argc, char *argv[])
{
const int port = 8888;
QApplication app(argc, argv);
app.setApplicationName("TrikServer");
trikKernel::ApplicationInitHelper initHelper(app);
if (!initHelper.parseCommandLine()) {
return 0;
}
initHelper.init();
QLOG_INFO() << "TrikServer started on port" << port;
qDebug() << "Running TrikServer on port" << port;
QScopedPointer<trikControl::BrickInterface> brick(trikControl::BrickFactory::create(
initHelper.configPath(), trikKernel::Paths::mediaPath()));
trikKernel::Configurer configurer(
initHelper.configPath() + "/system-config.xml"
, initHelper.configPath() + "/model-config.xml"
);
QScopedPointer<trikNetwork::MailboxInterface> mailbox(trikNetwork::MailboxFactory::create(configurer));
trikCommunicator::TrikCommunicator communicator(*brick, mailbox.data());
communicator.startServer(port);
return app.exec();
}
bool Application::configGetBool (const std::string name, bool &value, const int default_value)
{
string tmp;
tmp = communicator()->getProperties()->getProperty( name );
if ( tmp.length() == 0 )
{
if (default_value != -1)
{
std::cout << name << " property does not exist. Using default value." << std::endl;
value = default_value;
return false;
}
else
{
printf("Can't read configuration parameter %s. Got %s.", name.c_str(), tmp.c_str());
qFatal("Error\n");
}
}
if(tmp == "true" or tmp == "True" or tmp == "yes" or tmp == "Yes" or tmp == "1")
{
value = true;
}
else if (tmp == "false" or tmp == "False" or tmp == "no" or tmp == "No" or tmp == "0")
{
value = false;
}
else
{
qFatal("Reading config file: '%s' is not a valid boolean", name.c_str());
}
std::cout << name << " " << value << std::endl;
return true;
}
void QUHttpCollector::processNetworkOperationDone(bool error) {
int count = localFiles().size();
closeLocalFiles();
if(error) {
setState(Idle);
communicator()->send(http()->errorString());
communicator()->send(QUCommunicatorInterface::Failed);
return;
}
if(state() == SearchRequest) {
ignoredUrls = 0;
processSearchResults();
} else if(state() == ImageRequest)
processImageResults(count);
}
int MPIPortsCommunication:: getRemoteCommunicatorSize()
{
preciceTrace ( "getRemoteCommunicatorSize()" );
assertion ( _isConnection );
int remoteSize = 0;
MPI_Comm_remote_size ( communicator(), &remoteSize );
return remoteSize;
}
/*
* Read and execute commands from a specified command file.
*/
void McSimulation::readCommands(std::istream &in)
{
if (!isInitialized_) {
UTIL_THROW("McSimulation is not initialized");
}
std::string command;
std::string filename;
std::ifstream inputFile;
std::ofstream outputFile;
#ifndef UTIL_MPI
std::istream& inBuffer = in;
#else
std::stringstream inBuffer;
std::string line;
#endif
bool readNext = true;
while (readNext) {
// iffdef UTIL_MPI, read a line and copy to stringstream inBuffer
// ifndef UTIL_MPI, inBuffer is simply a reference to istream in.
#ifdef UTIL_MPI
// Read a command line, and broadcast if necessary.
if (!hasIoCommunicator() || isIoProcessor()) {
getNextLine(in, line);
}
if (hasIoCommunicator()) {
bcast<std::string>(communicator(), line, 0);
}
// Copy the command line to inBuffer.
inBuffer.clear();
for (unsigned i=0; i < line.size(); ++i) {
inBuffer.put(line[i]);
}
#endif
inBuffer >> command;
Log::file() << command;
if (isRestarting_) {
if (command == "RESTART") {
int endStep;
inBuffer >> endStep;
Log::file() << " " << iStep_ << " to "
<< endStep << std::endl;
simulate(endStep, isRestarting_);
isRestarting_ = false;
} else {
UTIL_THROW("Missing RESTART command");
}
} else {
virtual int run(int argc, char* argv[])
{
ClientOrServer::run(argc, argv);
// Register a new object adapter
Ice::ObjectAdapterPtr adapter = communicator()->createObjectAdapter("SonarEventListener.Subscriber");
// Subscribe to the event
IceStorm::QoS qos;
Ice::ObjectPrx subscriber = adapter->addWithUUID(new SonarEventPrinter);
sonarTopic->subscribe(qos, subscriber);
// Start the adapter and keep going until the program is killed
adapter->activate();
shutdownOnInterrupt();
communicator()->waitForShutdown();
sonarTopic->unsubscribe(subscriber);
return EXIT_SUCCESS;
}
void
diy::mpi::communicator::
duplicate(const communicator& other)
{
#ifndef DIY_NO_MPI
MPI_Comm newcomm;
MPI_Comm_dup(other.comm_, &newcomm);
(*this) = std::move(communicator(newcomm,true));
#endif
}
int Server::run(int argc, char* argv[])
{
int ret;
if ((ret = SPI_init())) return ret;
shutdownOnInterrupt();
Ice::ObjectAdapterPtr adapter = communicator()->
createObjectAdapterWithEndpoints("Guitar", "default -p 10000");
guitarlib::ApplicationI *app = new guitarlib::ApplicationI(adapter);
adapter->add(app, communicator()->stringToIdentity("Application"));
adapter->activate();
communicator()->waitForShutdown();
ret = SPI_exit();
return ret;
}
int ClientApp::initializeDetector(cvac::DetectorPrx detector)
{
cvac::DetectorData detectorData;
cvac::localAndClientMsg(VLogger::DEBUG, NULL, "Client using Detector-name: %s\n", detector->ice_getIdentity().name.c_str());
detectorData.type = ::cvac::FILE;
// If a detector dat file was passed then use that instead of getting it from the config file
if (m_detectorData.length() > 0)
{
detectorData.file.directory.relativePath = cvac::getFilePath(m_detectorData);
detectorData.file.filename = cvac::getFileName(m_detectorData);
}
else
{ // Determine which data files to pass as detector data based on .DetectorFilename in 'config.client'
std::string filenameStr = m_detectorName + ".DetectorFilename";
Ice::PropertiesPtr props = communicator()->getProperties();
std::string filename = props->getProperty(filenameStr);
if (filename.length() == 0)
{
cvac::localAndClientMsg(VLogger::WARN, NULL, "No .DetectorFilename pair found for %s.\n", m_detectorName.c_str());
return EXIT_FAILURE;
}
if ((filename.length() > 1 && filename[1] == ':' )||
filename[0] == '/' ||
filename[0] == '\\')
{ // absolute path
int idx = filename.find_last_of('/');
detectorData.file.directory.relativePath = filename.substr(0, idx);
detectorData.file.filename = filename.substr(idx+1, filename.length() - idx);
}
else
{ //Add the current directory
detectorData.file.directory.relativePath = "DefaultDetectors";
detectorData.file.filename = filename;
}
}
try
{
cvac::localAndClientMsg(VLogger::INFO, NULL, "Initializing detector with relativePath: %s\n", detectorData.file.directory.relativePath.c_str());
cvac::localAndClientMsg(VLogger::INFO, NULL, "Initializing detector with filename: %s\n", detectorData.file.filename.c_str());
detector->initialize(5, detectorData);
cvac::localAndClientMsg(VLogger::DEBUG, NULL, "IceBox test client: initialized the detector\n");
}
catch (const Ice::Exception& ex)
{
cvac::localAndClientMsg(VLogger::WARN, NULL, "Ice- name(): %s\n", ex.ice_name().c_str());
cvac::localAndClientMsg(VLogger::WARN, NULL, "Ice- stackTrace(): \n%s\n", ex.ice_stackTrace().c_str());
cvac::localAndClientMsg(VLogger::WARN, NULL, "%s\n", ex.what());
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
QFile* QUHttpCollector::openLocalFile(const QString &filePath) {
QFile *file = new QFile(filePath, this);
if(!file->open(QIODevice::WriteOnly | QIODevice::Truncate)) {
communicator()->send(tr("Could not open local file: \"%1\"").arg(filePath));
return 0;
}
_localFiles.append(file);
return file;
}
int main(int argc, char** argv) {
// Random seed
#ifdef _WIN32
srand(123456);
#else
srandom(123456);
#endif
// Client or server mode?
char* modeString = getOption(argc, argv, "-m");
CommunicatorMode mode = COMMUNICATOR_MODE_NONE;
if (!strcmp(modeString, "client"))
mode = COMMUNICATOR_MODE_CLIENT;
else if (!strcmp(modeString, "server"))
mode = COMMUNICATOR_MODE_SERVER;
else
printUsageAndExit(argv[0]);
// Declare our timeline. It will be initialized by the Communicator.
Timeline timeline;
// Declare our communicator
Communicator communicator(timeline, mode);
// If we're a client, who are we connecting to?
if (mode == COMMUNICATOR_MODE_CLIENT)
communicator.SetServer(getOption(argc, argv, "-s"));
// Otherwise, how many clients are we waiting for?
else {
int numClients = atoi(getOption(argc, argv, "-n"));
if (numClients < 0)
printUsageAndExit(argv[0]);
communicator.SetNumClientsExpected((unsigned) numClients);
}
// Start background music
sf::Music Music;
if (!Music.OpenFromFile("scenefiles/bgm.ogg"))
{
std::cout << "Error loading music file\n";
}
Music.SetLoop(true);
Music.Play();
Game growblesGame(timeline, communicator);
growblesGame.Setup();
return 0;
}
std::string Application::getProxyString(const std::string name)
{
std::string proxy;
proxy = communicator()->getProperties()->getProperty( name );
cout << "[" << __FILE__ << "]: Loading [" << proxy << "] proxy at '" << name << "'..." << endl;
//rInfo("Application::Loading "+QString::fromStdString(proxy)+" at "+QString::fromStdString(name));
if( proxy.empty() )
{
cout << "[" << __FILE__ << "]: Error loading proxy config!" << endl;
return "";
}
else
return proxy;
}
void PetscPreconditioner<T>::set_petsc_subpreconditioner_type(const PCType type, PC& pc)
#endif
{
// For catching PETSc error return codes
int ierr = 0;
// get the communicator from the PETSc object
Parallel::communicator comm;
PetscObjectGetComm((PetscObject)pc, &comm);
Parallel::Communicator communicator(comm);
// All docs say must call KSPSetUp or PCSetUp before calling PCBJacobiGetSubKSP.
// You must call PCSetUp after the preconditioner operators have been set, otherwise you get the:
//
// "Object is in wrong state!"
// "Matrix must be set first."
//
// error messages...
ierr = PCSetUp(pc);
CHKERRABORT(comm,ierr);
// To store array of local KSP contexts on this processor
KSP* subksps;
// the number of blocks on this processor
PetscInt n_local;
// The global number of the first block on this processor.
// This is not used, so we just pass PETSC_NULL instead.
// int first_local;
// Fill array of local KSP contexts
ierr = PCBJacobiGetSubKSP(pc, &n_local, PETSC_NULL, &subksps);
CHKERRABORT(comm,ierr);
// Loop over sub-ksp objects, set ILU preconditioner
for (PetscInt i=0; i<n_local; ++i)
{
// Get pointer to sub KSP object's PC
PC subpc;
ierr = KSPGetPC(subksps[i], &subpc);
CHKERRABORT(comm,ierr);
// Set requested type on the sub PC
ierr = PCSetType(subpc, type);
CHKERRABORT(comm,ierr);
}
}
bool Application::configGetString( const std::string name, std::string &value, const std::string default_value, QStringList *list)
{
value = communicator()->getProperties()->getProperty( name );
if ( value.length() == 0)
{
std::cout << name << " property does not exist. Using default value." << std::endl;
value = default_value;
return false;
}
if(list != NULL)
{
if (list->contains(QString::fromStdString(value)) == false)
{
qFatal("Reading config file: %s is not a valid string", name.c_str());
}
}
std::cout << name << " " << value << std::endl;
return true;
}
bool Application::configGetFloat( const std::string name, float &value, const float default_value, QList< int > *list )
{
QString tmp;
tmp = QString::fromStdString(communicator()->getProperties()->getProperty( name ));
if ( tmp.length() == 0)
{
std::cout << name << " property does not exist. Using default value." << std::endl;
value = default_value;
return false;
}
value = tmp.toFloat();
if(list != NULL)
{
if (list->contains(value) == false)
{
qFatal("Reading config file: '%s' is not a valid float", name.c_str());
}
}
std::cout << name << " " << value << std::endl;
return true;
}
bool Application::configGetInt( const std::string name, int &value, const int default_value, QList< int > *list )
{
string tmp;
tmp = communicator()->getProperties()->getProperty( name );
if ( tmp.length() == 0)
{
std::cout << name << " property does not exist. Using default value." << std::endl;
value = default_value;
return false;
}
value = std::atoi( tmp.c_str() );
if(list != NULL)
{
if (list->contains(value) == false)
{
qFatal("Reading config file: %s is not a valid integer", name.c_str());
}
}
std::cout << name << " " << value << std::endl;
return true;
}
void MPIPortsCommunication:: acceptConnection
(
const std::string& nameAcceptor,
const std::string& nameRequester,
int acceptorProcessRank,
int acceptorCommunicatorSize )
{
preciceTrace2 ( "acceptConnection()", nameAcceptor, nameRequester );
assertion ( not _isConnection );
int argc = 1;
char* arg = new char[8];
strcpy(arg, "precice");
char** argv = &arg;
utils::Parallel::initialize(&argc, &argv, nameAcceptor);
delete[] arg;
MPI_Open_port(MPI_INFO_NULL, _portname);
// Write portname to file
std::string portFilename ( _publishingDirectory + "." + nameRequester + "-portname" );
preciceDebug ( "Writing server connection info to file " + portFilename );
std::ofstream outFile;
outFile.open ((portFilename + "~").c_str(), std::ios::out);
outFile << _portname;
outFile.close();
// To give the file first a "wrong" name prevents early reading errors
rename( (portFilename + "~").c_str(), portFilename.c_str() );
preciceDebug("Calling MPI_Comm_accept() with portname = " << _portname);
MPI_Comm localComm = utils::Parallel::getLocalCommunicator();
MPI_Comm_accept ( _portname, MPI_INFO_NULL, 0, localComm, &communicator() );
if ( utils::Parallel::getLocalProcessRank() == 0 ){
if ( remove(portFilename.c_str()) != 0 ) {
preciceWarning ( "acceptConnection()", "Could not remove port information file!" );
}
}
_isConnection = true;
}
template<typename T, typename Op> inline void all_reduce(const Op& op, const std::vector<T>& in_values, const std::vector<int>& in_map, std::vector<T>& out_values, const std::vector<int>& out_map, const int stride=1)
{
mpi::all_reduce(communicator(), op, in_values, in_map, out_values, out_map, stride);
}
void QUHttpCollector::processNetworkStateChange(int state) {
if(state == QHttp::Sending)
communicator()->send(tr("Sending..."));
else if(state == QHttp::Reading)
communicator()->send(tr("Reading..."));
}
template<typename T, typename Op> inline void reduce(const Op& op, const std::vector<T>& in_values, std::vector<T>& out_values, const int root, const int stride=1)
{
mpi::reduce(communicator(), op, in_values, out_values, root, stride);
}
bool
FrontEndServer::start (int argc, char *argv[], int& status)
{
ClientMonitorPtr clientMonitor_Ptr;
Ice::LoggerPtr iceLogger_Ptr;
ReaperTaskPtr reaperTask_Ptr;
std::ostringstream oss;
int timeout;
bool logFlag;
IceStorm::TopicManagerPrx topicManager_Prx;
/* Publisher */
IceStorm::TopicPrx topicPub_Prx;
Ice::ObjectPrx publisher;
FrontEndComm::FrontEndToEntityCommPrx frontEndToEntityComm_Prx;
/* Subscriber */
IceStorm::TopicPrx topicSub_Prx;
Ice::ObjectPrx subscriber;
IceStorm::QoS qos;
timeout = communicator ()->getProperties ()->getPropertyAsIntWithDefault ("Reaper_Timeout", 10);
logFlag = communicator ()->getProperties ()->getPropertyAsIntWithDefault ("Server.Trace", 0) != 0;
iceLogger_Ptr = communicator ()->getLogger ();
reaperTask_Ptr = new ReaperTask (timeout, iceLogger_Ptr, logFlag);
timer_Ptr->scheduleRepeated (reaperTask_Ptr, IceUtil::Time::seconds (timeout));
try
{
/* Publisher handler */
topicManager_Prx = IceStorm::TopicManagerPrx::checkedCast
(communicator ()->propertyToProxy ("TopicManager.Proxy1"));
if (!topicManager_Prx)
{
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "Invalid proxy for topic manager (Publisher).";
iceLogger_Ptr->trace ("Error", oss.str ());
}
return false;
}
try
{
topicPub_Prx = topicManager_Prx->retrieve (SERVER_TO_SERVER_COMM_1);
}
catch (const IceStorm::NoSuchTopic& ex)
{
try
{
topicPub_Prx = topicManager_Prx->create (SERVER_TO_SERVER_COMM_1);
}
catch (const IceStorm::TopicExists& ex)
{
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "topic already exists (Publisher).";
iceLogger_Ptr->trace ("Error", oss.str ());
}
return false;
}
}
/// Retrieving the publisher
publisher = topicPub_Prx->getPublisher ();
frontEndToEntityComm_Prx = FrontEndComm::FrontEndToEntityCommPrx::uncheckedCast (publisher);
/* End of publisher handle */
adapter_Ptr = communicator ()->createObjectAdapter ("FrontEndServer");
clientMonitor_Ptr = new ClientMonitor (frontEndToEntityComm_Prx, iceLogger_Ptr, logFlag);
/*
if (!clientMonitor_Ptr->init ())
{
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "ClientMonitor failed to initialize.";
iceLogger_Ptr->trace ("Error", oss.str ());
}
return false;
}
*/
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "ClientMonitor started successfully.";
iceLogger_Ptr->trace ("Info", oss.str ());
}
/* Subscriber handler */
topicManager_Prx = IceStorm::TopicManagerPrx::checkedCast
(communicator ()->propertyToProxy ("TopicManager.Proxy0"));
while (true)
{
try
{
topicSub_Prx = topicManager_Prx->retrieve (SERVER_TO_SERVER_COMM_0);
break;
}
catch (const IceStorm::NoSuchTopic& ex)
{
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "unable to retrieve topic (Subscriber). Retrying...";
iceLogger_Ptr->trace ("Error", oss.str ());
}
sleep (1);
}
}
subscriber = adapter_Ptr-> add (new EntityToFrontEndComm_I (clientMonitor_Ptr),
communicator ()->stringToIdentity ("EntityToFrontEndComm"));
/// Ensuring that messages will be received in order
qos["reliability"] = "ordered";
try
{
topicSub_Prx->subscribeAndGetPublisher (qos, subscriber);
}
catch(const IceStorm::AlreadySubscribed& ex)
{
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "already subscribed (Subscriber).";
iceLogger_Ptr->trace ("Warning", oss.str ());
}
}
/* End of subscriber handle */
adapter_Ptr->add (new SessionManager (clientMonitor_Ptr, iceLogger_Ptr, logFlag),
communicator ()->stringToIdentity ("SessionManager"));
adapter_Ptr->add (new PollSessionManager_I (clientMonitor_Ptr, reaperTask_Ptr,
iceLogger_Ptr, logFlag), communicator ()->stringToIdentity ("PollSessionManager"));
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "SessionManager started successfully.";
iceLogger_Ptr->trace ("Info", oss.str ());
}
adapter_Ptr->activate ();
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "started successfully.";
iceLogger_Ptr->trace ("Info", oss.str ());
}
}
catch (const Ice::LocalException& ex)
{
status = EXIT_FAILURE;
timer_Ptr->destroy ();
if (logFlag)
{
oss.str ("");
oss << "FrontEndServer: " << "failed to initialize. Closing...";
iceLogger_Ptr->trace ("Error", oss.str ());
}
throw;
}
status = EXIT_SUCCESS;
return true;
}
// -------------------------------------------------------------
// GraphPartitionerImplementation::partition
// -------------------------------------------------------------
void
GraphPartitionerImplementation::partition(void)
{
static const bool verbose(false);
// Make sure that all GA communication has been flushed from the system
communicator().sync();
gridpack::utility::CoarseTimer *timer;
timer = NULL;
// timer = gridpack::utility::CoarseTimer::instance();
int t_total, t_adj, t_part, t_node_dest, t_edge_dest, t_gnode_dest, t_gedge_dest;
if (timer != NULL) {
t_total = timer->createCategory("GraphPartitioner::partition(): Total");
t_adj = timer->createCategory("GraphPartitioner::partition: Adjacency");
t_part = timer->createCategory("GraphPartitioner::partition: Partitioner");
t_node_dest = timer->createCategory("GraphPartitioner::partition: Node Destination");
t_edge_dest = timer->createCategory("GraphPartitioner::partition: Edge Destinations");
t_gnode_dest = timer->createCategory("GraphPartitioner::partition: Ghost Node Destination");
t_gedge_dest = timer->createCategory("GraphPartitioner::partition: Ghost Edge Destination");
}
if (timer != NULL) timer->start(t_total);
if (timer != NULL) timer->start(t_adj);
p_adjacency_list.ready();
int maxdim(2);
int dims[maxdim], lo[maxdim], hi[maxdim], ld[maxdim];
ld[0] = 1;
ld[1] = 1;
int locnodes(p_adjacency_list.nodes());
int locedges(p_adjacency_list.edges());
int allnodes;
int alledges;
communicator().barrier();
boost::mpi::all_reduce(communicator(),
locnodes, allnodes, std::plus<int>());
boost::mpi::all_reduce(communicator(),
locedges, alledges, std::plus<int>());
if (allnodes <= 0 || alledges <= 0) {
boost::format fmt("%d: GraphPartitioner::partition(): called without nodes (%d) or edges (%)");
std::string msg = boost::str(fmt % communicator().worldRank() % allnodes % alledges);
throw Exception(msg);
}
if (timer != NULL) timer->stop(t_adj);
if (timer != NULL) timer->start(t_part);
this->p_partition(); // fills p_node_destinations
if (timer != NULL) timer->stop(t_part);
// make two GAs, one that holds the node source and another that
// node destination; each is indexed by global node index
if (timer != NULL) timer->start(t_node_dest);
int theGAgroup(communicator().getGroup());
int oldGAgroup = GA_Pgroup_get_default();
GA_Pgroup_set_default(theGAgroup);
std::vector<int> nodeidx(locnodes);
std::vector<int *> stupid(locnodes);
for (Index n = 0; n < static_cast<Index>(locnodes); ++n) {
nodeidx[n] = p_adjacency_list.node_index(n);
stupid[n] = &nodeidx[n];
}
dims[0] = allnodes;
boost::scoped_ptr<GA::GlobalArray>
node_dest(new GA::GlobalArray(MT_C_INT, 1, dims, "Node Destinations Process", NULL)),
node_src(new GA::GlobalArray(MT_C_INT, 1, dims, "Node Source Process", NULL));
node_dest->scatter(&p_node_destinations[0], &stupid[0], locnodes);
{
std::vector<int> nsrc(locnodes, this->processor_rank());
node_src->scatter(&nsrc[0], &stupid[0], locnodes);
}
communicator().sync();
if (verbose) {
node_src->print();
node_dest->print();
}
if (timer != NULL) timer->stop(t_node_dest);
// edges are assigned to the same partition as the lowest numbered
// node to which it connects, which are extracted from the node
// destination GA.
if (timer != NULL) timer->start(t_edge_dest);
nodeidx.resize(locedges);
stupid.resize(locedges);
std::vector<int> e1dest(locedges);
for (Index e = 0; e < static_cast<Index>(locedges); ++e) {
Index n1, n2;
p_adjacency_list.edge(e, n1, n2);
nodeidx[e] = std::min(n1, n2);
stupid[e] = &nodeidx[e];
}
node_dest->gather(&e1dest[0], &stupid[0], locedges);
if (verbose) {
for (Index e = 0; e < static_cast<Index>(locedges); ++e) {
Index n1, n2;
p_adjacency_list.edge(e, n1, n2);
std::cout << processor_rank() << ": active edge " << e
<< " (" << n1 << "->" << n2 << "): "
<< "destination: " << e1dest[e] << std::endl;
}
}
p_edge_destinations.clear();
p_edge_destinations.reserve(locedges);
std::copy(e1dest.begin(), e1dest.end(),
std::back_inserter(p_edge_destinations));
if (timer != NULL) timer->stop(t_edge_dest);
// determine (possible) destinations for ghost edges (highest numbered node)
if (timer != NULL) timer->start(t_gedge_dest);
std::vector<int> e2dest(locedges);
for (Index e = 0; e < static_cast<Index>(locedges); ++e) {
Index n1, n2;
p_adjacency_list.edge(e, n1, n2);
nodeidx[e] = std::max(n1, n2);
stupid[e] = &nodeidx[e];
}
node_dest->gather(&e2dest[0], &stupid[0], locedges);
if (verbose) {
for (Index e = 0; e < static_cast<Index>(locedges); ++e) {
Index n1, n2;
p_adjacency_list.edge(e, n1, n2);
std::cout << processor_rank() << ": ghost edge " << e
<< " (" << n1 << "->" << n2 << "): "
<< "destination: " << e2dest[e] << std::endl;
}
}
communicator().sync();
// These are no longer needed
node_dest.reset();
node_src.reset();
p_ghost_edge_destinations.reserve(locedges);
std::copy(e2dest.begin(), e2dest.end(),
std::back_inserter(p_ghost_edge_destinations));
if (timer != NULL) timer->stop(t_gedge_dest);
if (timer != NULL) timer->start(t_gnode_dest);
// determine destinations for ghost nodes: go thru the edges and
// compare destinations of connected nodes; if they're different,
// then both ends need to be ghosted (to different processors)
// It's possible that edges are distributed over multiple processes,
// which could result in a different set of ghost destinations for a
// given node on each process. These need to be put together.
// In this approach, which is really slow, take each local list of
// ghost node, send it to all processes. Each process extracts the
// ghost node destination for its locally owned nodes.
// a particular node and destination needs to be unique, hence the
// use of set<>; this may be too slow with large networks and
// processors
// typedef std::set< std::pair<Index, int> > DestList;
// DestList gnodedest;
// for (Index e = 0; e < locedges; ++e) {
// Index n1, n2;
// p_adjacency_list.edge(e, n1, n2);
// int n1dest(e1dest[e]);
// int n2dest(e2dest[e]);
// if (verbose) {
// std::cout << processor_rank() << ": edge " << e
// << " (" << n1 << "->" << n2 << "): "
// << "destinations: " << n1dest << ", " << n2dest << std::endl;
// }
// if (n1dest != n2dest) {
// gnodedest.insert(std::make_pair(std::min(n1,n2), n2dest));
// gnodedest.insert(std::make_pair(std::max(n1,n2), n1dest));
// }
// }
// if (verbose) {
// if (this->processor_rank() == 0) {
// std::cout << "Ghost node destinations: " << std::endl;
// }
// for (int p = 0; p < this->processor_size(); ++p) {
// if (this->processor_rank() == p) {
// std::cout << p << ": ";
// for (DestList::const_iterator i = gnodedest.begin();
// i != gnodedest.end(); ++i) {
// std::cout << "(" << i->first << ":" << i->second << "),";
// }
// std::cout << std::endl;
// }
// this->communicator().barrier();
// }
// }
// p_ghost_node_destinations.resize(locnodes);
// DestList tmp;
// for (int p = 0; p < this->processor_size(); ++p) {
// tmp.clear();
// if (this->processor_rank() == p) {
// tmp = gnodedest;
// }
// broadcast(communicator().getCommunicator(), tmp, p);
// for (Index n = 0; n < locnodes; ++n) {
// Index nodeidx(p_adjacency_list.node_index(n));
// for (DestList::const_iterator i = tmp.begin();
// i != tmp.end(); ++i) {
// if (nodeidx == i->first) {
// p_ghost_node_destinations[n].push_back(i->second);
// }
// }
// }
// }
// Here, a 2D GA is used to store ghost node destinations. Each
// process takes it's set of ghost node destinations and appends
// those lists already in the GA.
// Determine the maximum node connectivity. There needs to be enough
// room in the GA to store all connections to a node.
size_t lconn(0), maxconn(0);
for (int l = 0; l < locnodes; ++l) {
lconn = std::max(lconn,
p_adjacency_list.node_neighbors(l));
}
boost::mpi::all_reduce(communicator(), lconn, maxconn,
boost::mpi::maximum<int>());
BOOST_ASSERT(maxconn >= lconn);
dims[0] = allnodes;
dims[1] = maxconn;
ld[0] = maxconn;
node_dest.reset(new GA::GlobalArray(MT_C_INT, 2, &dims[0],
"Ghost node dest processes", NULL));
boost::scoped_ptr<GA::GlobalArray>
node_dest_count(new GA::GlobalArray(MT_C_INT, 1, &dims[0],
"Ghost node dest count", NULL));
{
int bogus;
bogus = -1; node_dest->fill(&bogus);
bogus = 0; node_dest_count->fill(&bogus);
}
std::vector<int> lcount(allnodes, 0);
for (int p = 0; p < this->processor_size(); ++p) {
if (this->processor_rank() == p) {
lo[0] = 0; hi[0] = allnodes - 1;
node_dest_count->get(&lo[0], &hi[0], &lcount[0], &ld[0]);
for (Index e = 0; e < static_cast<Index>(locedges); ++e) {
Index n1, n2;
p_adjacency_list.edge(e, n1, n2);
int n1dest(e1dest[e]);
int n2dest(e2dest[e]);
if (verbose) {
std::cout << processor_rank() << ": edge " << e
<< " (" << n1 << "->" << n2 << "): "
<< "destinations: " << n1dest << ", " << n2dest << std::endl;
}
if (n1dest != n2dest) {
int nid, dest;
nid = std::min(n1,n2);
dest = n2dest;
lo[0] = nid; hi[0] = lo[0];
lo[1] = lcount[nid]; hi[1] = lo[1];
node_dest->put(&lo[0], &hi[0], &dest, &ld[0]);
lcount[nid] += 1;
nid = std::max(n1,n2);
dest = n1dest;
lo[0] = nid; hi[0] = lo[0];
lo[1] = lcount[nid]; hi[1] = lo[1];
node_dest->put(&lo[0], &hi[0], &dest, &ld[0]);
lcount[nid] += 1;
}
}
lo[0] = 0; hi[0] = allnodes - 1;
node_dest_count->put(&lo[0], &hi[0], &lcount[0], &ld[0]);
}
this->communicator().sync();
}
// After all processes have made their contribution to the ghost
// node destination GA, each process grabs that part that refers to
// its local nodes and fills p_ghost_edge_destinations.
lo[0] = 0; hi[0] = allnodes - 1;
node_dest_count->get(&lo[0], &hi[0], &lcount[0], &ld[0]);
p_ghost_node_destinations.clear();
p_ghost_node_destinations.resize(locnodes);
std::vector<int> tmpdest(this->processor_size(), 0);
for (Index n = 0; n < static_cast<Index>(locnodes); ++n) {
Index nid(p_adjacency_list.node_index(n));
p_ghost_node_destinations[n].clear();
if (lcount[nid] > 0) {
lo[0] = nid;
hi[0] = nid;
lo[1] = 0;
hi[1] = lcount[nid] - 1;
tmpdest.resize(lcount[nid]);
node_dest->get(&lo[0], &hi[0], &tmpdest[0], &ld[0]);
// there may be duplicates, so get rid of them
if (tmpdest.size() > 1) {
std::stable_sort(tmpdest.begin(), tmpdest.end());
std::unique(tmpdest.begin(), tmpdest.end());
}
p_ghost_node_destinations[n].reserve(tmpdest.size());
std::copy(tmpdest.begin(), tmpdest.end(),
std::back_inserter(p_ghost_node_destinations[n]));
}
}
if (timer != NULL) timer->stop(t_gnode_dest);
GA_Pgroup_set_default(oldGAgroup);
if (timer != NULL) timer->stop(t_total);
// if (timer) timer->dump();
}
template<typename T> inline void broadcast(const std::vector<T>& in_values, const std::vector<int>& in_map, std::vector<T>& out_values, const std::vector<int>& out_map, const int root, const int stride=1)
{
mpi::broadcast(communicator(), in_values, in_map, out_values, out_map, root, stride);
}
template<typename T> inline T* broadcast(const T* in_values, const int in_n, const int *in_map, T* out_values, const int *out_map, const int root, const int stride=1)
{
return mpi::broadcast(communicator(), in_values, in_n, in_map, out_values, out_map, root, stride);
}
template<typename T, typename Op> inline T* all_reduce(const Op& op, const T* in_values, const int in_n, const int *in_map, T* out_values, const int *out_map, const int stride=1)
{
return mpi::all_reduce(communicator(), op, in_values, in_n, in_map, out_values, out_map, stride);
}
