void dtkComposerEvaluatorProcess::run(void) { d->status = 0; if (!d->factory) { dtkFatal() << "No factory set ! abort process execution"; d->status = 1; return; } // int rank = d->comm->rank(); // int size = d->comm->size(); bool new_composition; QByteArray data; d->parent_comm->broadcast(data,0); dtkDebug() << "Ok, composition received, parse" ; QString composition(data); if (composition == "not-modified") { dtkInfo() << "composition hasn't changed"; new_composition = false; } else { new_composition = true; } if (new_composition && composition.isEmpty()) { dtkFatal() << "Empty composition, abort" ; d->status = 1; return; } if (new_composition) { dtkDebug() << "parse composition" ; d->reader->readString(composition); // if (dtkComposerNodeSpawn *spawn = dynamic_cast<dtkComposerNodeSpawn *>(d->scene->root()->nodes().first()->wrapee())) { // spawn->setCommunicator(d->parent_comm); // spawn->setInternalCommunicator(d->comm); // spawn->setApplication(d->application); // } else { // dtkFatal() << "Can't find spawn node in composition, abort"; // return 1; // } } dtkDebug() << "run composition" ; d->evaluator->run(); dtkDebug() << "finished" ; }
void dtkComposerNodeCommunicatorReceive::run(void) { if (!d->receiver_source.isEmpty()) { if (d->receiver_comm.isEmpty() && d->receiver_socket.isEmpty()) { dtkError() << "Send node at either a socket or a communicator."; return; } d->source = d->receiver_source.data(); d->tag = 0; if (!d->receiver_tag.isEmpty()) d->tag = d->receiver_tag.data(); d->emitter.clearData(); if (!d->receiver_data.isEmpty()) { d->emitter.setData(d->receiver_data.variant()); } if (d->receiver_comm.isEmpty()) { QTcpSocket *socket = d->receiver_socket.constData(); dtkDebug() << "TCP communicator. Parse message from socket, waiting for tag" << d->tag; if (d->msg_map.contains(d->tag)) { dtkDebug() << "msg already received for tag" << d->tag; // d->emitter.setTwinned(false); dtkDistributedMessage *msg = d->msg_map.take(d->tag); d->emitter.setData(msg->content()); // d->emitter.setTwinned(true); delete msg; return; } else { dtkTrace() << "msg not yet received, wait for data"; } socket->blockSignals(true); // needed ? if (!socket->waitForReadyRead(300000)) { dtkWarn() << "Data not ready in receive for rank " << d->source; } else { dtkDistributedMessage msg; msg.parse(socket); qlonglong msg_tag = msg.header("Tag").toLongLong(); if (msg_tag == d->tag || d->tag == dtkDistributedCommunicator::ANY_TAG) { dtkTrace() << "OK, this is the expected tag " << d->tag; // d->emitter.setTwinned(false); d->emitter.setData(msg.content()); // d->emitter.setTwinned(true); if (d->tag == dtkDistributedCommunicator::ANY_TAG) d->tag = msg_tag; } else { //store msg for another call with the right tag dtkInfo() << "Msg received, but wrong tag, store the msg" << d->tag << msg_tag; d->msg_map.insert(msg_tag, &msg); this->run(); // do it again } } socket->blockSignals(false); // needed ? } else { // MPI QByteArray array; dtkDistributedCommunicatorStatus status; dtkDistributedCommunicator *communicator = d->receiver_comm.constData(); //FIXME:[migration] new transmitter requieres a clone method in object, even if dataTransmission is set to reference communicator->receive(array, d->source, d->tag, status); if (d->tag == dtkDistributedCommunicator::ANY_TAG) d->tag = status.tag(); if (d->source == dtkDistributedCommunicator::ANY_SOURCE) d->source = status.source(); if (!array.isEmpty()) { // d->emitter.setTwinned(false); d->emitter.setData(array); // d->emitter.setTwinned(true); } else { dtkWarn() << "Empty data in receive"; d->emitter.clearData(); } } } else { d->emitter.clearData(); dtkWarn() << "Inputs not specified in receive. Nothing is done"; d->tag = 0; d->source = 0; } d->emitter_source.setData(d->source); d->emitter_tag.setData(d->tag); }
void dtkComposerNodeSpawn::begin(void) { if (!d->communicator) { // we are running on the controller, for the first time: // need to spawn processes if (d->size_receiver.isEmpty()) { d->np = QThread::idealThreadCount(); dtkInfo() << "Set number of spawned proceses to" << d->np; } else { d->np = d->size_receiver.data(); } d->policy.setNWorkers(d->np); // FIXME: don't use hardcoded plugin name d->policy.setType("qthreads"); d->communicator = d->policy.communicator(); // d->internal_comm = d->manager.spawn(); d->internal_comm_emitter.setData(d->internal_comm); d->rank = d->internal_comm->rank(); d->rank_emitter.setData(d->rank); if (!d->internal_comm) { dtkError() << "NULL internal communicator, spawn has failed !"; return; } dtkComposerEvaluatorProcess p; p.setInternalCommunicator(d->internal_comm); // p.setParentCommunicator(comm); // p.setFactory(factory); //FIXME: don't use manager // d->manager.exec(&p); } else { dtkTrace() << "communicator exists, no spawn"; } int first_transmitter = 3; if (d->is_parent) { int rank = dtkDistributedCommunicator::ROOT; if (d->current_hash != d->last_sent_hash){ // send sub-composition to rank 0 on remote node QByteArray compo = d->composition.toByteArray(); dtkDebug() << "running node remote begin statement on controller, send composition of size " << compo.size(); d->communicator->broadcast(compo, rank); d->last_sent_hash=d->current_hash; } else { dtkDebug() << "composition hash hasn't changed, send 'not-modified' to slave"; QByteArray data = QString("not-modified").toUtf8(); d->communicator->broadcast(data, rank); } // then send transmitters data int max = dtkComposerNodeComposite::receivers().count(); for (int i = first_transmitter; i < max; i++) { dtkComposerTransmitterReceiverVariant *t = dynamic_cast<dtkComposerTransmitterReceiverVariant *>(dtkComposerNodeComposite::receivers().at(i)); // FIXME: use our own transmitter variant list (see control nodes) QByteArray array; QDataStream stream(&array, QIODevice::WriteOnly); stream << t->variant(); dtkDebug() << "sending transmitter" << i << "of size" << array.size(); d->communicator->broadcast(array, rank); } } else if (d->communicator) { if (d->rank < 0) { dtkDebug() << "get rank/size on slave"; d->internal_comm_emitter.setData(d->internal_comm); d->rank = d->internal_comm->rank(); d->np = d->internal_comm->size(); dtkDebug() << "rank/size"<< d->rank << d->np; } dtkTrace() << "get transmitter data"; // running on the slave, receive data and set transmitters int max = dtkComposerNodeComposite::receivers().count(); for (int i = first_transmitter; i < max; i++) { dtkComposerTransmitterReceiverVariant *t = dynamic_cast<dtkComposerTransmitterReceiverVariant *>(dtkComposerNodeComposite::receivers().at(i)); QByteArray array; qint16 parent_rank = 0; d->communicator->broadcast(array, parent_rank); // t->setTwinned(false); QDataStream stream(&array, QIODevice::ReadOnly); QVariant variant; stream >> variant; // FIXME : how can we set data ? // t->setVariant(variant); // t->setTwinned(true); } } else {
// create a dataset or return the identifier if we already created it hid_t& hdf5IoDataModelPrivate::datasetId(const QString& dataset_name, const dtkIoDataModel::DataType& type=dtkIoDataModel::Int, const int& dim=-1, quint64 *shape=nullptr) { if(!file_is_open) { dtkError() << "file is not open! "; } if(!dataset_hash.contains(dataset_name)) { if(dim>0) { dtkInfo() << "Dataset not existing, creating DataSet" << dataset_name; QStringList dataset_name_split = dataset_name.split( "/" ); //erase first (root) and last (dataset) elements to only keep groups dataset_name_split.pop_front(); dataset_name_split.pop_back(); if ( dataset_name_split.length() > 0) { //the dataset is in a group, we need to open or create groups QString path = "" ; for(auto it: dataset_name_split) { path = path % "/" % it; if(!group_hash.contains(path)) { if(H5Lexists(file_id, path.toUtf8().constData(), H5P_DEFAULT)) //The group exists group_hash[path] = H5Gopen(file_id, path.toUtf8().constData(), H5P_DEFAULT); else group_hash[path] = H5Gcreate(file_id, path.toUtf8().constData(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } } } //now create the dataset hid_t dataspace = H5Screate_simple(dim, shape, NULL); switch(type) { case dtkIoDataModel::Int: { dataset_hash[dataset_name] = H5Dcreate(file_id, dataset_name.toUtf8().constData(), H5T_NATIVE_INT, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); break; } case dtkIoDataModel::LongLongInt: { dataset_hash[dataset_name] = H5Dcreate(file_id, dataset_name.toUtf8().constData(), H5T_NATIVE_LLONG, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); break; } case dtkIoDataModel::Double: { dataset_hash[dataset_name] = H5Dcreate(file_id, dataset_name.toUtf8().constData(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); break; } default: dtkError() <<" datatype not supported"; }; H5Sclose(dataspace); } else { //we just want to open it dataset_hash[dataset_name] = H5Dopen(file_id, dataset_name.toUtf8().constData(), H5P_DEFAULT); } } //dataset already opened return dataset_hash[dataset_name]; }
void dtkComposerEvaluatorSlave::run(void) { d->status = 0; if ( !d->communicator_i) { d->communicator_i = dtkDistributed::communicator::instance(); if (d->communicator_i->rank() == 0) { std::cout << QString("DTK_JOBID="+this->jobId()).toStdString() << std::endl << std::flush; } } if (!d->factory) { dtkFatal() << "No factory set ! abort slave execution"; d->status = 1; return; } int rank = d->communicator_i->rank(); int size = d->communicator_i->size(); dtkDebug() << "communicator size is" << size; dtkDebug() << "our rank is" << rank; bool new_composition; if ( rank == 0) { QScopedPointer<dtkDistributedMessage> msg; if (!this->isConnected()) { dtkDebug() << "connect to server" << d->server; this->connect(d->server); if (this->isConnected()) { if (!d->composition_socket) { dtkDebug() << "open second socket to server" << d->server; d->composition_socket = new QTcpSocket; d->composition_socket->connectToHost(d->server.host(), d->server.port()); if (d->composition_socket->waitForConnected()) { msg.reset(new dtkDistributedMessage(dtkDistributedMessage::SETRANK,this->jobId(), dtkDistributedMessage::SLAVE_RANK )); msg->send(d->composition_socket); } else { dtkError() << "Can't connect to server"; d->status = 1; return; } } dtkDebug() << "connected, send our jobid to server" << this->jobId(); msg.reset(new dtkDistributedMessage(dtkDistributedMessage::SETRANK,this->jobId(),0)); msg->send(this->socket()); this->socket()->flush(); this->socket()->setParent(0); } else { dtkFatal() << "Can't connect to server" << d->server; d->status = 1; return; } } QString composition; dtkDebug() << "Wait for composition from controller " ; if (d->composition_socket->bytesAvailable() > 10) { dtkInfo() << "data already available, try to parse composition " << d->composition_socket->bytesAvailable(); } else if (!d->composition_socket->waitForReadyRead(600000)) { dtkFatal() << "No data received from server after 10mn, abort " ; d->status = 1; return; } else { dtkDebug() << "Ok, data received on composition socket, parse" ; } msg.reset(new dtkDistributedMessage); msg->parse(d->composition_socket); if (msg->type() == "xml") { new_composition = true; composition = QString(msg->content()); d->last_controller_rank = msg->header("x-forwarded-for").toInt(); d->composition_cache.insert(d->last_controller_rank, composition); } else if (msg->type() == "not-modified") { // reuse the old composition if (msg->header("x-forwarded-for").toInt() == d->last_controller_rank) { new_composition = false; } else { d->last_controller_rank = msg->header("x-forwarded-for").toInt(); dtkDebug() << "not modified, but from another controller" << d->last_controller_rank; new_composition = true; composition = d->composition_cache.value(d->last_controller_rank); } } else { dtkFatal() << "Bad composition type, abort" << msg->type() << msg->content(); d->status = 1; return; } if (new_composition && composition.isEmpty()) { dtkFatal() << "Empty composition, abort" ; d->status = 1; return; } dtkDebug() << "got composition from controller:" << composition; if (new_composition) { dtkDebug() << "new composition"; if (size > 1) { dtkDebug() << "send composition to our slaves"; for (int i=1; i< size; i++) { d->communicator_i->send(composition,i,0); } } dtkDebug() << "parse composition" ; d->reader->readString(composition); } else { dtkInfo() << "composition hasn't changed"; for (int i=1; i< size; i++) d->communicator_i->send(QString("rerun"),i,0); } if (new_composition) { if (dtkComposerNodeRemote *remote = dynamic_cast<dtkComposerNodeRemote *>(d->scene->root()->nodes().first()->wrapee())) { //FIXME: can we remove this ? // this->communicator()->setProperty("jobid",this->jobId()); remote->setSlave(this); remote->setJob(this->jobId()); remote->setCommunicator(d->communicator_i); } else { dtkFatal() << "Can't find remote node in composition, abort"; d->status = 1; return; } } dtkDebug() << "run composition" ; if (QThread::currentThread() == qApp->thread()) { dtkTrace() << "running on main thread, create a thread for the evaluator" ; QThread *workerThread = new QThread(this); QObject::connect(workerThread, SIGNAL(started()), d->evaluator, SLOT(run()), Qt::DirectConnection); QObject::connect(d->evaluator, SIGNAL(evaluationStopped()), workerThread, SLOT(quit())); QEventLoop loop; loop.connect(d->evaluator, SIGNAL(evaluationStopped()), &loop, SLOT(quit())); loop.connect(qApp, SIGNAL(aboutToQuit()), &loop, SLOT(quit())); this->socket()->moveToThread(workerThread); workerThread->start(); loop.exec(); workerThread->wait(); workerThread->deleteLater(); } else { dtkTrace() << "running on dedicated thread,run the evaluator" ; d->evaluator->run_static(); } dtkDebug() << "finished" ; } else { QString composition; d->communicator_i->receive(composition,0,0); if (composition != "rerun") { dtkDebug() << "new/changed composition, read" ; dtkDebug() << " composition is " << composition ; d->reader->readString(composition); dtkDebug() << "read done" ; } else { dtkDebug() << "reuse composition" ; } if (dtkComposerNodeRemote *remote = dynamic_cast<dtkComposerNodeRemote *>(d->scene->root()->nodes().first()->wrapee())) { remote->setSlave(this); remote->setJob(this->jobId()); remote->setCommunicator(d->communicator_i); dtkDebug() << "run composition" ; QThread *workerThread = new QThread(this); QObject::connect(workerThread, SIGNAL(started()), d->evaluator, SLOT(run()), Qt::DirectConnection); QObject::connect(d->evaluator, SIGNAL(evaluationStopped()), workerThread, SLOT(quit())); QEventLoop loop; loop.connect(d->evaluator, SIGNAL(evaluationStopped()), &loop, SLOT(quit())); loop.connect(qApp, SIGNAL(aboutToQuit()), &loop, SLOT(quit())); workerThread->start(); loop.exec(); workerThread->wait(); workerThread->deleteLater(); // d->evaluator->run_static(); dtkDebug() << "finished" ; } else { dtkFatal() << "Can't find remote node in composition, abort"; d->status = 1; return; } } }