/*! write data in the open file * dimensions */ void hdf5IoDataModel::write(const QString& dataset_name, DataType type, int dimension, quint64 *shape, void *values) { d->status=0; hid_t dataset_id = d->datasetId(dataset_name, type, dimension, shape); if(values) { switch(type) { case dtkIoDataModel::Int: { d->status = H5Dwrite(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, values); break; } case dtkIoDataModel::LongLongInt: { d->status = H5Dwrite(dataset_id, H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, values); break; } case dtkIoDataModel::Double: { d->status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, values); break; } default: dtkError() << "write method: Datatype not supported"; }; } if(d->status<0) { dtkError() << "error writing" << dataset_name; } }
//point selection write // assuming dataset and values have the same shape and the points values are contiguous in values void hdf5IoDataModel::writeByCoord(const QString &dataset_name, DataType type, quint64 nb_points, quint64* points_coord, void *values) { hid_t dataset_id = d->datasetId(dataset_name); //the selection within the file dataset's dataspace hid_t file_dataspace = H5Dget_space(dataset_id); if(H5Sselect_elements(file_dataspace, H5S_SELECT_SET, nb_points, points_coord)<0) { dtkError() << "ERROR selecting hyperslab" << dataset_name; } //set the dimensions of values. memory dataspace and the selection within it hid_t values_dataspace = H5Screate_simple(1, &nb_points, NULL); switch(type) { case dtkIoDataModel::Int: // TODO put d->prop_list_id instead of H5P_DEFAULT ???????? d->status = H5Dwrite(dataset_id, H5T_NATIVE_INT, values_dataspace, file_dataspace, H5P_DEFAULT, values); break; case dtkIoDataModel::LongLongInt: d->status = H5Dwrite(dataset_id, H5T_NATIVE_LLONG, values_dataspace, file_dataspace, H5P_DEFAULT, values); break; case dtkIoDataModel::Double: d->status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, values_dataspace, file_dataspace, H5P_DEFAULT, values); break; default: dtkError() << "write method: Datatype not supported"; }; if(d->status<0) { dtkError() << "error writing hyperslab" << dataset_name; } H5Sclose(file_dataspace); H5Sclose(values_dataspace); }
void hdf5IoDataModel::fileOpen(const QString &file_name, FileMode mode) { //if file is already open tell it to the user and return if(!d->file_is_open) { switch (mode) { case dtkIoDataModel::Trunc: d->file_id = H5Fcreate(file_name.toUtf8().constData(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); break; case dtkIoDataModel::NotExisting: d->file_id = H5Fcreate(file_name.toUtf8().constData(), H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT); break; case dtkIoDataModel::ReadOnly: d->file_id = H5Fopen(file_name.toUtf8().constData(), H5F_ACC_RDONLY, H5P_DEFAULT); break; case dtkIoDataModel::ReadWrite: d->file_id = H5Fopen(file_name.toUtf8().constData(), H5F_ACC_RDWR, H5P_DEFAULT); break; default: dtkError() << "unsupported fileMode"; }; if(d->file_id<0) dtkError() << "error in fileOpen for file_name " << file_name; else d->file_is_open=true; } else { qDebug() << "File" << file_name << "is already open, please close it before opening a new one"; } }
void pHdf5IoDataModel::fileOpen(const QString &file_name, FileMode mode) { //if file is already open tell it to the user and return if(!d->file_is_open) { //if we didn't set the communicator error if(d->comm==nullptr) { dtkError() << __func__ << "communicator not set"; } // H5P_FILE_ACCESS applies to H5Fcreate and H5Fopen d->prop_list_id = H5Pcreate(H5P_FILE_ACCESS); MPI_Info info = MPI_INFO_NULL; MPI_Comm comm = *static_cast<MPI_Comm *>(d->comm->data()); H5Pset_fapl_mpio(d->prop_list_id, comm, info); switch (mode) { case dtkIoDataModel::Trunc: d->file_id = H5Fcreate (file_name.toUtf8().constData(), H5F_ACC_TRUNC, H5P_DEFAULT, d->prop_list_id); break; case dtkIoDataModel::NotExisting: d->file_id = H5Fcreate(file_name.toUtf8().constData(), H5F_ACC_EXCL, H5P_DEFAULT, d->prop_list_id); break; case dtkIoDataModel::ReadOnly: d->file_id = H5Fopen(file_name.toUtf8().constData(), H5F_ACC_RDONLY, d->prop_list_id); break; case dtkIoDataModel::ReadWrite: d->file_id = H5Fopen(file_name.toUtf8().constData(), H5F_ACC_RDWR, d->prop_list_id); break; default: dtkError() << "unsupported fileMode"; }; //close the property list for file H5Pclose(d->prop_list_id); if(d->file_id<0) { dtkError() << "error in fileOpen for file_name " << file_name; } else { //if the file is correctly open, create a propery list to collectively write datasets d->file_is_open = true; d->prop_list_id = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(d->prop_list_id, H5FD_MPIO_COLLECTIVE); } } else { qDebug() << "File" << file_name << "is already open, please close it before opening a new one"; } }
int dtkComposerEvaluatorProcess::exec(void) { if (!d->factory) { dtkError() << "No factory set ! abort process execution"; return 1; } // int rank = d->comm->rank(); // int size = d->comm->size(); bool new_composition; QByteArray data; d->parent_comm->broadcast(data,0); dtkDebug() << "Ok, composition received, parse" ; QString composition(data); if (composition == "not-modified") { dtkInfo() << "composition hasn't changed"; new_composition = false; } else { new_composition = true; } if (new_composition && composition.isEmpty()) { dtkError() << "Empty composition, abort" ; return 1; } if (new_composition) { dtkDebug() << "parse composition" ; d->reader->readString(composition); if (dtkComposerNodeSpawn *spawn = dynamic_cast<dtkComposerNodeSpawn *>(d->scene->root()->nodes().first()->wrapee())) { spawn->setCommunicator(d->parent_comm); spawn->setInternalCommunicator(d->comm); spawn->setApplication(d->application); } else { dtkError() << "Can't find spawn node in composition, abort"; return 1; } } dtkDebug() << "run composition" ; d->evaluator->run(); dtkDebug() << "finished" ; return 0; }
void dtkComposerNodeVectorRealExtractor::run(void) { if (!d->receiver_vector.isEmpty() && !d->receiver_index.isEmpty()) { dtkVectorReal *vector = d->receiver_vector.data(); qlonglong index = *d->receiver_index.data(); if (!vector) { dtkError() << "Input vector is not defined."; return; } if (index >= vector->size()) { dtkWarn() << "index > size of the input vector. Zero is returned."; d->value = 0; } else { d->value = (*vector)[index]; } } else { dtkWarn() << "Inputs not specified. Zero is returned."; d->value = 0; } }
void dtkComposerNodeArrayScalarOperatorModifierAllDivide::run(void) { if (!d->receiver_array.isEmpty() && !d->receiver_value.isEmpty() ) { dtkContainerVectorReal *array = d->receiver_array.data(); if (!array) { dtkError() << "Input array is not defined."; d->emitter_array.clearData(); return; } qreal value = *d->receiver_value.data<qreal>(); if (value == 0) { dtkWarn() << "Value is zero. Same array is returned." ; } else { for(qlonglong i = 0 ; i < array->count(); ++i) (*array)[i] /= value ; } d->emitter_array.setData(array); } else { dtkWarn() << "Inputs not specified. Nothing is done"; d->emitter_array.clearData(); } }
void dtkComposerNodeArrayScalarOperatorDivide::run(void) { if (!d->receiver_array.isEmpty() && !d->receiver_index.isEmpty() && !d->receiver_value.isEmpty() ) { dtkContainerVectorReal *array = d->receiver_array.data(); if (!array) { dtkError() << "Input array is not defined."; d->emitter_array.clearData(); return; } qlonglong index = *d->receiver_index.data<qlonglong>(); qreal value = *d->receiver_value.data<qreal>(); if (index >= array->count()) { dtkWarn() << "index > size of the vector. Same array is returned."; } else { if (value == 0) dtkWarn() << "Value is zero. Same array is returned." ; else (*array)[index] /= value; } d->emitter_array.setData(array); } else { dtkWarn() << "Inputs not specified. Nothing is done"; d->emitter_array.clearData(); } }
void pHdf5IoDataModel::setCommunicator(dtkDistributedCommunicator *comm) { if(d->file_is_open) { qDebug() << " ERROR set the communicator before opening a file in pHdf5IoDataModel"; dtkError() <<" ERROR set the communicator before opening a file in pHdf5IoDataModel"; } else d->comm = comm; }
dtkDistributedServerDaemon::dtkDistributedServerDaemon(quint16 port, QObject *parent) : QTcpServer(parent), d(new dtkDistributedServerDaemonPrivate) { d->manager = NULL; if (!this->listen(QHostAddress::Any, port)) { dtkError() << "Can't listen on port" << port << ", aborting"; exit(1); } else { dtkDebug() << "OK, server is waiting for incoming connection on port" << port ; } dtkDistributedServiceBase::instance()->logMessage("Server daemon listening on port " + QString::number(port)); }
//hyperslab write void pHdf5IoDataModel::writeHyperslab(const QString& dataset_name, DataType type, quint64 *offset, quint64 *stride, quint64 *count, quint64 *block, quint64 *values_shape, void *values) { d->status=0; hid_t dataset_id = d->datasetId(dataset_name); //the selection within the file dataset's dataspace hid_t file_dataspace = H5Dget_space(dataset_id); if(H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, offset, stride, count, block)<0) { dtkError() << "ERROR selecting hyperslab" << dataset_name; } //set the dimensions of values. memory dataspace and the selection within it hid_t values_dataspace = H5Screate_simple(H5Sget_simple_extent_ndims(file_dataspace), values_shape, NULL); switch(type) { case dtkIoDataModel::Int: d->status = H5Dwrite(dataset_id, H5T_NATIVE_INT, values_dataspace, file_dataspace, d->prop_list_id, values); break; case dtkIoDataModel::LongLongInt: d->status = H5Dwrite(dataset_id, H5T_NATIVE_LLONG, values_dataspace, file_dataspace, d->prop_list_id, values); break; case dtkIoDataModel::Double: d->status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, values_dataspace, file_dataspace, d->prop_list_id, values); break; default: dtkError() << "write method: Datatype not supported"; }; if(d->status<0) { dtkError() << "error writing hyperslab" << dataset_name; } H5Sclose(file_dataspace); H5Sclose(values_dataspace); }
void medMessageController::showError(const QString& text,unsigned int timeout) { if ( dynamic_cast<QApplication *>(QCoreApplication::instance()) ) { // GUI medMessageError *message = new medMessageError( text,0,timeout); emit addMessage(message); } else { dtkError() << text; } }
void dtkComposerNodeCommunicatorUninitialize::run(void) { if (!d->receiver.isEmpty()) { dtkDistributedCommunicator *communicator = NULL; //FIXME:[migration] new transmitter requieres a clone method in object, even if dataTransmission is set to reference // communicator = d->receiver.data(); if (communicator) { communicator->uninitialize(); } else { dtkError() << "Input communicator not valid."; return; } } else { dtkWarn() << "Communicator not connected. Nothing is done"; } }
void dtkComposerNodeVectorRealExtractorSubVector::run(void) { if (!d->receiver_vector.isEmpty() && !d->receiver_from.isEmpty()) { dtkVectorReal *vector = d->receiver_vector.data(); if (!vector) { dtkError() << DTK_PRETTY_FUNCTION << "Input vector is not defined."; d->emitter_subvector.clearData(); return; } qlonglong from = *d->receiver_from.data(); if (from >= vector->size()) { dtkWarn() << DTK_PRETTY_FUNCTION << "Starting value from is greater than vector size:" << from << ">=" << vector->size(); d->emitter_subvector.clearData(); return; } qlonglong length = vector->size(); if (!d->receiver_length.isEmpty()) length = *d->receiver_length.data(); if (length < 0) { dtkWarn() << DTK_PRETTY_FUNCTION << "Length value is negative:" << length << "<" << 0; d->emitter_subvector.clearData(); return; } qlonglong to = qMin((from + length), static_cast<qlonglong>(vector->size())); qlonglong size = to - from; if (d->subvector->size() != size) d->subvector->allocate(size); for(qlonglong i = from, count = 0; i < to; ++i, ++count) (*d->subvector)[count] = (*vector)[i]; } else { dtkWarn() << DTK_PRETTY_FUNCTION << "Inputs not specified. Nothing is done"; d->emitter_subvector.clearData(); } }
void dtkComposerNodeCommunicatorSend::run(void) { d->emitter.setData(false); if (!d->receiver_data.isEmpty() && !d->receiver_target.isEmpty() ) { if (d->receiver_comm.isEmpty() && d->receiver_socket.isEmpty()) { dtkError() << "Send node at either a socket or a communicator."; return; } QVariant v = d->receiver_data.variant(); qint32 tag = 0; if (!d->receiver_tag.isEmpty()) tag = d->receiver_tag.data(); qlonglong target = d->receiver_target.data(); if (d->receiver_comm.isEmpty()) { d->socket = d->receiver_socket.constData(); d->communicator = NULL; //FIXME: we need the jobid QString jobid; dtkDistributedMessage *msg = new dtkDistributedMessage(dtkDistributedMessage::DATA, jobid, target, v); msg->addHeader("Tag",QString::number(tag)); msg->send(d->socket); } else { //FIXME:[migration] new transmitter requieres a clone method in object, even if dataTransmission is set to reference d->communicator = d->receiver_comm.constData(); d->socket = NULL; // FIXME: [migration] handle variant in send (should be ok with new distributed layer // d->communicator->send(v, target , tag); } d->emitter.setData(true); dtkTrace() << "send to target: " << target ; } else { dtkWarn() << "Inputs not specified in Send node. Nothing is done" ; } }
void hdf5IoDataModel::read(const QString &dataset_name, DataType type, void *values) { switch(type) { case dtkIoDataModel::Int: { H5Dread(d->datasetId(dataset_name), H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, values); break; } case dtkIoDataModel::LongLongInt: { H5Dread(d->datasetId(dataset_name), H5T_NATIVE_LLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, values); break; } case dtkIoDataModel::Double: { H5Dread(d->datasetId(dataset_name), H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, values); break; } default: dtkError() << "read method: Datatype not supported"; }; }
void dtkComposerNodeArrayScalarOperatorPrepend::run(void) { if (!d->receiver_array.isEmpty() && !d->receiver_value.isEmpty() ) { dtkContainerVectorReal *array = d->receiver_array.data(); if (!array) { dtkError() << "Input array is not defined."; d->emitter_array.clearData(); return; } qreal value = *d->receiver_value.data<qreal>(); array->prepend(value); d->emitter_array.setData(array); } else { dtkWarn() << "Inputs not specified. Nothing is done"; d->emitter_array.clearData(); } }
void dtkComposerNodeCommunicatorSize::run(void) { if (!d->receiver.isEmpty()) { dtkDistributedCommunicator *communicator = d->receiver.constData(); if (!communicator) { d->size = 0; dtkError() << "Input communicator not valid."; return; } d->size = communicator->size(); } else { dtkWarn() << "Communicator not connected. Nothing is done"; d->size = 0; } d->emitter.setData(d->size); }
void dtkComposerEvaluatorSlave::run(void) { d->status = 0; if ( !d->communicator_i) { d->communicator_i = dtkDistributed::communicator::instance(); if (d->communicator_i->rank() == 0) { std::cout << QString("DTK_JOBID="+this->jobId()).toStdString() << std::endl << std::flush; } } if (!d->factory) { dtkFatal() << "No factory set ! abort slave execution"; d->status = 1; return; } int rank = d->communicator_i->rank(); int size = d->communicator_i->size(); dtkDebug() << "communicator size is" << size; dtkDebug() << "our rank is" << rank; bool new_composition; if ( rank == 0) { QScopedPointer<dtkDistributedMessage> msg; if (!this->isConnected()) { dtkDebug() << "connect to server" << d->server; this->connect(d->server); if (this->isConnected()) { if (!d->composition_socket) { dtkDebug() << "open second socket to server" << d->server; d->composition_socket = new QTcpSocket; d->composition_socket->connectToHost(d->server.host(), d->server.port()); if (d->composition_socket->waitForConnected()) { msg.reset(new dtkDistributedMessage(dtkDistributedMessage::SETRANK,this->jobId(), dtkDistributedMessage::SLAVE_RANK )); msg->send(d->composition_socket); } else { dtkError() << "Can't connect to server"; d->status = 1; return; } } dtkDebug() << "connected, send our jobid to server" << this->jobId(); msg.reset(new dtkDistributedMessage(dtkDistributedMessage::SETRANK,this->jobId(),0)); msg->send(this->socket()); this->socket()->flush(); this->socket()->setParent(0); } else { dtkFatal() << "Can't connect to server" << d->server; d->status = 1; return; } } QString composition; dtkDebug() << "Wait for composition from controller " ; if (d->composition_socket->bytesAvailable() > 10) { dtkInfo() << "data already available, try to parse composition " << d->composition_socket->bytesAvailable(); } else if (!d->composition_socket->waitForReadyRead(600000)) { dtkFatal() << "No data received from server after 10mn, abort " ; d->status = 1; return; } else { dtkDebug() << "Ok, data received on composition socket, parse" ; } msg.reset(new dtkDistributedMessage); msg->parse(d->composition_socket); if (msg->type() == "xml") { new_composition = true; composition = QString(msg->content()); d->last_controller_rank = msg->header("x-forwarded-for").toInt(); d->composition_cache.insert(d->last_controller_rank, composition); } else if (msg->type() == "not-modified") { // reuse the old composition if (msg->header("x-forwarded-for").toInt() == d->last_controller_rank) { new_composition = false; } else { d->last_controller_rank = msg->header("x-forwarded-for").toInt(); dtkDebug() << "not modified, but from another controller" << d->last_controller_rank; new_composition = true; composition = d->composition_cache.value(d->last_controller_rank); } } else { dtkFatal() << "Bad composition type, abort" << msg->type() << msg->content(); d->status = 1; return; } if (new_composition && composition.isEmpty()) { dtkFatal() << "Empty composition, abort" ; d->status = 1; return; } dtkDebug() << "got composition from controller:" << composition; if (new_composition) { dtkDebug() << "new composition"; if (size > 1) { dtkDebug() << "send composition to our slaves"; for (int i=1; i< size; i++) { d->communicator_i->send(composition,i,0); } } dtkDebug() << "parse composition" ; d->reader->readString(composition); } else { dtkInfo() << "composition hasn't changed"; for (int i=1; i< size; i++) d->communicator_i->send(QString("rerun"),i,0); } if (new_composition) { if (dtkComposerNodeRemote *remote = dynamic_cast<dtkComposerNodeRemote *>(d->scene->root()->nodes().first()->wrapee())) { //FIXME: can we remove this ? // this->communicator()->setProperty("jobid",this->jobId()); remote->setSlave(this); remote->setJob(this->jobId()); remote->setCommunicator(d->communicator_i); } else { dtkFatal() << "Can't find remote node in composition, abort"; d->status = 1; return; } } dtkDebug() << "run composition" ; if (QThread::currentThread() == qApp->thread()) { dtkTrace() << "running on main thread, create a thread for the evaluator" ; QThread *workerThread = new QThread(this); QObject::connect(workerThread, SIGNAL(started()), d->evaluator, SLOT(run()), Qt::DirectConnection); QObject::connect(d->evaluator, SIGNAL(evaluationStopped()), workerThread, SLOT(quit())); QEventLoop loop; loop.connect(d->evaluator, SIGNAL(evaluationStopped()), &loop, SLOT(quit())); loop.connect(qApp, SIGNAL(aboutToQuit()), &loop, SLOT(quit())); this->socket()->moveToThread(workerThread); workerThread->start(); loop.exec(); workerThread->wait(); workerThread->deleteLater(); } else { dtkTrace() << "running on dedicated thread,run the evaluator" ; d->evaluator->run_static(); } dtkDebug() << "finished" ; } else { QString composition; d->communicator_i->receive(composition,0,0); if (composition != "rerun") { dtkDebug() << "new/changed composition, read" ; dtkDebug() << " composition is " << composition ; d->reader->readString(composition); dtkDebug() << "read done" ; } else { dtkDebug() << "reuse composition" ; } if (dtkComposerNodeRemote *remote = dynamic_cast<dtkComposerNodeRemote *>(d->scene->root()->nodes().first()->wrapee())) { remote->setSlave(this); remote->setJob(this->jobId()); remote->setCommunicator(d->communicator_i); dtkDebug() << "run composition" ; QThread *workerThread = new QThread(this); QObject::connect(workerThread, SIGNAL(started()), d->evaluator, SLOT(run()), Qt::DirectConnection); QObject::connect(d->evaluator, SIGNAL(evaluationStopped()), workerThread, SLOT(quit())); QEventLoop loop; loop.connect(d->evaluator, SIGNAL(evaluationStopped()), &loop, SLOT(quit())); loop.connect(qApp, SIGNAL(aboutToQuit()), &loop, SLOT(quit())); workerThread->start(); loop.exec(); workerThread->wait(); workerThread->deleteLater(); // d->evaluator->run_static(); dtkDebug() << "finished" ; } else { dtkFatal() << "Can't find remote node in composition, abort"; d->status = 1; return; } } }
int main(int argc, char **argv) { if(argc != 2) { qDebug() << "Usage: " << argv[0] << " <composition>"; return 0; } bool useGUI = false; QSettings settings("inria", "dtk"); settings.beginGroup("evaluator"); if (settings.contains("log_level")) dtkLogger::instance().setLevel(settings.value("log_level").toString()); else dtkLogger::instance().setLevel(dtkLog::Debug); if (settings.contains("use_gui") && settings.value("use_gui").toString() == "true") useGUI =true; QApplication application(argc, argv, useGUI); application.setApplicationName("dtkComposerEvaluator"); application.setApplicationVersion("0.0.2"); application.setOrganizationName("inria"); application.setOrganizationDomain("fr"); dtkLogger::instance().attachFile(dtkLogPath(&application)); dtkPluginManager::instance()->initialize(); dtkComposerScene *scene; dtkComposerStack *stack; dtkComposerGraph *graph; dtkComposerFactory *factory; dtkComposerEvaluator *evaluator; factory = new dtkComposerFactory; graph = new dtkComposerGraph; stack = new dtkComposerStack; scene = new dtkComposerScene; evaluator = new dtkComposerEvaluator; scene->setFactory(factory); scene->setStack(stack); scene->setGraph(graph); evaluator->setGraph(graph); dtkComposerReader *reader; reader = new dtkComposerReader; reader->setFactory(factory); reader->setScene(scene); reader->setGraph(graph); int index= 1; if (argv[1] == "-pg") { index = 2; evaluator->setProfiling(true); } if (!reader->read(argv[index])) { dtkError() << "read failure for " << argv[index]; return 1; } QObject::connect(evaluator,SIGNAL(evaluationStopped()),&application, SLOT(quit())); QtConcurrent::run(evaluator, &dtkComposerEvaluator::run_static, false); application.exec(); dtkPluginManager::instance()->uninitialize(); }
void dtkComposerNodeSpawn::begin(void) { if (!d->communicator) { // we are running on the controller, for the first time: // need to spawn processes if (d->size_receiver.isEmpty()) { d->np = QThread::idealThreadCount(); dtkInfo() << "Set number of spawned proceses to" << d->np; } else { d->np = d->size_receiver.data(); } d->policy.setNWorkers(d->np); // FIXME: don't use hardcoded plugin name d->policy.setType("qthreads"); d->communicator = d->policy.communicator(); // d->internal_comm = d->manager.spawn(); d->internal_comm_emitter.setData(d->internal_comm); d->rank = d->internal_comm->rank(); d->rank_emitter.setData(d->rank); if (!d->internal_comm) { dtkError() << "NULL internal communicator, spawn has failed !"; return; } dtkComposerEvaluatorProcess p; p.setInternalCommunicator(d->internal_comm); // p.setParentCommunicator(comm); // p.setFactory(factory); //FIXME: don't use manager // d->manager.exec(&p); } else { dtkTrace() << "communicator exists, no spawn"; } int first_transmitter = 3; if (d->is_parent) { int rank = dtkDistributedCommunicator::ROOT; if (d->current_hash != d->last_sent_hash){ // send sub-composition to rank 0 on remote node QByteArray compo = d->composition.toByteArray(); dtkDebug() << "running node remote begin statement on controller, send composition of size " << compo.size(); d->communicator->broadcast(compo, rank); d->last_sent_hash=d->current_hash; } else { dtkDebug() << "composition hash hasn't changed, send 'not-modified' to slave"; QByteArray data = QString("not-modified").toUtf8(); d->communicator->broadcast(data, rank); } // then send transmitters data int max = dtkComposerNodeComposite::receivers().count(); for (int i = first_transmitter; i < max; i++) { dtkComposerTransmitterReceiverVariant *t = dynamic_cast<dtkComposerTransmitterReceiverVariant *>(dtkComposerNodeComposite::receivers().at(i)); // FIXME: use our own transmitter variant list (see control nodes) QByteArray array; QDataStream stream(&array, QIODevice::WriteOnly); stream << t->variant(); dtkDebug() << "sending transmitter" << i << "of size" << array.size(); d->communicator->broadcast(array, rank); } } else if (d->communicator) { if (d->rank < 0) { dtkDebug() << "get rank/size on slave"; d->internal_comm_emitter.setData(d->internal_comm); d->rank = d->internal_comm->rank(); d->np = d->internal_comm->size(); dtkDebug() << "rank/size"<< d->rank << d->np; } dtkTrace() << "get transmitter data"; // running on the slave, receive data and set transmitters int max = dtkComposerNodeComposite::receivers().count(); for (int i = first_transmitter; i < max; i++) { dtkComposerTransmitterReceiverVariant *t = dynamic_cast<dtkComposerTransmitterReceiverVariant *>(dtkComposerNodeComposite::receivers().at(i)); QByteArray array; qint16 parent_rank = 0; d->communicator->broadcast(array, parent_rank); // t->setTwinned(false); QDataStream stream(&array, QIODevice::ReadOnly); QVariant variant; stream >> variant; // FIXME : how can we set data ? // t->setVariant(variant); // t->setTwinned(true); } } else {
// create a dataset or return the identifier if we already created it hid_t& hdf5IoDataModelPrivate::datasetId(const QString& dataset_name, const dtkIoDataModel::DataType& type=dtkIoDataModel::Int, const int& dim=-1, quint64 *shape=nullptr) { if(!file_is_open) { dtkError() << "file is not open! "; } if(!dataset_hash.contains(dataset_name)) { if(dim>0) { dtkInfo() << "Dataset not existing, creating DataSet" << dataset_name; QStringList dataset_name_split = dataset_name.split( "/" ); //erase first (root) and last (dataset) elements to only keep groups dataset_name_split.pop_front(); dataset_name_split.pop_back(); if ( dataset_name_split.length() > 0) { //the dataset is in a group, we need to open or create groups QString path = "" ; for(auto it: dataset_name_split) { path = path % "/" % it; if(!group_hash.contains(path)) { if(H5Lexists(file_id, path.toUtf8().constData(), H5P_DEFAULT)) //The group exists group_hash[path] = H5Gopen(file_id, path.toUtf8().constData(), H5P_DEFAULT); else group_hash[path] = H5Gcreate(file_id, path.toUtf8().constData(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } } } //now create the dataset hid_t dataspace = H5Screate_simple(dim, shape, NULL); switch(type) { case dtkIoDataModel::Int: { dataset_hash[dataset_name] = H5Dcreate(file_id, dataset_name.toUtf8().constData(), H5T_NATIVE_INT, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); break; } case dtkIoDataModel::LongLongInt: { dataset_hash[dataset_name] = H5Dcreate(file_id, dataset_name.toUtf8().constData(), H5T_NATIVE_LLONG, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); break; } case dtkIoDataModel::Double: { dataset_hash[dataset_name] = H5Dcreate(file_id, dataset_name.toUtf8().constData(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); break; } default: dtkError() <<" datatype not supported"; }; H5Sclose(dataspace); } else { //we just want to open it dataset_hash[dataset_name] = H5Dopen(file_id, dataset_name.toUtf8().constData(), H5P_DEFAULT); } } //dataset already opened return dataset_hash[dataset_name]; }
void dtkDistributedSlave::error(QAbstractSocket::SocketError error) { switch(error) { case QAbstractSocket::ConnectionRefusedError: dtkError() << "The connection was refused by the peer (or timed out)."; break; case QAbstractSocket::RemoteHostClosedError: dtkError() << "The remote host closed the connection. Note that the slave socket (i.e., this socket) will be closed after the remote close notification has been sent."; break; case QAbstractSocket::HostNotFoundError: dtkError() << "The host address was not found."; break; case QAbstractSocket::SocketAccessError: dtkError() << "The socket operation failed because the application lacked the required privileges."; break; case QAbstractSocket::SocketResourceError: dtkError() << "The local system ran out of resources (e.g., too many sockets)."; break; case QAbstractSocket::SocketTimeoutError: dtkError() << "The socket operation timed out."; break; case QAbstractSocket::DatagramTooLargeError: dtkError() << "The datagram was larger than the operating system's limit (which can be as low as 8192 bytes)."; break; case QAbstractSocket::NetworkError: dtkError() << "An error occurred with the network (e.g., the network cable was accidentally plugged out)."; break; case QAbstractSocket::AddressInUseError: dtkError() << "The address specified to QUdpSocket::bind() is already in use and was set to be exclusive."; break; case QAbstractSocket::SocketAddressNotAvailableError: dtkError() << "The address specified to QUdpSocket::bind() does not belong to the host."; break; case QAbstractSocket::UnsupportedSocketOperationError: dtkError() << "The requested socket operation is not supported by the local operating system (e.g., lack of IPv6 support)."; break; case QAbstractSocket::ProxyAuthenticationRequiredError: dtkError() << "The socket is using a proxy, and the proxy requires authentication."; break; case QAbstractSocket::SslHandshakeFailedError: dtkError() << "The SSL/TLS handshake failed, so the connection was closed (only used in QSslSocket)"; break; case QAbstractSocket::UnfinishedSocketOperationError: dtkError() << "Used by QAbstractSocketEngine only, The last operation attempted has not finished yet (still in progress in the background)."; break; case QAbstractSocket::ProxyConnectionRefusedError: dtkError() << "Could not contact the proxy server because the connection to that server was denied"; break; case QAbstractSocket::ProxyConnectionClosedError: dtkError() << "The connection to the proxy server was closed unexpectedly (before the connection to the final peer was established)"; break; case QAbstractSocket::ProxyConnectionTimeoutError: dtkError() << "The connection to the proxy server timed out or the proxy server stopped responding in the authentication phase."; break; case QAbstractSocket::ProxyNotFoundError: dtkError() << "The proxy address set with setProxy() (or the application proxy) was not found."; break; case QAbstractSocket::ProxyProtocolError: dtkError() << "The connection negotiation with the proxy server because the response from the proxy server could not be understood."; break; case QAbstractSocket::UnknownSocketError: dtkError() << "An unidentified error occurred."; break; default: break; } }
void vtkLogForwarder::DisplayErrorText( const char *txt ) { dtkError() << txt; }
void dtkComposerNodeCommunicatorReceive::run(void) { if (!d->receiver_source.isEmpty()) { if (d->receiver_comm.isEmpty() && d->receiver_socket.isEmpty()) { dtkError() << "Send node at either a socket or a communicator."; return; } d->source = d->receiver_source.data(); d->tag = 0; if (!d->receiver_tag.isEmpty()) d->tag = d->receiver_tag.data(); d->emitter.clearData(); if (!d->receiver_data.isEmpty()) { d->emitter.setData(d->receiver_data.variant()); } if (d->receiver_comm.isEmpty()) { QTcpSocket *socket = d->receiver_socket.constData(); dtkDebug() << "TCP communicator. Parse message from socket, waiting for tag" << d->tag; if (d->msg_map.contains(d->tag)) { dtkDebug() << "msg already received for tag" << d->tag; // d->emitter.setTwinned(false); dtkDistributedMessage *msg = d->msg_map.take(d->tag); d->emitter.setData(msg->content()); // d->emitter.setTwinned(true); delete msg; return; } else { dtkTrace() << "msg not yet received, wait for data"; } socket->blockSignals(true); // needed ? if (!socket->waitForReadyRead(300000)) { dtkWarn() << "Data not ready in receive for rank " << d->source; } else { dtkDistributedMessage msg; msg.parse(socket); qlonglong msg_tag = msg.header("Tag").toLongLong(); if (msg_tag == d->tag || d->tag == dtkDistributedCommunicator::ANY_TAG) { dtkTrace() << "OK, this is the expected tag " << d->tag; // d->emitter.setTwinned(false); d->emitter.setData(msg.content()); // d->emitter.setTwinned(true); if (d->tag == dtkDistributedCommunicator::ANY_TAG) d->tag = msg_tag; } else { //store msg for another call with the right tag dtkInfo() << "Msg received, but wrong tag, store the msg" << d->tag << msg_tag; d->msg_map.insert(msg_tag, &msg); this->run(); // do it again } } socket->blockSignals(false); // needed ? } else { // MPI QByteArray array; dtkDistributedCommunicatorStatus status; dtkDistributedCommunicator *communicator = d->receiver_comm.constData(); //FIXME:[migration] new transmitter requieres a clone method in object, even if dataTransmission is set to reference communicator->receive(array, d->source, d->tag, status); if (d->tag == dtkDistributedCommunicator::ANY_TAG) d->tag = status.tag(); if (d->source == dtkDistributedCommunicator::ANY_SOURCE) d->source = status.source(); if (!array.isEmpty()) { // d->emitter.setTwinned(false); d->emitter.setData(array); // d->emitter.setTwinned(true); } else { dtkWarn() << "Empty data in receive"; d->emitter.clearData(); } } } else { d->emitter.clearData(); dtkWarn() << "Inputs not specified in receive. Nothing is done"; d->tag = 0; d->source = 0; } d->emitter_source.setData(d->source); d->emitter_tag.setData(d->tag); }