bool Port::saveStreams(QString fileName, QString fileType, QString &error)
{
bool ret = false;
QProgressDialog progress("Saving Streams", "Cancel", 0, 0, mainWindow);
AbstractFileFormat *fmt = AbstractFileFormat::fileFormatFromType(fileType);
OstProto::StreamConfigList streams;
if (fmt == NULL)
goto _fail;
progress.setAutoReset(false);
progress.setAutoClose(false);
progress.setMinimumDuration(0);
progress.show();
mainWindow->setDisabled(true);
progress.setEnabled(true); // to override the mainWindow disable
connect(fmt, SIGNAL(status(QString)),&progress,SLOT(setLabelText(QString)));
connect(fmt, SIGNAL(target(int)), &progress, SLOT(setMaximum(int)));
connect(fmt, SIGNAL(progress(int)), &progress, SLOT(setValue(int)));
connect(&progress, SIGNAL(canceled()), fmt, SLOT(cancel()));
progress.setLabelText("Preparing Streams...");
progress.setRange(0, mStreams.size());
streams.mutable_port_id()->set_id(0);
for (int i = 0; i < mStreams.size(); i++)
{
OstProto::Stream *s = streams.add_stream();
mStreams[i]->protoDataCopyInto(*s);
if (progress.wasCanceled())
goto _user_cancel;
progress.setValue(i);
if (i % 32 == 0)
qApp->processEvents();
}
fmt->saveStreamsOffline(streams, fileName, error);
qDebug("after save offline");
while (!fmt->isFinished())
qApp->processEvents();
qDebug("wait over for offline operation");
ret = fmt->result();
goto _exit;
_user_cancel:
goto _exit;
_fail:
error = QString("Unsupported File Type - %1").arg(fileType);
goto _exit;
_exit:
progress.close();
mainWindow->setEnabled(true);
return ret;
}
void PortGroup::when_configApply(int portIndex)
{
OstProto::StreamIdList *streamIdList;
OstProto::StreamConfigList *streamConfigList;
OstProto::Ack *ack;
PbRpcController *controller;
Q_ASSERT(portIndex < mPorts.size());
if (state() != QAbstractSocket::ConnectedState)
return;
QApplication::setOverrideCursor(QCursor(Qt::WaitCursor));
mainWindow->setDisabled(true);
qDebug("applying 'deleted streams' ...");
streamIdList = new OstProto::StreamIdList;
ack = new OstProto::Ack;
controller = new PbRpcController(streamIdList, ack);
streamIdList->mutable_port_id()->set_id(mPorts[portIndex]->id());
mPorts[portIndex]->getDeletedStreamsSinceLastSync(*streamIdList);
serviceStub->deleteStream(controller, streamIdList, ack,
NewCallback(this, &PortGroup::processDeleteStreamAck, controller));
qDebug("applying 'new streams' ...");
streamIdList = new OstProto::StreamIdList;
ack = new OstProto::Ack;
controller = new PbRpcController(streamIdList, ack);
streamIdList->mutable_port_id()->set_id(mPorts[portIndex]->id());
mPorts[portIndex]->getNewStreamsSinceLastSync(*streamIdList);
serviceStub->addStream(controller, streamIdList, ack,
NewCallback(this, &PortGroup::processAddStreamAck, controller));
qDebug("applying 'modified streams' ...");
streamConfigList = new OstProto::StreamConfigList;
ack = new OstProto::Ack;
controller = new PbRpcController(streamConfigList, ack);
streamConfigList->mutable_port_id()->set_id(mPorts[portIndex]->id());
mPorts[portIndex]->getModifiedStreamsSinceLastSync(*streamConfigList);
serviceStub->modifyStream(controller, streamConfigList, ack,
NewCallback(this, &PortGroup::processModifyStreamAck,
portIndex, controller));
}
void Port::getModifiedStreamsSinceLastSync(
OstProto::StreamConfigList &streamConfigList)
{
qDebug("In %s", __FUNCTION__);
//streamConfigList.mutable_port_id()->set_id(mPortId);
for (int i = 0; i < mStreams.size(); i++)
{
OstProto::Stream *s;
s = streamConfigList.add_stream();
mStreams[i]->protoDataCopyInto(*s);
}
qDebug("Done %s", __FUNCTION__);
}
void PortGroup::processStreamConfigList(int portIndex,
PbRpcController *controller)
{
OstProto::StreamConfigList *streamConfigList
= static_cast<OstProto::StreamConfigList*>(controller->response());
qDebug("In %s", __PRETTY_FUNCTION__);
Q_ASSERT(portIndex < numPorts());
if (controller->Failed())
{
qDebug("%s: rpc failed(%s)", __FUNCTION__,
qPrintable(controller->ErrorString()));
goto _exit;
}
Q_ASSERT(portIndex < numPorts());
if (streamConfigList->port_id().id() != mPorts[portIndex]->id())
{
qDebug("Invalid portId %d (expected %d) received for portIndex %d",
streamConfigList->port_id().id(), mPorts[portIndex]->id(), portIndex);
goto _exit;
}
for(int i = 0; i < streamConfigList->stream_size(); i++)
{
uint streamId;
streamId = streamConfigList->stream(i).stream_id().id();
mPorts[portIndex]->updateStream(streamId,
streamConfigList->mutable_stream(i));
}
// Are we done for all ports?
if (portIndex >= numPorts())
{
// FIXME(HI): some way to reset streammodel
}
_exit:
delete controller;
}
bool PythonFileFormat::save(const OstProto::StreamConfigList streams,
const QString fileName, QString &error)
{
QFile file(fileName);
QTextStream out(&file);
QSet<QString> imports;
if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate))
goto _open_fail;
// import standard modules
emit status("Writing imports ...");
emit target(0);
writeStandardImports(out);
emit target(streams.stream_size());
// import protocols from respective modules
// build the import list using a QSet to eliminate duplicates
for (int i = 0; i < streams.stream_size(); i++) {
const OstProto::Stream &stream = streams.stream(i);
for (int j = 0 ; j < stream.protocol_size(); j++) {
const OstProto::Protocol &protocol = stream.protocol(j);
const Reflection *refl = protocol.GetReflection();
std::vector<const FieldDescriptor*> fields;
refl->ListFields(protocol, &fields);
for (uint k = 0; k < fields.size(); k++) {
// skip non extension fields
if (!fields.at(k)->is_extension())
continue;
if (fields.at(k)->file()->name() !=
fields.at(k)->message_type()->file()->name()) {
imports.insert(
QString("%1 import %2").arg(
QString(fields.at(k)->message_type()
->file()->name().c_str())
.replace(".proto", "_pb2"),
fields.at(k)->message_type()->name().c_str()));
imports.insert(
QString("%1 import %2").arg(
QString(fields.at(k)
->file()->name().c_str())
.replace(".proto", "_pb2"),
fields.at(k)->name().c_str()));
}
else {
imports.insert(
QString("%1 import %2, %3").arg(
QString(fields.at(k)->file()->name().c_str())
.replace(".proto", "_pb2"),
fields.at(k)->message_type()->name().c_str(),
fields.at(k)->name().c_str()));
}
}
}
emit progress(i);
}
// write the import statements
out << "# import ostinato modules\n";
out << "from ostinato.core import DroneProxy, ost_pb\n";
foreach (QString str, imports)
out << "from ostinato.protocols." << str << "\n";
out << "\n";
// start of script - init, connect to drone etc.
emit status("Writing prologue ...");
emit target(0);
writePrologue(out);
// Add streams
emit status("Writing stream adds ...");
emit target(streams.stream_size());
out << " # ------------#\n";
out << " # add streams #\n";
out << " # ------------#\n";
out << " stream_id = ost_pb.StreamIdList()\n";
out << " stream_id.port_id.id = tx_port_number\n";
for (int i = 0; i < streams.stream_size(); i++) {
out << " stream_id.stream_id.add().id = "
<< streams.stream(i).stream_id().id() << "\n";
emit progress(i);
}
out << " drone.addStream(stream_id)\n";
out << "\n";
// Configure streams with actual values
emit status("Writing stream configuration ...");
emit target(streams.stream_size());
out << " # ------------------#\n";
out << " # configure streams #\n";
out << " # ------------------#\n";
out << " stream_cfg = ost_pb.StreamConfigList()\n";
out << " stream_cfg.port_id.id = tx_port_number\n";
for (int i = 0; i < streams.stream_size(); i++) {
const OstProto::Stream &stream = streams.stream(i);
const Reflection *refl;
std::vector<const FieldDescriptor*> fields;
out << "\n";
out << " # stream " << stream.stream_id().id() << " "
<< stream.core().name().c_str() << "\n";
out << " s = stream_cfg.stream.add()\n";
out << " s.stream_id.id = "
<< stream.stream_id().id() << "\n";
// Stream Core values
refl = stream.core().GetReflection();
refl->ListFields(stream.core(), &fields);
for (uint j = 0; j < fields.size(); j++) {
writeFieldAssignment(out, QString(" s.core.")
.append(fields.at(j)->name().c_str()),
stream.core(), refl, fields.at(j));
}
// Stream Control values
refl = stream.control().GetReflection();
refl->ListFields(stream.control(), &fields);
for (uint j = 0; j < fields.size(); j++) {
writeFieldAssignment(out, QString(" s.control.")
.append(fields.at(j)->name().c_str()),
stream.control(), refl, fields.at(j));
}
// Protocols
for (int j = 0 ; j < stream.protocol_size(); j++) {
const OstProto::Protocol &protocol = stream.protocol(j);
out << "\n"
<< " p = s.protocol.add()\n"
<< " p.protocol_id.id = "
<< QString(OstProto::Protocol_k_descriptor()
->FindValueByNumber(protocol.protocol_id().id())
->full_name().c_str())
.replace("OstProto", "ost_pb");
out << "\n";
refl = protocol.GetReflection();
refl->ListFields(protocol, &fields);
for (uint k = 0; k < fields.size(); k++) {
// skip protocol_id field
if (fields.at(k)->number() ==
OstProto::Protocol::kProtocolIdFieldNumber)
continue;
QString pfx(" p.Extensions[X]");
pfx.replace(fields.at(k)->is_extension()? "X": "Extensions[X]",
fields.at(k)->name().c_str());
writeFieldAssignment(out, pfx, protocol,
refl, fields.at(k));
}
}
emit progress(i);
}
out << "\n";
out << " drone.modifyStream(stream_cfg)\n";
// end of script - transmit streams, disconnect from drone etc.
emit status("Writing epilogue ...");
emit target(0);
writeEpilogue(out);
out.flush();
file.close();
return true;
_open_fail:
error = QString(tr("Error opening %1 (Error Code = %2)"))
.arg(fileName)
.arg(file.error());
return false;
}
bool FileFormat::openStreams(const QString fileName,
OstProto::StreamConfigList &streams, QString &error)
{
QFile file(fileName);
QByteArray buf;
int size, contentOffset, contentSize;
quint32 calcCksum;
OstProto::FileMagic magic;
OstProto::FileMeta meta;
OstProto::FileContent content;
OstProto::FileChecksum cksum, zeroCksum;
if (!file.open(QIODevice::ReadOnly))
goto _open_fail;
if (file.size() < kFileMagicSize)
goto _magic_missing;
if (file.size() < kFileMinSize)
goto _checksum_missing;
buf.resize(file.size());
size = file.read(buf.data(), buf.size());
if (size < 0)
goto _read_fail;
Q_ASSERT(file.atEnd());
file.close();
qDebug("%s: file.size() = %lld", __FUNCTION__, file.size());
qDebug("%s: size = %d", __FUNCTION__, size);
//qDebug("Read %d bytes", buf.size());
//qDebug("%s", QString(buf.toHex()).toAscii().constData());
// Parse and verify magic
if (!magic.ParseFromArray(
(void*)(buf.constData() + kFileMagicOffset),
kFileMagicSize))
{
goto _magic_parse_fail;
}
if (magic.value() != kFileMagicValue)
goto _magic_match_fail;
// Parse and verify checksum
if (!cksum.ParseFromArray(
(void*)(buf.constData() + size - kFileChecksumSize),
kFileChecksumSize))
{
goto _cksum_parse_fail;
}
zeroCksum.set_value(0);
if (!zeroCksum.SerializeToArray(
(void*) (buf.data() + size - kFileChecksumSize),
kFileChecksumSize))
{
goto _zero_cksum_serialize_fail;
}
calcCksum = checksumCrc32C((quint8*) buf.constData(), size);
qDebug("checksum \nExpected:%x Actual:%x",
calcCksum, cksum.value());
if (cksum.value() != calcCksum)
goto _cksum_verify_fail;
// Parse the metadata first before we parse the full contents
if (!meta.ParseFromArray(
(void*)(buf.constData() + kFileMetaDataOffset),
size - kFileMetaDataOffset))
{
goto _metadata_parse_fail;
}
qDebug("%s: File MetaData (INFORMATION) - \n%s", __FUNCTION__,
QString().fromStdString(meta.DebugString()).toAscii().constData());
// MetaData Validation(s)
if (meta.data().file_type() != OstProto::kStreamsFileType)
goto _unexpected_file_type;
if (meta.data().format_version_major() != kFileFormatVersionMajor)
goto _incompatible_file_version;
if (meta.data().format_version_minor() > kFileFormatVersionMinor)
goto _incompatible_file_version;
if (meta.data().format_version_minor() < kFileFormatVersionMinor)
{
// TODO: need to modify 'buf' such that we can parse successfully
// assuming the native minor version
}
if (meta.data().format_version_revision() > kFileFormatVersionRevision)
{
error = QString(tr("%1 was created using a newer version of Ostinato."
" New features/protocols will not be available.")).arg(fileName);
}
Q_ASSERT(meta.data().format_version_major() == kFileFormatVersionMajor);
// ByteSize() does not include the Tag/Key, so we add 2 for that
contentOffset = kFileMetaDataOffset + meta.data().ByteSize() + 2;
contentSize = size - contentOffset - kFileChecksumSize;
// Parse full contents
if (!content.ParseFromArray(
(void*)(buf.constData() + contentOffset),
contentSize))
{
goto _content_parse_fail;
}
if (!content.matter().has_streams())
goto _missing_streams;
postParseFixup(meta.data(), content);
streams.CopyFrom(content.matter().streams());
return true;
_missing_streams:
error = QString(tr("%1 does not contain any streams")).arg(fileName);
goto _fail;
_content_parse_fail:
error = QString(tr("Failed parsing %1 contents")).arg(fileName);
qDebug("Error: %s", QString().fromStdString(
content.matter().InitializationErrorString())
.toAscii().constData());
qDebug("Debug: %s", QString().fromStdString(
content.matter().DebugString()).toAscii().constData());
goto _fail;
_incompatible_file_version:
error = QString(tr("%1 is in an incompatible format version - %2.%3.%4"
" (Native version is %5.%6.%7)"))
.arg(fileName)
.arg(meta.data().format_version_major())
.arg(meta.data().format_version_minor())
.arg(meta.data().format_version_revision())
.arg(kFileFormatVersionMajor)
.arg(kFileFormatVersionMinor)
.arg(kFileFormatVersionRevision);
goto _fail;
_unexpected_file_type:
error = QString(tr("%1 is not a streams file")).arg(fileName);
goto _fail;
_metadata_parse_fail:
error = QString(tr("Failed parsing %1 meta data")).arg(fileName);
qDebug("Error: %s", QString().fromStdString(
meta.data().InitializationErrorString())
.toAscii().constData());
goto _fail;
_cksum_verify_fail:
error = QString(tr("%1 checksum validation failed!\nExpected:%2 Actual:%3"))
.arg(fileName)
.arg(calcCksum, 0, kBaseHex)
.arg(cksum.value(), 0, kBaseHex);
goto _fail;
_zero_cksum_serialize_fail:
error = QString(tr("Internal Error: Zero Checksum Serialize failed!\n"
"Error: %1\nDebug: %2"))
.arg(QString().fromStdString(
cksum.InitializationErrorString()))
.arg(QString().fromStdString(cksum.DebugString()));
goto _fail;
_cksum_parse_fail:
error = QString(tr("Failed parsing %1 checksum")).arg(fileName);
qDebug("Error: %s", QString().fromStdString(
cksum.InitializationErrorString())
.toAscii().constData());
goto _fail;
_magic_match_fail:
error = QString(tr("%1 is not an Ostinato file")).arg(fileName);
goto _fail;
_magic_parse_fail:
error = QString(tr("%1 does not look like an Ostinato file")).arg(fileName);
qDebug("Error: %s", QString().fromStdString(
magic.InitializationErrorString())
.toAscii().constData());
goto _fail;
_read_fail:
error = QString(tr("Error reading from %1")).arg(fileName);
goto _fail;
_checksum_missing:
error = QString(tr("%1 is too small (missing checksum)")).arg(fileName);
goto _fail;
_magic_missing:
error = QString(tr("%1 is too small (missing magic value)"))
.arg(fileName);
goto _fail;
_open_fail:
error = QString(tr("Error opening %1")).arg(fileName);
goto _fail;
_fail:
qDebug("%s", error.toAscii().constData());
return false;
}
bool FileFormat::saveStreams(const OstProto::StreamConfigList streams,
const QString fileName, QString &error)
{
OstProto::FileMagic magic;
OstProto::FileMeta meta;
OstProto::FileContent content;
OstProto::FileChecksum cksum;
QFile file(fileName);
int metaSize, contentSize;
int contentOffset, cksumOffset;
QByteArray buf;
quint32 calcCksum;
magic.set_value(kFileMagicValue);
Q_ASSERT(magic.IsInitialized());
cksum.set_value(0);
Q_ASSERT(cksum.IsInitialized());
initFileMetaData(*(meta.mutable_data()));
meta.mutable_data()->set_file_type(OstProto::kStreamsFileType);
Q_ASSERT(meta.IsInitialized());
if (!streams.IsInitialized())
goto _stream_not_init;
content.mutable_matter()->mutable_streams()->CopyFrom(streams);
Q_ASSERT(content.IsInitialized());
metaSize = meta.ByteSize();
contentSize = content.ByteSize();
contentOffset = kFileMetaDataOffset + metaSize;
cksumOffset = contentOffset + contentSize;
Q_ASSERT(magic.ByteSize() == kFileMagicSize);
Q_ASSERT(cksum.ByteSize() == kFileChecksumSize);
buf.resize(kFileMagicSize + metaSize + contentSize + kFileChecksumSize);
// Serialize everything
if (!magic.SerializeToArray((void*) (buf.data() + kFileMagicOffset),
kFileMagicSize))
{
goto _magic_serialize_fail;
}
if (!meta.SerializeToArray((void*) (buf.data() + kFileMetaDataOffset),
metaSize))
{
goto _meta_serialize_fail;
}
if (!content.SerializeToArray((void*) (buf.data() + contentOffset),
contentSize))
{
goto _content_serialize_fail;
}
if (!cksum.SerializeToArray((void*) (buf.data() + cksumOffset),
kFileChecksumSize))
{
goto _zero_cksum_serialize_fail;
}
emit status("Calculating checksum...");
// Calculate and write checksum
calcCksum = checksumCrc32C((quint8*)buf.constData(), buf.size());
cksum.set_value(calcCksum);
if (!cksum.SerializeToArray(
(void*) (buf.data() + cksumOffset),
kFileChecksumSize))
{
goto _cksum_serialize_fail;
}
qDebug("Writing %d bytes", buf.size());
//qDebug("%s", QString(buf.toHex()).toAscii().constData());
emit status("Writing to disk...");
if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate))
goto _open_fail;
if (file.write(buf) < 0)
goto _write_fail;
file.close();
return true;
_write_fail:
error = QString(tr("Error writing to %1")).arg(fileName);
goto _fail;
_open_fail:
error = QString(tr("Error opening %1 (Error Code = %2)"))
.arg(fileName)
.arg(file.error());
goto _fail;
_cksum_serialize_fail:
error = QString(tr("Internal Error: Checksum Serialize failed\n%1\n%2"))
.arg(QString().fromStdString(
cksum.InitializationErrorString()))
.arg(QString().fromStdString(cksum.DebugString()));
goto _fail;
_zero_cksum_serialize_fail:
error = QString(tr("Internal Eror: Zero Checksum Serialize failed\n%1\n%2"))
.arg(QString().fromStdString(
cksum.InitializationErrorString()))
.arg(QString().fromStdString(cksum.DebugString()));
goto _fail;
_content_serialize_fail:
error = QString(tr("Internal Error: Content Serialize failed\n%1\n%2"))
.arg(QString().fromStdString(
content.InitializationErrorString()))
.arg(QString().fromStdString(content.DebugString()));
goto _fail;
_meta_serialize_fail:
error = QString(tr("Internal Error: Meta Data Serialize failed\n%1\n%2"))
.arg(QString().fromStdString(
meta.InitializationErrorString()))
.arg(QString().fromStdString(meta.DebugString()));
goto _fail;
_magic_serialize_fail:
error = QString(tr("Internal Error: Magic Serialize failed\n%1\n%2"))
.arg(QString().fromStdString(
magic.InitializationErrorString()))
.arg(QString().fromStdString(magic.DebugString()));
goto _fail;
_stream_not_init:
error = QString(tr("Internal Error: Streams not initialized\n%1\n%2"))
.arg(QString().fromStdString(
streams.InitializationErrorString()))
.arg(QString().fromStdString(streams.DebugString()));
goto _fail;
_fail:
qDebug("%s", error.toAscii().constData());
return false;
}
bool Port::openStreams(QString fileName, bool append, QString &error)
{
bool ret = false;
QDialog *optDialog;
QProgressDialog progress("Opening Streams", "Cancel", 0, 0, mainWindow);
OstProto::StreamConfigList streams;
AbstractFileFormat *fmt = AbstractFileFormat::fileFormatFromFile(fileName);
if (fmt == NULL)
goto _fail;
if ((optDialog = fmt->openOptionsDialog()))
{
int ret;
optDialog->setParent(mainWindow, Qt::Dialog);
ret = optDialog->exec();
optDialog->setParent(0, Qt::Dialog);
if (ret == QDialog::Rejected)
goto _user_opt_cancel;
}
progress.setAutoReset(false);
progress.setAutoClose(false);
progress.setMinimumDuration(0);
progress.show();
mainWindow->setDisabled(true);
progress.setEnabled(true); // to override the mainWindow disable
connect(fmt, SIGNAL(status(QString)),&progress,SLOT(setLabelText(QString)));
connect(fmt, SIGNAL(target(int)), &progress, SLOT(setMaximum(int)));
connect(fmt, SIGNAL(progress(int)), &progress, SLOT(setValue(int)));
connect(&progress, SIGNAL(canceled()), fmt, SLOT(cancel()));
fmt->openStreamsOffline(fileName, streams, error);
qDebug("after open offline");
while (!fmt->isFinished())
qApp->processEvents();
qDebug("wait over for offline operation");
if (!fmt->result())
goto _fail;
// process any remaining events posted from the thread
for (int i = 0; i < 10; i++)
qApp->processEvents();
if (!append)
{
int n = numStreams();
progress.setLabelText("Deleting existing streams...");
progress.setRange(0, n);
for (int i = 0; i < n; i++)
{
if (progress.wasCanceled())
goto _user_cancel;
deleteStreamAt(0);
progress.setValue(i);
if (i % 32 == 0)
qApp->processEvents();
}
}
progress.setLabelText("Constructing new streams...");
progress.setRange(0, streams.stream_size());
for (int i = 0; i < streams.stream_size(); i++)
{
if (progress.wasCanceled())
goto _user_cancel;
newStreamAt(mStreams.size(), &streams.stream(i));
progress.setValue(i);
if (i % 32 == 0)
qApp->processEvents();
}
_user_cancel:
emit streamListChanged(mPortGroupId, mPortId);
_user_opt_cancel:
ret = true;
_fail:
progress.close();
mainWindow->setEnabled(true);
recalculateAverageRates();
return ret;
}
