void OGRGDALImportExtension::runPrecheck()
{
if (mp_PrecheckImportDlg != NULL)
delete mp_PrecheckImportDlg;
mp_PrecheckImportDlg = new PrecheckImportDialog(OGRGDALEXT_PRECHECK_STEPS,this);
mp_PrecheckImportDlg->open();
QThread* WThread = new QThread;
PrecheckWorker* Worker = new PrecheckWorker(m_SourcesInfos,mp_AdvancedDesc);
Worker->moveToThread(WThread);
connect(Worker, SIGNAL(stepEntered(QString)), mp_PrecheckImportDlg, SLOT(handleStepEntered(QString)));
connect(Worker, SIGNAL(stepCompleted(int,QString)), mp_PrecheckImportDlg, SLOT(handleStepCompleted(int,QString)));
connect(Worker, SIGNAL(completed(QString)), mp_PrecheckImportDlg, SLOT(handleCompleted(QString)));
connect(Worker, SIGNAL(finished()), mp_PrecheckImportDlg, SLOT(handleFinished()));
connect(WThread, SIGNAL(started()), Worker, SLOT(run()));
connect(Worker, SIGNAL(finished()), WThread, SLOT(quit()));
connect(Worker, SIGNAL(finished()), Worker, SLOT(deleteLater()));
connect(WThread, SIGNAL(finished()), WThread, SLOT(deleteLater()));
WThread->start();
}
void OGRGDALImportExtension::proceedToImport()
{
if (mp_PrecheckImportDlg != NULL)
delete mp_PrecheckImportDlg;
mp_PrecheckImportDlg = new PrecheckImportDialog(OGRGDALEXT_PRECHECK_STEPS+m_SourcesInfos.size()+2,this);
mp_PrecheckImportDlg->open();
QThread* WThread = new QThread;
ImportWorker* Worker = new ImportWorker(m_SourcesInfos,mp_AdvancedDesc,m_InputDir);
Worker->moveToThread(WThread);
connect(Worker, SIGNAL(stepEntered(QString)), mp_PrecheckImportDlg, SLOT(handleStepEntered(QString)));
connect(Worker, SIGNAL(stepCompleted(int,QString)), mp_PrecheckImportDlg, SLOT(handleStepCompleted(int,QString)));
connect(Worker, SIGNAL(completed(QString)), mp_PrecheckImportDlg, SLOT(handleCompleted(QString)));
connect(Worker, SIGNAL(finished()), mp_PrecheckImportDlg, SLOT(handleFinished()));
// for automatic closing on successful import
connect(Worker, SIGNAL(closeRequired()), mp_PrecheckImportDlg, SLOT(handleCloseRequired()));
connect(mp_PrecheckImportDlg, SIGNAL(closeRequired()), this, SLOT(handleCloseRequired()));
connect(WThread, SIGNAL(started()), Worker, SLOT(run()));
connect(Worker, SIGNAL(finished()), WThread, SLOT(quit()));
connect(Worker, SIGNAL(finished()), Worker, SLOT(deleteLater()));
connect(WThread, SIGNAL(finished()), WThread, SLOT(deleteLater()));
WThread->start();
}
bool ImportWorker::importLayer(int Step,int Index)
{
emit stepEntered(tr("Importing layer \"%1\"...").arg(m_SourcesInfos[Index].LayerName));
QString UnitsClass = m_SourcesInfos[Index].UnitsClass;
bool AtttrsCreated = false;
QMapIterator<int, SourceUnit> It(m_SourcesData[Index].Units);
while (It.hasNext())
{
It.next();
openfluid::fluidx::SpatialUnitDescriptor* UDesc = new openfluid::fluidx::SpatialUnitDescriptor();
UDesc->setUnitsClass(UnitsClass.toStdString());
UDesc->setID(It.key());
UDesc->setProcessOrder(It.value().ProcessOrder);
mp_AdvDesc->spatialDomain().addUnit(UDesc);
QMapIterator<QString,QString> AttrsIt(It.value().Attributes);
// create attributes on first unit of the units class
if (!AtttrsCreated)
{
while (AttrsIt.hasNext())
{
AttrsIt.next();
mp_AdvDesc->spatialDomain().addAttribute(UnitsClass.toStdString(),
AttrsIt.key().toStdString(),"-");
}
AttrsIt.toFront();
AtttrsCreated = true;
}
// set attributes values for the current imported unit
while (AttrsIt.hasNext())
{
AttrsIt.next();
mp_AdvDesc->spatialDomain().attribute(UnitsClass.toStdString(),
It.key(),
AttrsIt.key().toStdString()) = AttrsIt.value().toStdString();
}
}
emit stepCompleted(Step,getStyledText(tr("[OK]"),"green"));
return true;
}
bool DataProcessingWorker::checkConnectivity(int Step)
{
// Check connectivity against both existing and imported units
emit stepEntered(tr("Checking connectivity consistency..."));
for (int i=0; i<m_SourcesData.size();i++)
{
QMapIterator<int, SourceUnit> It(m_SourcesData[i].Units);
while (It.hasNext())
{
It.next();
QList<SourceConnection> Conns = It.value().ToConn;
for (int j=0; j<Conns.size(); j++)
{
if (!isUnitExists(Conns[j].DestClass,Conns[j].DestID))
{
emit stepCompleted(Step,getStyledText(tr("[Error] Destination unit of \"To\" connection "
"does not exist for unit %2 of layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(It.key()),
"red"));
return false;
}
}
Conns = It.value().ChildofConn;
for (int j=0; j<Conns.size(); j++)
{
if (!isUnitExists(Conns[j].DestClass,Conns[j].DestID))
{
emit stepCompleted(Step,getStyledText(tr("[Error] Parent unit of \"Child Of\" connection "
"does not exist for unit %2 of layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(It.key()),
"red"));
return false;
}
}
}
}
emit stepCompleted(Step,getStyledText(tr("[OK]"),"green"));
return true;
}
bool ImportWorker::buildConnections(int Step)
{
emit stepEntered(tr("Building connections..."));
int i=0;
bool OKToContinue = true;
while (i<m_SourcesInfos.size() && OKToContinue)
{
QMapIterator<int, SourceUnit> It(m_SourcesData[i].Units);
while (It.hasNext())
{
It.next();
// "to" connections
for (int j=0;j<It.value().ToConn.size();j++)
{
mp_AdvDesc->spatialDomain().addFromToRelation(std::make_pair(m_SourcesInfos[i].UnitsClass.toStdString(),
It.key()),
std::make_pair(It.value().ToConn[j].DestClass.toStdString(),
It.value().ToConn[j].DestID));
}
// "childof" connections
for (int j=0;j<It.value().ChildofConn.size();j++)
{
mp_AdvDesc->spatialDomain().addParentChildRelation(
std::make_pair(It.value().ChildofConn[j].DestClass.toStdString(),
It.value().ChildofConn[j].DestID),
std::make_pair(m_SourcesInfos[i].UnitsClass.toStdString(),It.key()));
}
}
i++;
}
emit stepCompleted(Step,getStyledText(tr("[OK]"),"green"));
return OKToContinue;
}
bool ImportWorker::processFilesAndDatastore(int Step)
{
emit stepEntered(tr("Copying files and populating datastore..."));
int i=0;
bool OKToContinue = true;
while (i<m_SourcesInfos.size() && OKToContinue)
{
if (m_SourcesInfos[i].IsDatasetImport)
{
// local copy of files in dataset
QString FullSrcFilePath = m_SourcesInfos[i].SourceURI;
if (!m_SourcesInfos[i].CachedSourceURI.isEmpty())
FullSrcFilePath = m_SourcesInfos[i].CachedSourceURI;
QString FullDestFilePath = m_InputDir + "/" + m_SourcesInfos[i].RelativeDatasetPath;
QString FullDestPath = QFileInfo(FullDestFilePath).absolutePath();
QString DestExtension = QFileInfo(FullDestFilePath).suffix();
QDir().mkpath(FullDestPath);
GDALAllRegister_COMPAT();
GDALDataset_COMPAT* SrcDS = GDALOpenRO_COMPAT(FullSrcFilePath.toStdString().c_str());
if (SrcDS != nullptr)
{
GDALDriver_COMPAT *CopyDriver;
QString DriverName = OGRGDALHelpers::getDriverFromFileExt(DestExtension);
CopyDriver = GDALGetDriverByName_COMPAT(DriverName.toStdString().c_str());
if (CopyDriver != nullptr)
{
GDALDataset_COMPAT* DestDS = GDALCopy_COMPAT(CopyDriver,SrcDS,FullDestFilePath.toStdString().c_str());
GDALClose_COMPAT(DestDS);
}
GDALClose_COMPAT(SrcDS);
}
}
if (m_SourcesInfos[i].IsDatastore)
{
QString RelativePath = m_SourcesInfos[i].RelativeDatasetPath;
if (m_SourcesInfos[i].IsAlreadyInDataset)
RelativePath = QDir(m_InputDir).relativeFilePath(m_SourcesInfos[i].SourceURI);
// populate datastore
openfluid::fluidx::DatastoreItemDescriptor* DSItem =
new openfluid::fluidx::DatastoreItemDescriptor(m_SourcesInfos[i].DatastoreID.toStdString(),
m_InputDir.toStdString(),
RelativePath.toStdString(),
openfluid::core::UnstructuredValue::GeoVectorValue);
DSItem->setUnitsClass(m_SourcesInfos[i].UnitsClass.toStdString());
mp_AdvDesc->datastore().appendItem(DSItem);
}
i++;
}
emit stepCompleted(Step,getStyledText(tr("[OK]"),"green"));
return true;
}
bool DataProcessingWorker::runCheck(int StartStep)
{
// Check unit class does not exist
emit stepEntered(tr("Checking units classes..."));
for (int i=0; i<m_SourcesInfos.size();i++)
{
if (m_SourcesInfos[i].UnitsClass.isEmpty())
{
emit stepCompleted(StartStep,getStyledText(tr("[Error] Class name for layer \"%1\" is empty")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
if (mp_Desc->spatialDomain().isClassNameExists(m_SourcesInfos[i].UnitsClass.toStdString()))
{
emit stepCompleted(StartStep,getStyledText(tr("[Error] Class name for layer \"%1\" already exists")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
for (int j=0; j<m_SourcesInfos.size();j++)
{
if (i!=j && m_SourcesInfos[j].UnitsClass == m_SourcesInfos[i].UnitsClass)
{
emit stepCompleted(StartStep,getStyledText(tr("[Error] Units class \"%1\" is imported twice")
.arg(m_SourcesInfos[i].UnitsClass),
"red"));
return false;
}
}
}
emit stepCompleted(StartStep,getStyledText(tr("[OK]"),"green"));
emit stepEntered(tr("Checking configuration of files copies and datastore..."));
for (int i=0; i<m_SourcesInfos.size();i++)
{
// check if dataset import file is not empty
if (m_SourcesInfos[i].IsDatasetImport && m_SourcesInfos[i].RelativeDatasetPath.isEmpty())
{
emit stepCompleted(StartStep,
getStyledText(tr("[Error] Missing file name for copy of layer \"%1\" in project dataset")
.arg(m_SourcesInfos[i].LayerName),"red"));
return false;
}
QFileInfo DatasetFile(m_SourcesInfos[i].RelativeDatasetPath);
QString Ext = DatasetFile.suffix();
std::set<std::string> RegisteredExtsList = openfluid::utils::getOGRFilesExtensionsForOpenFLUID();
// check if dataset import file contains an extension
if (m_SourcesInfos[i].IsDatasetImport && Ext.isEmpty())
{
emit stepCompleted(StartStep,getStyledText(tr("[Error] Missing file extension for layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
// check if dataset import file extension exists in drivers
if (m_SourcesInfos[i].IsDatasetImport && RegisteredExtsList.find(Ext.toStdString()) == RegisteredExtsList.end())
{
emit stepCompleted(StartStep,getStyledText(tr("[Error] Wrong file extension for layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
// check if datastore ID is not empty
if (m_SourcesInfos[i].IsDatastore && m_SourcesInfos[i].DatastoreID.isEmpty())
{
emit stepCompleted(StartStep,getStyledText(tr("[Error] Missing datastore ID for layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
// check if datastore ID already exists
if (m_SourcesInfos[i].IsDatastore &&
mp_Desc->datastore().isItemExist(m_SourcesInfos[i].DatastoreID.toStdString()))
{
emit stepCompleted(StartStep,getStyledText(tr("[Error] Datastore ID for layer \"%1\" already exists")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
}
emit stepCompleted(StartStep+1,getStyledText(tr("[OK]"),"green"));
// load data from source
if (!loadDataFromSources(StartStep+2))
{
return false;
}
if (!checkConnectivity(StartStep+3))
{
return false;
}
return true;
}
bool DataProcessingWorker::loadDataFromSources(int Step)
{
emit stepEntered(tr("Loading and checking data from sources..."));
GDALAllRegister_COMPAT();
for (int i=0; i<m_SourcesInfos.size();i++)
{
QString RealURI = m_SourcesInfos[i].SourceURI;
if (!m_SourcesInfos[i].CachedSourceURI.isEmpty())
RealURI = m_SourcesInfos[i].CachedSourceURI;
GDALDataset_COMPAT* DS = GDALOpenRO_COMPAT(RealURI.toStdString().c_str());
if (DS == nullptr)
{
emit stepCompleted(Step,getStyledText(tr("[Error] Unable to open datasource for layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),"red"));
return false;
}
OGRLayer *Layer;
OGRFeature *Feature;
Layer = DS->GetLayerByName(m_SourcesInfos[i].LayerName.toStdString().c_str());
// For cached layers as GeoJSON files
// TODO do better for that!
if (Layer == nullptr)
{
Layer = DS->GetLayer(0);
}
Layer->ResetReading();
while((Feature = Layer->GetNextFeature()) != nullptr )
{
SourceUnit CurrentUnit;
int CurrentUnitID;
int FieldIndex = 0;
QString FieldValue;
// === Unit ID ===
FieldIndex = Feature->GetFieldIndex(OGRGDAL_UNITID_FIELD);
if (FieldIndex < 0)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Field for unit ID not found in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),"red"));
return false;
}
CurrentUnitID = Feature->GetFieldAsInteger(FieldIndex);
if (CurrentUnitID <= 0)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Wrong field format for unit ID in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),"red"));
return false;
}
if (m_SourcesData[i].isUnitExists(CurrentUnitID))
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Unit ID %2 already exist in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(CurrentUnitID),
"red"));
return false;
}
// === Process order ===
if (m_SourcesInfos[i].UnitsPcsOrdField.isEmpty())
{
CurrentUnit.ProcessOrder = 1;
}
else
{
FieldIndex = Feature->GetFieldIndex(m_SourcesInfos[i].UnitsPcsOrdField.toStdString().c_str());
if (FieldIndex < 0)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Field for process order not found in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
int PcsOrd = Feature->GetFieldAsInteger(FieldIndex);
if (PcsOrd <= 0)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Wrong field format for process order in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),
"red"));
return false;
}
CurrentUnit.ProcessOrder = PcsOrd;
}
// === To connections ===
if (!m_SourcesInfos[i].ToConnectionsField.isEmpty())
{
FieldIndex = Feature->GetFieldIndex(m_SourcesInfos[i].ToConnectionsField.toStdString().c_str());
if (FieldIndex < 0)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Field for \"To\" connections not found in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),"red"));
return false;
}
CurrentUnit.ToConnStr = QString(Feature->GetFieldAsString(FieldIndex));
if (!OGRGDALHelpers::convertConnectionsStringToList(CurrentUnit.ToConnStr,CurrentUnit.ToConn))
{
emit stepCompleted(Step,getStyledText(tr("[Error] Wrong field \"%2\" format for \"To\" connections "
"in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].ToConnectionsField),
"red"));
return false;
}
}
// === Childof connections ===
if (!m_SourcesInfos[i].ChildofConnectionsField.isEmpty())
{
FieldIndex = Feature->GetFieldIndex(m_SourcesInfos[i].ChildofConnectionsField.toStdString().c_str());
if (FieldIndex < 0)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,
getStyledText(tr("[Error] Field for \"Child Of\" connections not found in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName),"red"));
return false;
}
CurrentUnit.ChildofConnStr = QString(Feature->GetFieldAsString(FieldIndex));
if (!OGRGDALHelpers::convertConnectionsStringToList(CurrentUnit.ChildofConnStr,CurrentUnit.ChildofConn))
{
emit stepCompleted(Step,getStyledText(tr("[Error] Wrong field \"%2\" format for \"Child Of\" connections "
"in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].ChildofConnectionsField),
"red"));
return false;
}
}
// === Attributes ===
for (int j=0;j<m_SourcesInfos[i].ImportedFields.size(); j++)
{
FieldIndex = Feature->GetFieldIndex(m_SourcesInfos[i].ImportedFields[j].toStdString().c_str());
if (FieldIndex <0)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Field for attribute \"%2\" not found in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].ImportedFields[j]),
"red"));
return false;
}
// replacing unwanted chars (space, tab) by underscore
QString Attr;
if (!OGRGDALHelpers::convertFieldToAttribute(Feature,FieldIndex,Attr))
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Wrong field format for attribute \"%2\" in layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].ImportedFields[j]),
"red"));
return false;
}
CurrentUnit.Attributes[m_SourcesInfos[i].ImportedFields[j]] = Attr.replace(" ","_")
.replace("\t","_");
}
// === Area computed attribute ===
if (m_SourcesInfos[i].IsAreaCompute)
{
if (m_SourcesInfos[i].ImportedFields.contains(m_SourcesInfos[i].AreaComputeAttribute))
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Attribute \"%2\" for computed area attribute "
"is already imported from layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].AreaComputeAttribute),
"red"));
return false;
}
else
{
double Area = ((OGRPolygon*)(Feature->GetGeometryRef()))->get_Area();
CurrentUnit.Attributes[m_SourcesInfos[i].AreaComputeAttribute] = QString::number(Area,'g');
}
}
// === Length computed attribute ===
if (m_SourcesInfos[i].IsLengthCompute)
{
if (m_SourcesInfos[i].ImportedFields.contains(m_SourcesInfos[i].LengthComputeAttribute))
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Attribute \"%2\" for computed length attribute "
"is already imported from layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].LengthComputeAttribute),
"red"));
return false;
}
else
{
double Length = ((OGRLineString*)(Feature->GetGeometryRef()))->get_Length();
CurrentUnit.Attributes[m_SourcesInfos[i].LengthComputeAttribute] = QString::number(Length,'g');
}
}
// === Preprocess for computed centroid
OGRPoint Centroid;
if (Feature->GetGeometryRef()->Centroid(&Centroid) != OGRERR_NONE)
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,
getStyledText(tr("[Error] Unable to compute centroid coordinates "
"for geometries in layer \"%1\"."
"Computing centroid coordinates should be unchecked "
"in computed attributes.")
.arg(m_SourcesInfos[i].LayerName),"red"));
return false;
}
// === XCentroid computed attribute ===
if (m_SourcesInfos[i].IsXCentroidCompute)
{
if (m_SourcesInfos[i].ImportedFields.contains(m_SourcesInfos[i].XCentroidComputeAttribute))
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Attribute \"%2\" for computed centroid X attribute "
"is already imported from layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].XCentroidComputeAttribute),
"red"));
return false;
}
else
{
CurrentUnit.Attributes[m_SourcesInfos[i].XCentroidComputeAttribute] = QString::number(Centroid.getX(),'g');
}
}
// === YCentroid computed attribute ===
if (m_SourcesInfos[i].IsYCentroidCompute)
{
if (m_SourcesInfos[i].ImportedFields.contains(m_SourcesInfos[i].YCentroidComputeAttribute))
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Attribute \"%2\" for computed centroid Y attribute "
"is already imported from layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].YCentroidComputeAttribute),
"red"));
return false;
}
else
{
CurrentUnit.Attributes[m_SourcesInfos[i].YCentroidComputeAttribute] = QString::number(Centroid.getY(),'g');
}
}
// === ZCentroid computed attribute ===
if (m_SourcesInfos[i].IsZCentroidCompute)
{
if (m_SourcesInfos[i].ImportedFields.contains(m_SourcesInfos[i].ZCentroidComputeAttribute))
{
GDALClose_COMPAT(DS);
emit stepCompleted(Step,getStyledText(tr("[Error] Attribute \"%2\" for computed centroid Z attribute "
"is already imported from layer \"%1\"")
.arg(m_SourcesInfos[i].LayerName)
.arg(m_SourcesInfos[i].ZCentroidComputeAttribute),
"red"));
return false;
}
else
{
CurrentUnit.Attributes[m_SourcesInfos[i].ZCentroidComputeAttribute] = QString::number(Centroid.getZ(),'g');
}
}
m_SourcesData[i].Units[CurrentUnitID] = CurrentUnit;
}
GDALClose_COMPAT(DS);
DS = nullptr;
}
emit stepCompleted(Step,getStyledText(tr("[OK]"),"green"));
return true;
}