boolean checkStartEnd(struct bed *bedList, char *table, struct hTableInfo *hti,
int chromSize)
/* invariant: 0 <= start <= end <= chromSize. Return TRUE if any problems. */
/* NOTE: to save time, all elements of bedList are assumed to be from the
* same chrom! so chromSize can be applied to all. */
{
struct bed *bed = NULL;
boolean gotError = 0;
int startLTZ = 0, endLTStart = 0, endGTChrom = 0;
for (bed = bedList; bed != NULL; bed = bed->next)
{
if (bed->chromEnd < bed->chromStart)
{
verbose(2, "%s.%s item %s %s:%d-%d: %s < %s\n",
db, table, bed->name ? bed->name : "",
bed->chrom, bed->chromStart, bed->chromEnd,
hti->endField, hti->startField);
endLTStart++;
}
if (bed->chromEnd > chromSize)
{
verbose(2, "%s.%s item %s %s:%d-%d: %s > chromSize %d\n",
db, table, bed->name ? bed->name : "",
bed->chrom, bed->chromStart, bed->chromEnd,
hti->endField, chromSize);
endGTChrom++;
}
}
gotError |= reportErrors(START_LT_ZERO, table, startLTZ);
gotError |= reportErrors(END_LT_START, table, endLTStart);
gotError |= reportErrors(END_GT_CHROMSIZE, table, endGTChrom);
return gotError;
}
/*! \brief Launch YARPRead Inspection modality*/
void ApplicationViewWidget::onYARPRead()
{
if(manager.busy()){
return;
}
yarp::manager::ErrorLogger* logger = yarp::manager::ErrorLogger::Instance();
for(int i=0;i<ui->connectionList->topLevelItemCount();i++){
QTreeWidgetItem *it = ui->connectionList->topLevelItem(i);
if(it->isSelected()){
QString from = it->text(3);
QString to = QString("/inspect/read%1").arg(from);
#if defined(WIN32)
QString cmd = "cmd.exe";
QString param;
param = QString("/C yarp read %1").arg(to);
#else
QString cmd = "xterm";
QString param;
param = QString("-hold -title %1 -e yarp read %2").arg(from).arg(to);
#endif
yarp::manager::LocalBroker launcher;
launcher.showConsole(true);
if(launcher.init(cmd.toLatin1().data(), param.toLatin1().data(), NULL, NULL, NULL, NULL))
{
if(!launcher.start() && strlen(launcher.error()))
{
QString msg;
msg = QString("Error while launching yarpread. %1").arg(launcher.error());
logger->addError(msg.toLatin1().data());
reportErrors();
}
else
{
// waiting for the port to get open
double base = yarp::os::Time::now();
while(!timeout(base, 3.0)){
if(launcher.exists(to.toLatin1().data())){
break;
}
}
if(!launcher.connect(from.toLatin1().data(), to.toLatin1().data(), "udp")){
QString msg;
msg = QString("Cannot inspect '%1' : %2").arg(from).arg(launcher.error());
logger->addError(msg.toLatin1().data());
launcher.stop();
reportErrors();
}
}
}
}
}
yarp::os::Time::delay(0.1);
}
/*! \brief Launch YARPScope Inspection modality*/
void ApplicationViewWidget::onYARPScope()
{
if(manager.busy()){
return;
}
yarp::manager::ErrorLogger* logger = yarp::manager::ErrorLogger::Instance();
YscopeWindow dlg;
dlg.setModal(true);
if(dlg.exec() != QDialog::Accepted){
return;
}
int strIndex = dlg.getIndex();
for(int i=0;i<ui->connectionList->topLevelItemCount();i++){
QTreeWidgetItem *it = ui->connectionList->topLevelItem(i);
if(it->isSelected()){
QString from = it->text(3);
QString to = QString("/inspect").arg(from);
QString env = QString("YARP_PORT_PREFIX=%1").arg(to);
to += "/yarpscope";
QString param;
param = QString("--title %1:%2 --bgcolor white --color blue --graph_size 2 --index %2").arg(from).arg(strIndex);
yarp::manager::LocalBroker launcher;
if(launcher.init("yarpscope", param.toLatin1().data(), NULL, NULL, NULL, env.toLatin1().data())){
if(!launcher.start() && strlen(launcher.error())){
QString msg;
msg = QString("Error while launching yarpscope. %1").arg(launcher.error());
logger->addError(msg.toLatin1().data());
reportErrors();
}
else{
// waiting for the port to get open
double base = yarp::os::Time::now();
while(!timeout(base, 3.0))
if(launcher.exists(to.toLatin1().data())) break;
if(!launcher.connect(from.toLatin1().data(), to.toLatin1().data(), "udp")){
QString msg;
msg = QString("Cannot inspect '%1' : %2").arg(from).arg(launcher.error());
logger->addError(msg.toLatin1().data());
launcher.stop();
reportErrors();
}
}
}
}
}
yarp::os::Time::delay(0.1);
}
boolean checkCDSStartEnd(struct bed *bedList, char *table,
struct hTableInfo *hti)
/* invariant: start <= thickStart <= thickEnd <= end,
* unless allowThickZero and thickStart == thickEnd == 0.
* Return TRUE if any problems. */
{
boolean gotError = FALSE;
struct bed *bed = NULL;
int cdsSLTStart=0, cdsELTCS=0, cdsEGTEnd=0, cdsOnlySZ=0;
for (bed = bedList; bed != NULL; bed = bed->next)
{
if (bed->thickStart < bed->chromStart &&
!(allowThickZero && bed->thickStart == 0))
{
verbose(2, "%s.%s item %s %s:%d-%d: %s (%d) < %s (%d)\n",
db, table, bed->name ? bed->name : "",
bed->chrom, bed->chromStart, bed->chromEnd,
hti->cdsStartField, bed->thickStart,
hti->startField, bed->chromStart);
cdsSLTStart++;
}
if (bed->thickEnd < bed->thickStart)
{
verbose(2, "%s.%s item %s %s:%d-%d: %s (%d) < %s (%d)\n",
db, table, bed->name ? bed->name : "",
bed->chrom, bed->chromStart, bed->chromEnd,
hti->cdsEndField, bed->thickEnd,
hti->cdsStartField, bed->thickStart);
cdsELTCS++;
}
if (bed->chromEnd < bed->thickEnd)
{
verbose(2, "%s.%s item %s %s:%d-%d: %s (%d) > %s (%d)\n",
db, table, bed->name ? bed->name : "",
bed->chrom, bed->chromStart, bed->chromEnd,
hti->cdsEndField, bed->thickEnd,
hti->endField, bed->chromEnd);
cdsEGTEnd++;
}
if (allowThickZero &&
(bed->thickStart == 0 && bed->thickEnd != 0 && bed->chromStart != 0))
{
verbose(2, "%s.%s item %s %s:%d-%d: %s (%d) > 0 but not %s (%d)\n",
db, table, bed->name ? bed->name : "",
bed->chrom, bed->chromStart, bed->chromEnd,
hti->cdsEndField, bed->thickEnd,
hti->cdsStartField, bed->thickStart);
cdsOnlySZ++;
}
}
gotError |= reportErrors(CDSSTART_LT_START, table, cdsSLTStart);
gotError |= reportErrors(CDSEND_LT_CDSSTART, table, cdsELTCS);
gotError |= reportErrors(CDSEND_GT_END, table, cdsEGTEnd);
gotError |= reportErrors(ONLY_CDSSTART_ZERO, table, cdsOnlySZ);
return gotError;
}
/*! \brief Launch YARPView Inspection modality*/
void ApplicationViewWidget::onYARPView()
{
if(manager.busy()){
return;
}
yarp::manager::ErrorLogger* logger = yarp::manager::ErrorLogger::Instance();
for(int i=0;i<ui->connectionList->topLevelItemCount();i++){
QTreeWidgetItem *it = ui->connectionList->topLevelItem(i);
if(it->isSelected()){
QString from = it->text(3);
QString to = QString("/inspect").arg(from);
QString env = QString("YARP_PORT_PREFIX=%1").arg(to);
to += "/yarpview/img:i";
yarp::manager::LocalBroker launcher;
if(launcher.init("yarpview", NULL, NULL, NULL, NULL, env.toLatin1().data()))
{
if(!launcher.start() && strlen(launcher.error()))
{
QString msg;
msg = QString("Error while launching yarpview. %1").arg(launcher.error());
logger->addError(msg.toLatin1().data());
reportErrors();
}
else
{
// waiting for the port to get open
double base = yarp::os::Time::now();
while(!timeout(base, 3.0)){
if(launcher.exists(to.toLatin1().data())){
break;
}
}
if(!launcher.connect(from.toLatin1().data(), to.toLatin1().data(), "udp")){
QString msg;
msg = QString("Cannot inspect '%1' : %2").arg(from).arg(launcher.error());
logger->addError(msg.toLatin1().data());
launcher.stop();
reportErrors();
}
}
}
}
}
yarp::os::Time::delay(0.1);
}
void CallSite::finish() {
// Find the best of the remaining candidates.
ConversionRank best = Incompatible;
for (const_iterator ci = begin(), ciEnd = end(); ci != ciEnd; ++ci) {
// Ignore culled candidates unless there were no un-culled ones.
CallCandidate * cc = *ci;
// Don't bother producing a report if any of the candidates have errors.
if (cc->hasErrors()) {
return;
}
if (!cc->isCulled() || remaining_ == 0) {
ConversionRank rank = cc->conversionRank();
if (rank > best) {
best = rank;
}
}
}
if (remaining_ != 1) {
if (best == Incompatible) {
reportErrors("No method found matching arguments ");
} else {
reportErrors("Ambiguous overloaded methods for call to ");
}
} else if (isConversionWarning(best)) {
std::string msg(compatibilityError(best));
msg.append(" attempting to call ");
reportErrors(msg.c_str());
}
// Remove all culled candidates from the list.
Candidates & cd = callExpr_->candidates();
for (iterator ci = cd.begin(); ci != cd.end();) {
if ((*ci)->isCulled()) {
ci = cd.erase(ci);
} else {
++ci;
}
}
// Regardless of whether there was an ambiguity or not, just pick the
// first candidate.
CallCandidate * c = cd.front();
if (c->method() != NULL) {
callExpr_->setFunction(LValueExpr::get(callExpr_->location(), c->base(), c->method()));
}
}
asynStatus isisdaeDriver::readArray(asynUser *pasynUser, const char* functionName, T *value, size_t nElements, size_t *nIn)
{
int function = pasynUser->reason;
asynStatus status = asynSuccess;
const char *paramName = NULL;
getParamName(function, ¶mName);
m_iface->resetMessages();
try
{
asynPrint(pasynUser, ASYN_TRACEIO_DRIVER,
"%s:%s: function=%d, name=%s\n",
driverName, functionName, function, paramName);
reportMessages();
return status;
}
catch(const std::exception& ex)
{
*nIn = 0;
epicsSnprintf(pasynUser->errorMessage, pasynUser->errorMessageSize,
"%s:%s: status=%d, function=%d, name=%s, error=%s",
driverName, functionName, status, function, paramName, ex.what());
reportErrors(ex.what());
return asynError;
}
}
asynStatus isisdaeDriver::readValue(asynUser *pasynUser, const char* functionName, T* value)
{
int function = pasynUser->reason;
asynStatus status = asynSuccess;
const char *paramName = NULL;
getParamName(function, ¶mName);
m_iface->resetMessages();
try
{
status = asynPortDriver::readValue(pasynUser, functionName, &value);
asynPrint(pasynUser, ASYN_TRACEIO_DRIVER,
"%s:%s: function=%d, name=%s, value=%s\n",
driverName, functionName, function, paramName, convertToString(*value).c_str());
reportMessages();
return status;
}
catch(const std::exception& ex)
{
epicsSnprintf(pasynUser->errorMessage, pasynUser->errorMessageSize,
"%s:%s: status=%d, function=%d, name=%s, value=%s, error=%s",
driverName, functionName, status, function, paramName, convertToString(*value).c_str(), ex.what());
reportErrors(ex.what());
return asynError;
}
}
MultiTopicImpl::MultiTopicImpl(const char* name,
const char* type_name, const char* subscription_expression,
const DDS::StringSeq& expression_parameters,
DomainParticipantImpl* participant)
: TopicDescriptionImpl(name, type_name,
findTypeSupport(participant, type_name),
participant)
, subscription_expression_(subscription_expression)
, expression_parameters_(expression_parameters)
, filter_eval_(NULL)
{
const char* out = subscription_expression
+ std::strlen(subscription_expression);
yard::SimpleTextParser parser(subscription_expression, out);
if (!parser.Parse<TopicExpressionGrammar::TopicCompleteInput>()) {
reportErrors(parser, subscription_expression);
}
for (AstNode* iter = parser.GetAstRoot()->GetFirstChild(); iter;
iter = iter->GetSibling()) {
if (iter->TypeMatches<TopicExpressionGrammar::SubjectFieldSpec>()) {
AstNode* fieldName = iter->GetFirstChild();
aggregation_.push_back(SubjectFieldSpec(toString(fieldName),
toString(fieldName->GetSibling())));
} else if (iter->TypeMatches<TopicExpressionGrammar::TopicName>()) {
selection_.push_back(toString(iter));
} else {
filter_eval_ = new FilterEvaluator(iter);
}
}
}
bool luaScript::doString(const char* script)
{
if (luaL_loadstring(L, script))
{
reportErrors();
return false;
}
else
{
if (lua_pcall(L,0,0,0))
{
reportErrors();
return false;
}
}
return true;
}
bool luaScript::endExecuteFunction(int args, int rets)
{
if (lua_pcall(L, args, rets, 0))
{
reportErrors();
return false;
}
return true;
}
/*! \brief Called when a disconnection has been performed
\param which
*/
void ApplicationViewWidget::onConDisconnect(int which)
{
QTreeWidgetItem *it = ui->connectionList->topLevelItem(which);
if(it){
it->setText(2,"disconnected");
it->setIcon(0,QIcon(":/images/disconnected_ico.png"));
it->setTextColor(2,QColor("#BF0303"));
}
reportErrors();
}
/*! \brief Called when a modlue has been stopped
\param which
*/
void ApplicationViewWidget::onModStop(int which)
{
QTreeWidgetItem *it = ui->moduleList->topLevelItem(which);
if(it){
it->setText(2,"stopped");
it->setIcon(0,QIcon(":/images/suspended_ico.png"));
it->setTextColor(2,QColor("#BF0303"));
}
reportErrors();
}
asynStatus isisdaeDriver::readOctet(asynUser *pasynUser, char *value, size_t maxChars, size_t *nActual, int *eomReason)
{
int function = pasynUser->reason;
int status=0;
const char *functionName = "readOctet";
const char *paramName = NULL;
getParamName(function, ¶mName);
m_iface->resetMessages();
// we don't do much yet
return asynPortDriver::readOctet(pasynUser, value, maxChars, nActual, eomReason);
std::string value_s;
try
{
if (m_iface == NULL)
{
throw std::runtime_error("m_iface is NULL");
}
// m_iface->getLabviewValue(paramName, &value_s);
if ( value_s.size() > maxChars ) // did we read more than we have space for?
{
*nActual = maxChars;
if (eomReason) { *eomReason = ASYN_EOM_CNT | ASYN_EOM_END; }
asynPrint(pasynUser, ASYN_TRACEIO_DRIVER,
"%s:%s: function=%d, name=%s, value=\"%s\" (TRUNCATED from %d chars)\n",
driverName, functionName, function, paramName, value_s.substr(0,*nActual).c_str(), value_s.size());
}
else
{
*nActual = value_s.size();
if (eomReason) { *eomReason = ASYN_EOM_END; }
asynPrint(pasynUser, ASYN_TRACEIO_DRIVER,
"%s:%s: function=%d, name=%s, value=\"%s\"\n",
driverName, functionName, function, paramName, value_s.c_str());
}
strncpy(value, value_s.c_str(), maxChars); // maxChars will NULL pad if possible, change to *nActual if we do not want this
setStringParam(P_AllMsgs, m_iface->getAllMessages().c_str());
setStringParam(P_ErrMsgs, "");
m_iface->resetMessages();
callParamCallbacks(); // this flushes P_ErrMsgs
return asynSuccess;
}
catch(const std::exception& ex)
{
epicsSnprintf(pasynUser->errorMessage, pasynUser->errorMessageSize,
"%s:%s: status=%d, function=%d, name=%s, value=\"%s\", error=%s",
driverName, functionName, status, function, paramName, value_s.c_str(), ex.what());
reportErrors(ex.what());
callParamCallbacks(); // this flushes P_ErrMsgs
*nActual = 0;
if (eomReason) { *eomReason = ASYN_EOM_END; }
value[0] = '\0';
return asynError;
}
}
bool luaScript::doBinary(const char* buffer, int size, const wchar_t* name)
{
char cName[MAX_PATH*2];
widePathToShortAsciiPath(name, cName);
if (luaL_loadbuffer(L, buffer, size, cName))
{
reportErrors();
return false;
}
else
{
if (lua_pcall(L,0,0,0))
{
reportErrors();
return false;
}
}
return true;
}
bool luaScript::doFile(const wchar_t* filename)
{
char cFileName[MAX_PATH*2];
if (!widePathToShortAsciiPath(filename, cFileName))
return false;
if (luaL_loadfile(L, cFileName))
{
reportErrors();
return false;
}
else
{
if (lua_pcall(L,0,0,0))
{
reportErrors();
return false;
}
}
}
void ApplicationViewWidget::prepareManagerFrom(yarp::manager::Manager* lazy)
{
manager.prepare(lazy, m_pConfig,dynamic_cast<ApplicationEvent*>(this));
// loading application
if(manager.loadApplication(app->getName())){
updateApplicationWindow();
}
reportErrors();
}
/*! \brief Called when a modlue has been started
\param which
*/
void ApplicationViewWidget::onModStart(int which)
{
QTreeWidgetItem *it = ui->moduleList->topLevelItem(which);
if(it){
it->setText(2,"running");
it->setIcon(0,QIcon(":/images/runnin_ico.png"));
it->setTextColor(2,QColor("#008C00"));
//row[m_modColumns.m_col_editable] = false;
//row[m_modColumns.m_col_color] = Gdk::Color("#008C00");
//row.set_value(0, m_refPixRunning);
}
reportErrors();
}
/*! \brief Export the current Graph*/
void ApplicationViewWidget::exportGraph()
{
QString fileName = QFileDialog::getSaveFileName(this,"Export Graph",QApplication::applicationDirPath(),"GraphViz format (*.dot)");
if(!fileName.isEmpty()){
if(!manager.exportDependencyGraph(fileName.toLatin1().data()))
{
yarp::manager::ErrorLogger* logger = yarp::manager::ErrorLogger::Instance();
logger->addError("Cannot export graph");
reportErrors();
return;
}
}
}
/*! \brief Called when a resource become unavaible
\param which
*/
void ApplicationViewWidget::onResUnAvailable(int which)
{
QTreeWidgetItem *it = ui->resourcesList->topLevelItem(which);
if(it){
it->setText(3,"unavailable");
if(it->text(2) == "computer"){
it->setIcon(0,QIcon(":/images/nores_ico.png"));
it->setTextColor(3,QColor("#BF0303"));
}else{
it->setIcon(0,QIcon(":/images/port_unavail_ico.png"));
it->setTextColor(3,QColor("#BF0303"));
}
}
reportErrors();
}
/*! \brief Called when a connection become unavaible
\param which
*/
void ApplicationViewWidget::onConUnAvailable(int from, int to)
{
if(from >= 0){
int row;
if(getConRowByID(from, &row))
ui->connectionList->topLevelItem(row)->setTextColor(3,QColor("#BF0303"));
}
if(to >= 0){
int row;
if(getConRowByID(to, &row))
ui->connectionList->topLevelItem(row)->setTextColor(4,QColor("#BF0303"));
}
reportErrors();
}
/*! \brief Called when a connection has been performed
\param which
*/
void ApplicationViewWidget::onConConnect(int which)
{
QTreeWidgetItem *it = ui->connectionList->topLevelItem(which);
if(it){
it->setText(2,"connected");
it->setIcon(0,QIcon(":/images/connected_ico.png"));
it->setTextColor(2,QColor("#008C00"));
}
/* row[m_conColumns.m_col_status] = "connected";
row[m_conColumns.m_col_editable] = false;
row[m_conColumns.m_col_color] = Gdk::Color("#008C00");
row.set_value(0, m_refPixConnected);*/
reportErrors();
}
/*! \brief Called when a resource became avaible
\param which
*/
void ApplicationViewWidget::onResAvailable(int which)
{
QTreeWidgetItem *it = ui->resourcesList->topLevelItem(which);
if(it){
it->setText(3,"available");
if(it->text(2) == "computer"){
it->setIcon(0,QIcon(":/images/computer_ico.png"));
it->setTextColor(3,QColor("#008C00"));
}else{
it->setIcon(0,QIcon(":/images/port_avail_ico.png"));
it->setTextColor(3,QColor("#008C00"));
}
/*if(row[m_resColumns.m_col_type] == Glib::ustring("computer"))
row.set_value(0, m_refPixAvailable);
else
row.set_value(0, m_refPixPortAvaibable);*/
}
reportErrors();
}
bool CallSite::unify(BindingEnv & env, int searchDepth) {
bool canUnify = false;
unsigned stateBeforeAny = env.stateCount();
for (const_iterator ci = begin(), ciEnd = end(); ci != ciEnd; ++ci) {
CallCandidate * c = *ci;
unsigned stateBeforeNext = env.stateCount();
if (c->unify(callExpr_, env)) {
canUnify = true;
} else {
if (showInference) {
diag.debug() << Format_Type << "Unification failed for " << c->method() << ":";
}
env.backtrack(stateBeforeNext);
c->cull(searchDepth);
}
}
if (!canUnify) {
reportErrors("No methods match calling signature: ");
backtrack(stateBeforeAny);
DBREAK;
if (showInference) {
unifyVerbose = true;
for (const_iterator ci = begin(), ciEnd = end(); ci != ciEnd; ++ci) {
CallCandidate * c = *ci;
unsigned stateBeforeNext = env.stateCount();
c->unify(callExpr_, env);
env.backtrack(stateBeforeNext);
}
unifyVerbose = false;
}
return false;
}
return true;
}
boolean checkSplitTableOnlyChrom(struct bed *bedList, char *table,
struct hTableInfo *hti, char *tableChrom)
/* invariant: if table is split and has a chrom field, then all
* entries in the table will have chrom matching the table name.
* Return TRUE if any errors. */
{
struct bed *bed = NULL;
boolean gotError = FALSE;
int errCount = 0;
for (bed = bedList; bed != NULL; bed = bed->next)
{
if (! sameString(bed->chrom, tableChrom))
{
verbose(2, "%s.%s item %s %s:%d-%d has %s = \"%s\" "
"inconsistent with split table name\n",
db, table,
bed->name, bed->chrom, bed->chromStart, bed->chromEnd,
hti->chromField, bed->chrom);
errCount++;
}
gotError |= reportErrors(SPLIT_WRONG_CHROM, table, errCount);
}
return(gotError);
}
void ApplicationViewWidget::onAssignHost()
{
if(areAllShutdown() && !manager.busy()) {
for(int i=0; i<ui->moduleList->topLevelItemCount(); i++) {
QTreeWidgetItem *it = ui->moduleList->topLevelItem(i);
if(it->isSelected()) {
//QString waitHost = QString("%1").arg("?");
it->setText(2,"waiting");
it->setText(3,"?");
}
}
manager.safeLoadBalance();
yarp::os::Time::delay(0.1);
/*typedef Gtk::TreeModel::Children type_children;
type_children children = m_refTreeModModel->children();
for(type_children::iterator iter = children.begin(); iter!=children.end(); ++iter)
{
Gtk::TreeModel::Row row = (*iter);
row[m_modColumns.m_col_status] = "waiting";
row[m_modColumns.m_col_editable] = false;
row[m_modColumns.m_col_host] = "?";
row[m_modColumns.m_col_color] = Gdk::Color("#000000");
row.set_value(0, m_refPixWaiting);
}
manager.safeLoadBalance();
*/
} else {
yarp::manager::ErrorLogger* logger = yarp::manager::ErrorLogger::Instance();
logger->addError("Running modules should be stopped before assigning hosts.");
reportErrors();
}
}
void QgsDelimitedTextProvider::rescanFile()
{
mRescanRequired = false;
resetIndexes();
bool buildSpatialIndex = mSpatialIndex != 0;
bool buildSubsetIndex = mBuildSubsetIndex && ( mSubsetExpression || mGeomRep != GeomNone );
// In case file has been rewritten check that it is still valid
mValid = mLayerValid && mFile->isValid();
if ( ! mValid ) return;
// Open the file and get number of rows, etc. We assume that the
// file has a header row and process accordingly. Caller should make
// sure that the delimited file is properly formed.
QStringList messages;
if ( mGeomRep == GeomAsWkt )
{
mWktFieldIndex = mFile->fieldIndex( mWktFieldName );
if ( mWktFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Wkt", mWktFieldName ) );
}
}
else if ( mGeomRep == GeomAsXy )
{
mXFieldIndex = mFile->fieldIndex( mXFieldName );
mYFieldIndex = mFile->fieldIndex( mYFieldName );
if ( mXFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "X", mWktFieldName ) );
}
if ( mYFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Y", mWktFieldName ) );
}
}
if ( messages.size() > 0 )
{
reportErrors( messages );
QgsDebugMsg( "Delimited text source invalid on rescan - missing geometry fields" );
mValid = false;
return;
}
// Reset the field columns
for ( int i = 0; i < attributeFields.size(); i++ )
{
attributeColumns[i] = mFile->fieldIndex( attributeFields.at( i ).name() );
}
// Scan through the features in the file
mSubsetIndex.clear();
mUseSubsetIndex = false;
QgsFeatureIterator fi = getFeatures( QgsFeatureRequest() );
mNumberFeatures = 0;
mExtent = QgsRectangle();
QgsFeature f;
while ( fi.nextFeature( f ) )
{
if ( mGeometryType != QGis::NoGeometry )
{
if ( mNumberFeatures == 0 )
{
mExtent = f.constGeometry()->boundingBox();
}
else
{
QgsRectangle bbox( f.constGeometry()->boundingBox() );
mExtent.combineExtentWith( &bbox );
}
if ( buildSpatialIndex ) mSpatialIndex->insertFeature( f );
}
if ( buildSubsetIndex ) mSubsetIndex.append(( quintptr ) f.id() );
mNumberFeatures++;
}
if ( buildSubsetIndex )
{
long recordCount = mFile->recordCount();
recordCount -= recordCount / SUBSET_ID_THRESHOLD_FACTOR;
mUseSubsetIndex = recordCount < mSubsetIndex.size();
if ( ! mUseSubsetIndex ) mSubsetIndex.clear();
}
mUseSpatialIndex = buildSpatialIndex;
}
void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
{
QStringList messages;
// assume the layer is invalid until proven otherwise
mLayerValid = false;
mValid = false;
mRescanRequired = false;
clearInvalidLines();
// Initiallize indexes
resetIndexes();
bool buildSpatialIndex = buildIndexes && mSpatialIndex != 0;
// No point building a subset index if there is no geometry, as all
// records will be included.
bool buildSubsetIndex = buildIndexes && mBuildSubsetIndex && mGeomRep != GeomNone;
if ( ! mFile->isValid() )
{
// uri is invalid so the layer must be too...
messages.append( tr( "File cannot be opened or delimiter parameters are not valid" ) );
reportErrors( messages );
QgsDebugMsg( "Delimited text source invalid - filename or delimiter parameters" );
return;
}
// Open the file and get number of rows, etc. We assume that the
// file has a header row and process accordingly. Caller should make
// sure that the delimited file is properly formed.
if ( mGeomRep == GeomAsWkt )
{
mWktFieldIndex = mFile->fieldIndex( mWktFieldName );
if ( mWktFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Wkt", mWktFieldName ) );
}
}
else if ( mGeomRep == GeomAsXy )
{
mXFieldIndex = mFile->fieldIndex( mXFieldName );
mYFieldIndex = mFile->fieldIndex( mYFieldName );
if ( mXFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "X", mWktFieldName ) );
}
if ( mYFieldIndex < 0 )
{
messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( "Y", mWktFieldName ) );
}
}
if ( messages.size() > 0 )
{
reportErrors( messages );
QgsDebugMsg( "Delimited text source invalid - missing geometry fields" );
return;
}
// Scan the entire file to determine
// 1) the number of fields (this is handled by QgsDelimitedTextFile mFile
// 2) the number of valid features. Note that the selection of valid features
// should match the code in QgsDelimitedTextFeatureIterator
// 3) the geometric extents of the layer
// 4) the type of each field
//
// Also build subset and spatial indexes.
QStringList parts;
long nEmptyRecords = 0;
long nBadFormatRecords = 0;
long nIncompatibleGeometry = 0;
long nInvalidGeometry = 0;
long nEmptyGeometry = 0;
mNumberFeatures = 0;
mExtent = QgsRectangle();
QList<bool> isEmpty;
QList<bool> couldBeInt;
QList<bool> couldBeLongLong;
QList<bool> couldBeDouble;
while ( true )
{
QgsDelimitedTextFile::Status status = mFile->nextRecord( parts );
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk )
{
nBadFormatRecords++;
recordInvalidLine( tr( "Invalid record format at line %1" ) );
continue;
}
// Skip over empty records
if ( recordIsEmpty( parts ) )
{
nEmptyRecords++;
continue;
}
// Check geometries are valid
bool geomValid = true;
if ( mGeomRep == GeomAsWkt )
{
if ( mWktFieldIndex >= parts.size() || parts[mWktFieldIndex].isEmpty() )
{
nEmptyGeometry++;
mNumberFeatures++;
}
else
{
// Get the wkt - confirm it is valid, get the type, and
// if compatible with the rest of file, add to the extents
QString sWkt = parts[mWktFieldIndex];
QgsGeometry *geom = 0;
if ( !mWktHasPrefix && sWkt.indexOf( WktPrefixRegexp ) >= 0 )
mWktHasPrefix = true;
if ( !mWktHasZM && sWkt.indexOf( WktZMRegexp ) >= 0 )
mWktHasZM = true;
geom = geomFromWkt( sWkt, mWktHasPrefix, mWktHasZM );
if ( geom )
{
QGis::WkbType type = geom->wkbType();
if ( type != QGis::WKBNoGeometry )
{
if ( mGeometryType == QGis::UnknownGeometry || geom->type() == mGeometryType )
{
mGeometryType = geom->type();
if ( mNumberFeatures == 0 )
{
mNumberFeatures++;
mWkbType = type;
mExtent = geom->boundingBox();
}
else
{
mNumberFeatures++;
if ( geom->isMultipart() ) mWkbType = type;
QgsRectangle bbox( geom->boundingBox() );
mExtent.combineExtentWith( &bbox );
}
if ( buildSpatialIndex )
{
QgsFeature f;
f.setFeatureId( mFile->recordId() );
f.setGeometry( geom );
mSpatialIndex->insertFeature( f );
// Feature now has ownership of geometry, so set to null
// here to avoid deleting twice.
geom = 0;
}
}
else
{
nIncompatibleGeometry++;
geomValid = false;
}
}
if ( geom ) delete geom;
}
else
{
geomValid = false;
nInvalidGeometry++;
recordInvalidLine( tr( "Invalid WKT at line %1" ) );
}
}
}
else if ( mGeomRep == GeomAsXy )
{
// Get the x and y values, first checking to make sure they
// aren't null.
QString sX = mXFieldIndex < parts.size() ? parts[mXFieldIndex] : QString();
QString sY = mYFieldIndex < parts.size() ? parts[mYFieldIndex] : QString();
if ( sX.isEmpty() && sY.isEmpty() )
{
nEmptyGeometry++;
mNumberFeatures++;
}
else
{
QgsPoint pt;
bool ok = pointFromXY( sX, sY, pt, mDecimalPoint, mXyDms );
if ( ok )
{
if ( mNumberFeatures > 0 )
{
mExtent.combineExtentWith( pt.x(), pt.y() );
}
else
{
// Extent for the first point is just the first point
mExtent.set( pt.x(), pt.y(), pt.x(), pt.y() );
mWkbType = QGis::WKBPoint;
mGeometryType = QGis::Point;
}
mNumberFeatures++;
if ( buildSpatialIndex && qIsFinite( pt.x() ) && qIsFinite( pt.y() ) )
{
QgsFeature f;
f.setFeatureId( mFile->recordId() );
f.setGeometry( QgsGeometry::fromPoint( pt ) );
mSpatialIndex->insertFeature( f );
}
}
else
{
geomValid = false;
nInvalidGeometry++;
recordInvalidLine( tr( "Invalid X or Y fields at line %1" ) );
}
}
}
else
{
mWkbType = QGis::WKBNoGeometry;
mNumberFeatures++;
}
if ( ! geomValid ) continue;
if ( buildSubsetIndex ) mSubsetIndex.append( mFile->recordId() );
// If we are going to use this record, then assess the potential types of each colum
for ( int i = 0; i < parts.size(); i++ )
{
QString &value = parts[i];
// Ignore empty fields - spreadsheet generated CSV files often
// have random empty fields at the end of a row
if ( value.isEmpty() )
continue;
// Expand the columns to include this non empty field if necessary
while ( couldBeInt.size() <= i )
{
isEmpty.append( true );
couldBeInt.append( false );
couldBeLongLong.append( false );
couldBeDouble.append( false );
}
// If this column has been empty so far then initiallize it
// for possible types
if ( isEmpty[i] )
{
isEmpty[i] = false;
couldBeInt[i] = true;
couldBeLongLong[i] = true;
couldBeDouble[i] = true;
}
// Now test for still valid possible types for the field
// Types are possible until first record which cannot be parsed
if ( couldBeInt[i] )
{
value.toInt( &couldBeInt[i] );
}
if ( couldBeLongLong[i] && ! couldBeInt[i] )
{
value.toLongLong( &couldBeLongLong[i] );
}
if ( couldBeDouble[i] && ! couldBeLongLong[i] )
{
if ( ! mDecimalPoint.isEmpty() )
{
value.replace( mDecimalPoint, "." );
}
value.toDouble( &couldBeDouble[i] );
}
}
}
// Now create the attribute fields. Field types are integer by preference,
// failing that double, failing that text.
QStringList fieldNames = mFile->fieldNames();
mFieldCount = fieldNames.size();
attributeColumns.clear();
attributeFields.clear();
QString csvtMessage;
QStringList csvtTypes = readCsvtFieldTypes( mFile->fileName(), &csvtMessage );
for ( int i = 0; i < fieldNames.size(); i++ )
{
// Skip over WKT field ... don't want to display in attribute table
if ( i == mWktFieldIndex ) continue;
// Add the field index lookup for the column
attributeColumns.append( i );
QVariant::Type fieldType = QVariant::String;
QString typeName = "text";
if ( i < csvtTypes.size() )
{
if ( csvtTypes[i] == "integer" )
{
fieldType = QVariant::Int;
typeName = "integer";
}
else if ( csvtTypes[i] == "long" || csvtTypes[i] == "longlong" || csvtTypes[i] == "int8" )
{
fieldType = QVariant::LongLong; //QVariant doesn't support long
typeName = "longlong";
}
else if ( csvtTypes[i] == "real" || csvtTypes[i] == "double" )
{
fieldType = QVariant::Double;
typeName = "double";
}
}
else if ( i < couldBeInt.size() )
{
if ( couldBeInt[i] )
{
fieldType = QVariant::Int;
typeName = "integer";
}
else if ( couldBeLongLong[i] )
{
fieldType = QVariant::LongLong;
typeName = "longlong";
}
else if ( couldBeDouble[i] )
{
fieldType = QVariant::Double;
typeName = "double";
}
}
attributeFields.append( QgsField( fieldNames[i], fieldType, typeName ) );
}
QgsDebugMsg( "Field count for the delimited text file is " + QString::number( attributeFields.size() ) );
QgsDebugMsg( "geometry type is: " + QString::number( mWkbType ) );
QgsDebugMsg( "feature count is: " + QString::number( mNumberFeatures ) );
QStringList warnings;
if ( ! csvtMessage.isEmpty() ) warnings.append( csvtMessage );
if ( nBadFormatRecords > 0 )
warnings.append( tr( "%1 records discarded due to invalid format" ).arg( nBadFormatRecords ) );
if ( nEmptyGeometry > 0 )
warnings.append( tr( "%1 records have missing geometry definitions" ).arg( nEmptyGeometry ) );
if ( nInvalidGeometry > 0 )
warnings.append( tr( "%1 records discarded due to invalid geometry definitions" ).arg( nInvalidGeometry ) );
if ( nIncompatibleGeometry > 0 )
warnings.append( tr( "%1 records discarded due to incompatible geometry types" ).arg( nIncompatibleGeometry ) );
reportErrors( warnings );
// Decide whether to use subset ids to index records rather than simple iteration through all
// If more than 10% of records are being skipped, then use index. (Not based on any experimentation,
// could do with some analysis?)
if ( buildSubsetIndex )
{
long recordCount = mFile->recordCount();
recordCount -= recordCount / SUBSET_ID_THRESHOLD_FACTOR;
mUseSubsetIndex = mSubsetIndex.size() < recordCount;
if ( ! mUseSubsetIndex ) mSubsetIndex = QList<quintptr>();
}
mUseSpatialIndex = buildSpatialIndex;
mValid = mGeometryType != QGis::UnknownGeometry;
mLayerValid = mValid;
// If it is valid, then watch for changes to the file
connect( mFile, SIGNAL( fileUpdated() ), this, SLOT( onFileUpdated() ) );
}
/*! \brief Refresh all and reports errors
*/
void ApplicationViewWidget::onLoadBalance(void)
{
updateApplicationWindow();
reportErrors();
}
/*! \brief Called whne an error occurred
*/
void ApplicationViewWidget::onError(void)
{
reportErrors();
}
