FeatureID MigrateNodeFeatureIndex(osm::Editor::ForEachFeaturesNearByFn & forEach,
XMLFeature const & xml,
FeatureStatus const featureStatus,
GenerateIDFn const & generateID)
{
if (featureStatus == FeatureStatus::Created)
return generateID();
FeatureID fid;
auto count = 0;
forEach(
[&fid, &count](FeatureType const & ft) {
if (ft.GetFeatureType() != feature::GEOM_POINT)
return;
// TODO(mgsergio): Check that ft and xml correspond to the same feature.
fid = ft.GetID();
++count;
},
MercatorBounds::FromLatLon(xml.GetCenter()));
if (count == 0)
MYTHROW(MigrationError, ("No pointed features returned."));
if (count > 1)
{
LOG(LWARNING,
(count, "features returned for point", MercatorBounds::FromLatLon(xml.GetCenter())));
}
return fid;
}
bool Editor::Save(string const & fullFilePath) const
{
// TODO(AlexZ): Improve synchronization in Editor code.
static mutex saveMutex;
lock_guard<mutex> lock(saveMutex);
if (m_features.empty())
{
my::DeleteFileX(GetEditorFilePath());
return true;
}
xml_document doc;
xml_node root = doc.append_child(kXmlRootNode);
// Use format_version for possible future format changes.
root.append_attribute("format_version") = 1;
for (auto const & mwm : m_features)
{
xml_node mwmNode = root.append_child(kXmlMwmNode);
mwmNode.append_attribute("name") = mwm.first.GetInfo()->GetCountryName().c_str();
mwmNode.append_attribute("version") = static_cast<long long>(mwm.first.GetInfo()->GetVersion());
xml_node deleted = mwmNode.append_child(kDeleteSection);
xml_node modified = mwmNode.append_child(kModifySection);
xml_node created = mwmNode.append_child(kCreateSection);
for (auto const & index : mwm.second)
{
FeatureTypeInfo const & fti = index.second;
// TODO: Do we really need to serialize deleted features in full details? Looks like mwm ID and meta fields are enough.
XMLFeature xf = fti.m_feature.ToXML(true /*type serializing helps during migration*/);
xf.SetMWMFeatureIndex(index.first);
if (!fti.m_street.empty())
xf.SetTagValue(kAddrStreetTag, fti.m_street);
ASSERT_NOT_EQUAL(0, fti.m_modificationTimestamp, ());
xf.SetModificationTime(fti.m_modificationTimestamp);
if (fti.m_uploadAttemptTimestamp != my::INVALID_TIME_STAMP)
{
xf.SetUploadTime(fti.m_uploadAttemptTimestamp);
ASSERT(!fti.m_uploadStatus.empty(), ("Upload status updates with upload timestamp."));
xf.SetUploadStatus(fti.m_uploadStatus);
if (!fti.m_uploadError.empty())
xf.SetUploadError(fti.m_uploadError);
}
switch (fti.m_status)
{
case FeatureStatus::Deleted: VERIFY(xf.AttachToParentNode(deleted), ()); break;
case FeatureStatus::Modified: VERIFY(xf.AttachToParentNode(modified), ()); break;
case FeatureStatus::Created: VERIFY(xf.AttachToParentNode(created), ()); break;
case FeatureStatus::Untouched: CHECK(false, ("Not edited features shouldn't be here."));
}
}
}
return my::WriteToTempAndRenameToFile(
fullFilePath,
[&doc](string const & fileName)
{
return doc.save_file(fileName.data(), " ");
});
}
FeatureID MigrateWayOrRelatonFeatureIndex(
osm::Editor::ForEachFeaturesNearByFn & forEach, XMLFeature const & xml,
FeatureStatus const /* Unused for now (we don't create/delete area features)*/,
GenerateIDFn const & /*Unused for the same reason*/)
{
boost::optional<FeatureID> fid;
auto bestScore = 0.6; // initial score is used as a threshold.
auto geometry = xml.GetGeometry();
auto count = 0;
if (geometry.empty())
MYTHROW(MigrationError, ("Feature has invalid geometry", xml));
// This can be any point on a feature.
auto const someFeaturePoint = geometry[0];
forEach(
[&fid, &geometry, &count, &bestScore](FeatureType & ft) {
if (ft.GetFeatureType() != feature::GEOM_AREA)
return;
++count;
auto ftGeometry = ft.GetTriangesAsPoints(FeatureType::BEST_GEOMETRY);
double score = 0.0;
try
{
score = matcher::ScoreTriangulatedGeometries(geometry, ftGeometry);
}
catch (matcher::NotAPolygonException & ex)
{
// Support migration for old application versions.
// TODO(a): To remove it when version 8.0.x will no longer be supported.
base::SortUnique(geometry);
base::SortUnique(ftGeometry);
score = matcher::ScoreTriangulatedGeometriesByPoints(geometry, ftGeometry);
}
if (score > bestScore)
{
bestScore = score;
fid = ft.GetID();
}
},
someFeaturePoint);
if (count == 0)
MYTHROW(MigrationError, ("No ways returned for point", someFeaturePoint));
if (!fid)
{
MYTHROW(MigrationError,
("None of returned ways suffice. Possibly, the feature has been deleted."));
}
return fid.get();
}
FeatureID MigrateFeatureIndex(osm::Editor::ForEachFeaturesNearByFn & forEach,
XMLFeature const & xml,
FeatureStatus const featureStatus,
GenerateIDFn const & generateID)
{
switch (xml.GetType())
{
case XMLFeature::Type::Unknown:
MYTHROW(MigrationError, ("Migration for XMLFeature::Type::Unknown is not possible"));
case XMLFeature::Type::Node:
return MigrateNodeFeatureIndex(forEach, xml, featureStatus, generateID);
case XMLFeature::Type::Way:
case XMLFeature::Type::Relation:
return MigrateWayOrRelatonFeatureIndex(forEach, xml, featureStatus, generateID);
}
UNREACHABLE();
}
void Editor::UploadChanges(string const & key, string const & secret, TChangesetTags tags,
TFinishUploadCallback callBack)
{
if (m_notes->NotUploadedNotesCount())
UploadNotes(key, secret);
if (!HaveMapEditsToUpload())
{
LOG(LDEBUG, ("There are no local edits to upload."));
return;
}
alohalytics::LogEvent("Editor_DataSync_started");
// TODO(AlexZ): features access should be synchronized.
auto const upload = [this](string key, string secret, TChangesetTags tags, TFinishUploadCallback callBack)
{
// This lambda was designed to start after app goes into background. But for cases when user is immediately
// coming back to the app we work with a copy, because 'for' loops below can take a significant amount of time.
auto features = m_features;
int uploadedFeaturesCount = 0, errorsCount = 0;
ChangesetWrapper changeset({key, secret}, tags);
for (auto & id : features)
{
for (auto & index : id.second)
{
FeatureTypeInfo & fti = index.second;
// Do not process already uploaded features or those failed permanently.
if (!NeedsUpload(fti.m_uploadStatus))
continue;
string ourDebugFeatureString;
try
{
switch (fti.m_status)
{
case FeatureStatus::Untouched:
CHECK(false, ("It's impossible."));
continue;
case FeatureStatus::Obsolete:
continue; // Obsolete features will be deleted by OSMers.
case FeatureStatus::Created:
{
XMLFeature feature = fti.m_feature.ToXML(true);
if (!fti.m_street.empty())
feature.SetTagValue(kAddrStreetTag, fti.m_street);
ourDebugFeatureString = DebugPrint(feature);
ASSERT_EQUAL(feature.GetType(), XMLFeature::Type::Node,
("Linear and area features creation is not supported yet."));
try
{
XMLFeature osmFeature = changeset.GetMatchingNodeFeatureFromOSM(fti.m_feature.GetCenter());
// If we are here, it means that object already exists at the given point.
// To avoid nodes duplication, merge and apply changes to it instead of creating an new one.
XMLFeature const osmFeatureCopy = osmFeature;
osmFeature.ApplyPatch(feature);
// Check to avoid uploading duplicates into OSM.
if (osmFeature == osmFeatureCopy)
{
LOG(LWARNING, ("Local changes are equal to OSM, feature has not been uploaded.", osmFeatureCopy));
// Don't delete this local change right now for user to see it in profile.
// It will be automatically deleted by migration code on the next maps update.
}
else
{
LOG(LDEBUG, ("Create case: uploading patched feature", osmFeature));
changeset.Modify(osmFeature);
}
}
catch (ChangesetWrapper::OsmObjectWasDeletedException const &)
{
// Object was never created by anyone else - it's safe to create it.
changeset.Create(feature);
}
catch (ChangesetWrapper::EmptyFeatureException const &)
{
// There is another node nearby, but it should be safe to create a new one.
changeset.Create(feature);
}
catch (...)
{
// Pass network or other errors to outside exception handler.
throw;
}
}
break;
case FeatureStatus::Modified:
{
// Do not serialize feature's type to avoid breaking OSM data.
// TODO: Implement correct types matching when we support modifying existing feature types.
XMLFeature feature = fti.m_feature.ToXML(false);
if (!fti.m_street.empty())
feature.SetTagValue(kAddrStreetTag, fti.m_street);
ourDebugFeatureString = DebugPrint(feature);
auto const originalFeaturePtr = GetOriginalFeature(fti.m_feature.GetID());
if (!originalFeaturePtr)
{
LOG(LERROR, ("A feature with id", fti.m_feature.GetID(), "cannot be loaded."));
alohalytics::LogEvent("Editor_MissingFeature_Error");
RemoveFeatureFromStorageIfExists(fti.m_feature.GetID());
continue;
}
XMLFeature osmFeature = GetMatchingFeatureFromOSM(
changeset, *originalFeaturePtr);
XMLFeature const osmFeatureCopy = osmFeature;
osmFeature.ApplyPatch(feature);
// Check to avoid uploading duplicates into OSM.
if (osmFeature == osmFeatureCopy)
{
LOG(LWARNING, ("Local changes are equal to OSM, feature has not been uploaded.", osmFeatureCopy));
// Don't delete this local change right now for user to see it in profile.
// It will be automatically deleted by migration code on the next maps update.
}
else
{
LOG(LDEBUG, ("Uploading patched feature", osmFeature));
changeset.Modify(osmFeature);
}
}
break;
case FeatureStatus::Deleted:
auto const originalFeaturePtr = GetOriginalFeature(fti.m_feature.GetID());
if (!originalFeaturePtr)
{
LOG(LERROR, ("A feature with id", fti.m_feature.GetID(), "cannot be loaded."));
alohalytics::LogEvent("Editor_MissingFeature_Error");
RemoveFeatureFromStorageIfExists(fti.m_feature.GetID());
continue;
}
changeset.Delete(GetMatchingFeatureFromOSM(
changeset, *originalFeaturePtr));
break;
}
fti.m_uploadStatus = kUploaded;
fti.m_uploadError.clear();
++uploadedFeaturesCount;
}
catch (ChangesetWrapper::OsmObjectWasDeletedException const & ex)
{
fti.m_uploadStatus = kDeletedFromOSMServer;
fti.m_uploadError = ex.what();
++errorsCount;
LOG(LWARNING, (ex.what()));
}
catch (ChangesetWrapper::RelationFeatureAreNotSupportedException const & ex)
{
fti.m_uploadStatus = kRelationsAreNotSupported;
fti.m_uploadError = ex.what();
++errorsCount;
LOG(LWARNING, (ex.what()));
}
catch (ChangesetWrapper::EmptyFeatureException const & ex)
{
fti.m_uploadStatus = kWrongMatch;
fti.m_uploadError = ex.what();
++errorsCount;
LOG(LWARNING, (ex.what()));
}
catch (RootException const & ex)
{
fti.m_uploadStatus = kNeedsRetry;
fti.m_uploadError = ex.what();
++errorsCount;
LOG(LWARNING, (ex.what()));
}
// TODO(AlexZ): Use timestamp from the server.
fti.m_uploadAttemptTimestamp = time(nullptr);
if (fti.m_uploadStatus != kUploaded)
{
ms::LatLon const ll = MercatorBounds::ToLatLon(feature::GetCenter(fti.m_feature));
alohalytics::LogEvent("Editor_DataSync_error", {{"type", fti.m_uploadStatus},
{"details", fti.m_uploadError}, {"our", ourDebugFeatureString},
{"mwm", fti.m_feature.GetID().GetMwmName()},
{"mwm_version", strings::to_string(fti.m_feature.GetID().GetMwmVersion())}
},
alohalytics::Location::FromLatLon(ll.lat, ll.lon));
}
// Call Save every time we modify each feature's information.
SaveUploadedInformation(fti);
}
}
alohalytics::LogEvent("Editor_DataSync_finished", {{"errors", strings::to_string(errorsCount)},
{"uploaded", strings::to_string(uploadedFeaturesCount)},
{"changeset", strings::to_string(changeset.GetChangesetId())}
});
if (callBack)
{
UploadResult result = UploadResult::NothingToUpload;
if (uploadedFeaturesCount)
result = UploadResult::Success;
else if (errorsCount)
result = UploadResult::Error;
callBack(result);
}
};
// Do not run more than one upload thread at a time.
static auto future = async(launch::async, upload, key, secret, tags, callBack);
auto const status = future.wait_for(milliseconds(0));
if (status == future_status::ready)
future = async(launch::async, upload, key, secret, tags, callBack);
}
void Editor::LoadMapEdits()
{
if (!m_delegate)
{
LOG(LERROR, ("Can't load any map edits, delegate has not been set."));
return;
}
xml_document doc;
if (!m_storage->Load(doc))
return;
array<pair<FeatureStatus, char const *>, 4> const sections =
{{
{FeatureStatus::Deleted, kDeleteSection},
{FeatureStatus::Modified, kModifySection},
{FeatureStatus::Obsolete, kObsoleteSection},
{FeatureStatus::Created, kCreateSection}
}
};
int deleted = 0, obsolete = 0, modified = 0, created = 0;
bool needRewriteEdits = false;
// TODO(mgsergio): synchronize access to m_features.
m_features.clear();
for (xml_node mwm : doc.child(kXmlRootNode).children(kXmlMwmNode))
{
string const mapName = mwm.attribute("name").as_string("");
int64_t const mapVersion = mwm.attribute("version").as_llong(0);
MwmSet::MwmId const mwmId = GetMwmIdByMapName(mapName);
// TODO(mgsergio, AlexZ): Is it normal to have isMwmIdAlive and mapVersion
// NOT equal to mwmId.GetInfo()->GetVersion() at the same time?
auto const needMigrateEdits = !mwmId.IsAlive() || mapVersion != mwmId.GetInfo()->GetVersion();
needRewriteEdits |= needMigrateEdits;
for (auto const & section : sections)
{
for (auto const nodeOrWay : mwm.child(section.second).select_nodes("node|way"))
{
try
{
XMLFeature const xml(nodeOrWay.node());
// TODO(mgsergio): A map could be renamed, we'll treat it as deleted.
// The right thing to do is to try to migrate all changes anyway.
if (!mwmId.IsAlive())
{
LOG(LINFO, ("Mwm", mapName, "was deleted"));
goto SECTION_END;
}
TForEachFeaturesNearByFn forEach = [this](TFeatureTypeFn && fn,
m2::PointD const & point) {
return ForEachFeatureAtPoint(move(fn), point);
};
// TODO(mgsergio): Deleted features are not properly handled yet.
auto const fid = needMigrateEdits
? editor::MigrateFeatureIndex(
forEach, xml, section.first,
[this, &mwmId] { return GenerateNewFeatureId(mwmId); })
: FeatureID(mwmId, xml.GetMWMFeatureIndex());
// Remove obsolete changes during migration.
if (needMigrateEdits && IsObsolete(xml, fid))
continue;
FeatureTypeInfo fti;
if (section.first == FeatureStatus::Created)
{
fti.m_feature.FromXML(xml);
}
else
{
auto const originalFeaturePtr = GetOriginalFeature(fid);
if (!originalFeaturePtr)
{
LOG(LERROR, ("A feature with id", fid, "cannot be loaded."));
alohalytics::LogEvent("Editor_MissingFeature_Error");
goto SECTION_END;
}
fti.m_feature = *originalFeaturePtr;
fti.m_feature.ApplyPatch(xml);
}
fti.m_feature.SetID(fid);
fti.m_street = xml.GetTagValue(kAddrStreetTag);
fti.m_modificationTimestamp = xml.GetModificationTime();
ASSERT_NOT_EQUAL(my::INVALID_TIME_STAMP, fti.m_modificationTimestamp, ());
fti.m_uploadAttemptTimestamp = xml.GetUploadTime();
fti.m_uploadStatus = xml.GetUploadStatus();
fti.m_uploadError = xml.GetUploadError();
fti.m_status = section.first;
switch (section.first)
{
case FeatureStatus::Deleted:
++deleted;
break;
case FeatureStatus::Modified:
++modified;
break;
case FeatureStatus::Obsolete:
++obsolete;
break;
case FeatureStatus::Created:
++created;
break;
case FeatureStatus::Untouched:
ASSERT(false, ());
continue;
}
// Insert initialized structure at the end: exceptions are possible in above code.
m_features[fid.m_mwmId].emplace(fid.m_index, move(fti));
}
catch (editor::XMLFeatureError const & ex)
{
ostringstream s;
nodeOrWay.node().print(s, " ");
LOG(LERROR, (ex.what(), "Can't create XMLFeature in section", section.second, s.str()));
}
catch (editor::MigrationError const & ex)
{
LOG(LWARNING, (ex.Msg(), "mwmId =", mwmId, XMLFeature(nodeOrWay.node())));
}
} // for nodes
} // for sections
SECTION_END:
;
} // for mwms
// Save edits with new indexes and mwm version to avoid another migration on next startup.
if (needRewriteEdits)
Save();
LOG(LINFO, ("Loaded", modified, "modified,",
created, "created,", deleted, "deleted and", obsolete, "obsolete features."));
}
void Editor::LoadMapEdits()
{
if (!m_mwmIdByMapNameFn)
{
LOG(LERROR, ("Can't load any map edits, MwmIdByNameAndVersionFn has not been set."));
return;
}
xml_document doc;
{
string const fullFilePath = GetEditorFilePath();
xml_parse_result const res = doc.load_file(fullFilePath.c_str());
// Note: status_file_not_found is ok if user has never made any edits.
if (res != status_ok && res != status_file_not_found)
{
LOG(LERROR, ("Can't load map edits from disk:", fullFilePath));
return;
}
}
array<pair<FeatureStatus, char const *>, 3> const sections =
{{
{FeatureStatus::Deleted, kDeleteSection},
{FeatureStatus::Modified, kModifySection},
{FeatureStatus::Created, kCreateSection}
}};
int deleted = 0, modified = 0, created = 0;
bool needRewriteEdits = false;
for (xml_node mwm : doc.child(kXmlRootNode).children(kXmlMwmNode))
{
string const mapName = mwm.attribute("name").as_string("");
int64_t const mapVersion = mwm.attribute("version").as_llong(0);
MwmSet::MwmId const mwmId = m_mwmIdByMapNameFn(mapName);
// TODO(mgsergio, AlexZ): Is it normal to have isMwmIdAlive and mapVersion
// NOT equal to mwmId.GetInfo()->GetVersion() at the same time?
auto const needMigrateEdits = !mwmId.IsAlive() || mapVersion != mwmId.GetInfo()->GetVersion();
needRewriteEdits |= needMigrateEdits;
for (auto const & section : sections)
{
for (auto const nodeOrWay : mwm.child(section.second).select_nodes("node|way"))
{
try
{
XMLFeature const xml(nodeOrWay.node());
auto const fid = needMigrateEdits
? editor::MigrateFeatureIndex(m_forEachFeatureAtPointFn, xml)
: FeatureID(mwmId, xml.GetMWMFeatureIndex());
// Remove obsolete edit during migration.
if (needMigrateEdits && IsObsolete(xml, fid))
continue;
FeatureTypeInfo fti;
if (section.first == FeatureStatus::Created)
{
fti.m_feature.FromXML(xml);
}
else
{
fti.m_feature = *m_getOriginalFeatureFn(fid);
fti.m_feature.ApplyPatch(xml);
}
fti.m_feature.SetID(fid);
fti.m_street = xml.GetTagValue(kAddrStreetTag);
fti.m_modificationTimestamp = xml.GetModificationTime();
ASSERT_NOT_EQUAL(my::INVALID_TIME_STAMP, fti.m_modificationTimestamp, ());
fti.m_uploadAttemptTimestamp = xml.GetUploadTime();
fti.m_uploadStatus = xml.GetUploadStatus();
fti.m_uploadError = xml.GetUploadError();
fti.m_status = section.first;
switch (section.first)
{
case FeatureStatus::Deleted: ++deleted; break;
case FeatureStatus::Modified: ++modified; break;
case FeatureStatus::Created: ++created; break;
case FeatureStatus::Untouched: ASSERT(false, ()); continue;
}
// Insert initialized structure at the end: exceptions are possible in above code.
m_features[fid.m_mwmId].emplace(fid.m_index, move(fti));
}
catch (editor::XMLFeatureError const & ex)
{
ostringstream s;
nodeOrWay.node().print(s, " ");
LOG(LERROR, (ex.what(), "Can't create XMLFeature in section", section.second, s.str()));
}
catch (editor::MigrationError const & ex)
{
LOG(LWARNING, (ex.Msg(), "mwmId =", mwmId, XMLFeature(nodeOrWay.node())));
}
} // for nodes
} // for sections
} // for mwms
// Save edits with new indexes and mwm version to avoid another migration on next startup.
if (needRewriteEdits)
Save(GetEditorFilePath());
LOG(LINFO, ("Loaded", modified, "modified,", created, "created and", deleted, "deleted features."));
}