void testAll()
{
srand(0);
OsmMap::resetCounters();
Settings::getInstance().clear();
conf().set(ConfigOptions().getUuidHelperRepeatableKey(), true);
conf().set(ConfigOptions().getUnifyOptimizerTimeLimitKey(), -1);
string outDir = "test-output/hadoop/HadoopTileWorkerTest/";
Hdfs fs;
if (fs.exists(outDir))
{
fs.deletePath(outDir, true);
}
fs.copyFromLocal("test-files/DcTigerRoads.pbf", outDir + "in1.pbf/DcTigerRoads.pbf");
fs.copyFromLocal("test-files/DcGisRoads.pbf", outDir + "in2.pbf/DcGisRoads.pbf");
shared_ptr<TileWorker> worker(new HadoopTileWorker());
TileConflator uut(worker);
// ~240m
uut.setBuffer(8.0 / 3600.0);
uut.setMaxNodesPerBox(5000);
uut.setSources(QString::fromStdString(outDir) + "in1.pbf",
QString::fromStdString(outDir) + "in2.pbf");
uut.conflate(QString::fromStdString(outDir) + "HadoopTileWorkerTest.pbf");
shared_ptr<OsmMap> map(new OsmMap);
PbfReader reader(true);
reader.setUseFileStatus(true);
std::vector<FileStatus> status = fs.listStatus(outDir + "HadoopTileWorkerTest.pbf", true);
for (size_t i = 0; i < status.size(); i++)
{
const string& path = status[i].getPath();
LOG_INFO(path);
if (QString::fromStdString(path).endsWith(".pbf"))
{
shared_ptr<istream> is(fs.open(path));
reader.parse(is.get(), map);
}
}
QDir().mkpath(QString::fromStdString(outDir));
OsmWriter writer;
writer.setIncludeHootInfo(true);
writer.write(map, QString::fromStdString(outDir + "/result.osm"));
HOOT_FILE_EQUALS("test-files/hadoop/HadoopTileWorkerTest/result.osm",
"test-output/hadoop/HadoopTileWorkerTest/result.osm");
}
void testAll()
{
string outDir = "test-output/hadoop/HadoopTileWorker2Test/";
Hdfs fs;
if (fs.exists(outDir))
{
fs.deletePath(outDir, true);
}
fs.copyFromLocal("test-files/DcTigerRoads.pbf", outDir + "in1.pbf/DcTigerRoads.pbf");
fs.copyFromLocal("test-files/DcGisRoads.pbf", outDir + "in2.pbf/DcGisRoads.pbf");
shared_ptr<TileWorker2> worker(new HadoopTileWorker2());
FourPassManager uut(worker);
// ~240m
uut.setBuffer(8.0 / 3600.0);
uut.setMaxNodesPerBox(5000);
uut.setSources(QString::fromStdString(outDir) + "in1.pbf",
QString::fromStdString(outDir) + "in2.pbf");
Envelope env(-77.039, -77.033, 38.892, 38.896);
shared_ptr<OpList> op(new OpList());
op->addOp(shared_ptr<OsmMapOperation>(new MapCropper(env)));
op->addOp(shared_ptr<OsmMapOperation>(new MergeNearbyNodes(10)));
uut.setOperation(op);
uut.apply(QString::fromStdString(outDir) + "HadoopTileWorker2Test.pbf");
shared_ptr<OsmMap> map(new OsmMap);
PbfReader reader(true);
reader.setUseFileStatus(true);
std::vector<FileStatus> status = fs.listStatus(outDir + "HadoopTileWorker2Test.pbf");
for (size_t i = 0; i < status.size(); i++)
{
const string& path = status[i].getPath();
LOG_INFO(path);
if (QString::fromStdString(path).endsWith(".pbf"))
{
shared_ptr<istream> is(fs.open(path));
reader.parse(is.get(), map);
}
}
QDir().mkpath(QString::fromStdString(outDir));
OsmWriter writer;
writer.setIncludeHootInfo(true);
writer.write(map, QString::fromStdString(outDir + "/result.osm"));
HOOT_FILE_EQUALS("test-files/hadoop/HadoopTileWorker2Test/result.osm",
"test-output/hadoop/HadoopTileWorker2Test/result.osm");
}
void testBasics()
{
string outDir = "test-output/WayJoin2InputFormat";
string pbfIn = outDir + "/SmallSplits.pbf";
string csqIn = outDir + "/sample.csq";
Hdfs fs;
if (fs.exists(pbfIn) == false)
{
fs.copyFromLocal("test-files/io/SmallSplits.pbf", pbfIn);
}
// create a sequence file that looks like the output of WayJoin1Reducer
WayJoin1Reducer::Value v;
int64_t wid;
boost::shared_ptr<ostream> sample(fs.create(csqIn));
pp::CppSeqFile::Writer writer(*sample);
v.nodeId = 1;
v.x = -104;
v.y = 38;
wid = 2;
writer.appendFixed<int64_t, WayJoin1Reducer::Value>(wid, v);
v.nodeId = 3;
v.x = -104;
v.y = 38;
wid = 4;
writer.appendFixed<int64_t, WayJoin1Reducer::Value>(wid, v);
writer.close();
sample.reset();
// create an input format
WayJoin2InputFormat uut;
uut.setPath(pbfIn + "," + csqIn);
// verify that the input splits look as expected.
CPPUNIT_ASSERT_EQUAL(2, uut.getSplitCount());
CPPUNIT_ASSERT_EQUAL(pbfIn, uut.getSplit(0).getPath());
CPPUNIT_ASSERT_EQUAL((long long)1125, uut.getSplit(0).getStart());
CPPUNIT_ASSERT_EQUAL((long long)13490, uut.getSplit(0).getLength());
CPPUNIT_ASSERT_EQUAL(csqIn, uut.getSplit(1).getPath());
CPPUNIT_ASSERT_EQUAL((long long)0, uut.getSplit(1).getStart());
CPPUNIT_ASSERT_EQUAL((long long)100, uut.getSplit(1).getLength());
}
void init(const string& outDir, const QString outFile)
{
if (QFile::exists(outFile))
{
QFile::remove(outFile);
}
Hdfs fs;
if (fs.exists(outDir))
{
fs.deletePath(outDir, true);
}
QDir().mkpath(QString::fromStdString(outDir));
fs.copyFromLocal(
/*"test-files/DcGisRoads.pbf"*/
"test-files/conflate/unified/AllDataTypesA.osm.pbf",
outDir + "/input.osm.pbf");
//init db
ServicesDbTestUtils::deleteDataFromOsmApiTestDatabase();
const QString scriptDir = "test-files/servicesdb";
ApiDb::execSqlFile(ServicesDbTestUtils::getOsmApiDbUrl().toString(), scriptDir + "/users.sql");
}
void testJob()
{
LOG_INFO("Starting.");
string outDir = "test-output/hadoop/PaintNodesDriverTest/";
Hdfs fs;
if (fs.exists(outDir))
{
fs.deletePath(outDir, true);
}
fs.copyFromLocal("test-files/io/SmallSplits.pbf", outDir + "input/SmallSplits.pbf");
Envelope e(-180, 180, -90, 90);
double pixelSize = 0.1;
{
PaintNodesDriver uut;
const cv::Mat& m = uut.calculateDensity(e, pixelSize, QString::fromStdString(outDir) + "input");
CPPUNIT_ASSERT_EQUAL(3601, m.cols);
CPPUNIT_ASSERT_EQUAL(1801, m.rows);
//CPPUNIT_ASSERT_EQUAL(9, m.ptr<int>(1288)[750]);
//CPPUNIT_ASSERT_EQUAL(27, m.ptr<int>(1288)[751]);
long sum = 0;
for (int py = 0; py < m.rows; py++)
{
const int32_t* row1 = m.ptr<int32_t>(py);
for (int px = 0; px < m.cols; px++)
{
sum += row1[px];
if (row1[px] > 0)
{
LOG_INFO("px: " << px << " py: " << py << " v: " << row1[px]);
}
}
}
CPPUNIT_ASSERT_EQUAL(36l, sum);
}
// does it work when cached?
{
PaintNodesDriver uut;
const cv::Mat& m = uut.calculateDensity(e, pixelSize, QString::fromStdString(outDir) + "input");
CPPUNIT_ASSERT_EQUAL(3601, m.cols);
CPPUNIT_ASSERT_EQUAL(1801, m.rows);
//CPPUNIT_ASSERT_EQUAL(9, m.ptr<int>(1288)[750]);
//CPPUNIT_ASSERT_EQUAL(27, m.ptr<int>(1288)[751]);
long sum = 0;
for (int py = 0; py < m.rows; py++)
{
const int32_t* row1 = m.ptr<int32_t>(py);
for (int px = 0; px < m.cols; px++)
{
sum += row1[px];
if (row1[px] > 0)
{
LOG_INFO("px: " << px << " py: " << py << " v: " << row1[px]);
}
}
}
CPPUNIT_ASSERT_EQUAL(36l, sum);
}
}