void Log(ELogLevel const & lv, const char * tag, const char * logLine) override
{
{
lock_guard<mutex> scopeLock(m_LogMutex);
static char sCurBuffer[4096] = { 0 }; // 4K buffer less than websocket buffer size
snprintf(sCurBuffer, 4096, "[%s]@[%s]:%s", GetLocalTime(), Os::Thread::GetCurrentThreadName().c_str(), logLine);
m_Logs.push({ sCurBuffer, tag, lv });
}
m_CV.notify_one();
}
void ThreadDB::setThreadCreated(pthread_t id) {
ScopeLock scopeLock(_mutex);
_map.erase(id); // not strictly necessary
_map.insert(std::make_pair(id, RUNNING));
if (LOG_DEBUG) {
std::cerr << "Thread " << id << " created" << std::endl;
}
}
// Get the context for a single job from the queue.
Context* JobQueue::NextJob()
{
std::unique_lock<std::mutex> scopeLock(lock);
conditionVariable.wait(scopeLock, [this] { return nextJob != nextSlot; });
// Unlock once we get the index; we can mask off without it.
size_t currentJob = nextJob++;
scopeLock.unlock();
nextJob &= QueueIndexMask;
return queue[currentJob];
}
void rspfPixelFlipper::setTargetRange(rspf_float64 target_min, rspf_float64 target_max)
{
//---
// Since this is the value to replace we will allow for any value as it
// won't affect the output null, min and max ranges. This will fix a
// tiled nitf with max of 2047(11bit) with edge tile fill values of 2048.
//---
OpenThreads::ScopedLock<OpenThreads::Mutex> scopeLock(theMutex);
theTargetValueLo = target_min;
theTargetValueHi = target_max;
}
WebSocketLogger() : net::WebSocket()
{
m_Thread = new Os::Thread([this]()->void {
this->BindAndListen();
Os::IPv4Address unnamedClient("");
Os::SocketHandle client;
while (true)
{
client = this->Accept(unnamedClient);
if (client == -1)
{
Os::Sleep(1000);
continue;
}
char buffer[BUF_LEN + 1] = { 0 };
uint64 recvLen = this->Receive(client, buffer, BUF_LEN);
if (recvLen > 0 && recvLen < BUF_LEN)
{
//OutputDebugStringA(buffer);
}
while (true)
{
bool canQuit = false;
if (m_Logs.empty())
{
unique_lock<std::mutex> uSignal(m_Signal);
m_CV.wait(uSignal);
}
else
{
lock_guard<mutex> scopeLock(m_LogMutex);
while (!m_Logs.empty())
{
LogItem log = m_Logs.front();
string output = log.JsonStr();
uint64 sent = Send(client, output.data(), (uint32)output.size());
m_Logs.pop();
if (sent <= 0)
canQuit = true;
}
}
if (canQuit)
{
break;
}
}
}
}, "WebsocketLogger");
m_Thread->Start();
}
// Add a context back on the queue.
void JobQueue::Queue(Context& context)
{
std::unique_lock<std::mutex> scopeLock(lock);
// Push and mask index to wrap around.
size_t currentSlot = nextSlot++;
nextSlot &= QueueIndexMask;
queue[currentSlot] = &context;
// Unlock before notifying to avoid having the waiting thread sleep on mutex immediately.
scopeLock.unlock();
conditionVariable.notify_one();
}
Logger*
Logger::Instance(void)
{
static Logger* ptr = 0;
if (!ptr) {
static Mutex mutex;
ScopeLock scopeLock(mutex);
if (!ptr) {
// Create new instance ...
ptr = new Logger(&std::cerr);
}
}
return ptr;
}
int CClientSession::AddSockIdByService(const string& aService, int aSockId) {
CLock scopeLock(iAddrMutex);
DEBUG_FUNC_NAME("CClientSession::AddSockIdByService");
DEBUG(LL_ALL, "%s:Begin.", funcName);
typeof(iService2SockId.end()) it = iService2SockId.find(aService);
if (it != iService2SockId.end()) {
it->second.push(aSockId);
} else {
std::queue<int> sockQueue;
sockQueue.push(aSockId);
iService2SockId[aService] = sockQueue;
}
DEBUG(LL_ALL, "%s:End.", funcName);
return RET_OK;
}
void
FiberSemaphore::wait()
{
MORDOR_ASSERT(Scheduler::getThis());
{
boost::mutex::scoped_lock scopeLock(m_mutex);
MORDOR_ASSERT(std::find(m_waiters.begin(), m_waiters.end(),
std::make_pair(Scheduler::getThis(), Fiber::getThis()))
== m_waiters.end());
if (m_concurrency > 0u) {
--m_concurrency;
return;
}
m_waiters.push_back(std::make_pair(Scheduler::getThis(),
Fiber::getThis()));
}
Scheduler::yieldTo();
#ifdef DEBUG
boost::mutex::scoped_lock scopeLock(m_mutex);
MORDOR_ASSERT(std::find(m_waiters.begin(), m_waiters.end(),
std::make_pair(Scheduler::getThis(), Fiber::getThis()))
== m_waiters.end());
#endif
}
void ThreadDB::setThreadState(pthread_t id, rilakkuma_thread_state state) {
ScopeLock scopeLock(_mutex);
// If we didn't see the thread get created, we ignore any updates to its state.
// This keeps us from using the main thread, the MultithreadedTestRunner's background
// thread, and any other threads created outside of the test suite in our wait for
// tick algorithm.
stateMap::iterator it = _map.find(id);
if (it == _map.end()) {
return;
}
_map.erase(it);
_map.insert(std::make_pair(id, state));
}
void OsmAnd::OnlineMapRasterTileProvider_P::obtainTileDeffered( const std::shared_ptr<TileEntry>& tileEntry )
{
if(!_networkAccessAllowed)
return;
{
QMutexLocker scopedLock(&_currentDownloadsCounterMutex);
QWriteLocker scopeLock(&tileEntry->stateLock);
if(tileEntry->state != TileState::LocalLookup && tileEntry->state != TileState::EnqueuedForDownload)
return;
// If all download slots are taken, enqueue
if(_currentDownloadsCount == owner->maxConcurrentDownloads)
{
tileEntry->state = TileState::EnqueuedForDownload;
return;
}
// Else, simply increment counter
_currentDownloadsCount++;
tileEntry->state = TileState::Downloading;
}
Concurrent::pools->network->start(new Concurrent::HostedTask(_taskHostBridge,
[tileEntry](const Concurrent::Task* task, QEventLoop& eventLoop)
{
const auto pThis = reinterpret_cast<OnlineMapRasterTileProvider_P*>(static_cast<const Concurrent::HostedTask*>(task)->lockedOwner);
QNetworkAccessManager networkAccessManager;
QNetworkRequest request;
request.setUrl(tileEntry->sourceUrl);
request.setRawHeader("User-Agent", "OsmAnd Core");
auto reply = networkAccessManager.get(request);
QObject::connect(reply, &QNetworkReply::finished,
[pThis, reply, tileEntry, &eventLoop, &networkAccessManager]()
{
pThis->replyFinishedHandler(reply, tileEntry, eventLoop, networkAccessManager);
});
eventLoop.exec();
return;
}));
}
void checkQueue()
{
{
MyScopeLock scopeLock(mLock);
while(!mClientTaskQueue.empty())
{
MySockClientTask *task = mClientTaskQueue.front();
mClientTaskQueue.pop();
mClientTaskQueueTmp.push(task);
}
}
while(!mClientTaskQueueTmp.empty())
{
MySockClientTask *task = mClientTaskQueueTmp.front();
add(task);
mClientTaskQueueTmp.pop();
}
}
// ---------------------------------------------------------------------------
// int CClientSession::GetSockIdByService
//
// getting socket id by service from map.
// ---------------------------------------------------------------------------
//
int CClientSession::GetSockIdByService(const string& aService) {
CLock scopeLock(iAddrMutex);
DEBUG_FUNC_NAME("CClientSession::GetSockIdByService");
DEBUG(LL_ALL, "%s:Begin.", funcName);
int retSockId = RET_ERROR;
typeof(iService2SockId.end()) it = iService2SockId.find(aService);
if (it != iService2SockId.end()) {
if (!it->second.empty()) {
retSockId = it->second.front();
it->second.pop();
}
}
if (retSockId == RET_ERROR) {
retSockId = ReNewService(aService);
}
LOG(LL_DBG, "%s:service:(%s),sockid:(%d).",
funcName, aService.c_str(), retSockId);
DEBUG(LL_ALL, "%s:End", funcName);
return retSockId;
}
bool OsmAnd::TileDB::openIndex()
{
QMutexLocker scopeLock(&_indexMutex);
bool ok;
bool shouldRebuild = indexFilename.isEmpty() || !QFile(indexFilename).exists();
_indexDb.setDatabaseName(indexFilename.isEmpty() ? ":memory:" : indexFilename);
ok = _indexDb.open();
if(!ok)
{
LogPrintf(LogSeverityLevel::Error, "Failed to open TileDB index from '%s': %s", qPrintable(indexFilename), qPrintable(_indexDb.lastError().text()));
return false;
}
if(shouldRebuild)
rebuildIndex();
return true;
}
virtual bool Run() override
{
while (1)
{
ULONG loopInterval = UpdateInterval(TaskWorkerInterval);
if (CheckKillEvent(loopInterval))
{
// Kill Event signaled
break;
}
SharedObjectType* sharedPtr = nullptr;
BR::SharedPointerT<SharedObjectType> localShared;
{
BR::MutexScopeLock scopeLock(stm_StorageLock);
long randID = rand() % MAX_TEST_ID;
if (stm_PointerStorage[randID] == nullptr)
{
sharedPtr = new SharedObjectType(randID);
m_referenceManager.RegisterSharedObject(sharedPtr);
localShared = BR::SharedPointerT<SharedObjectType>(sharedPtr);
stm_PointerStorage[randID] = localShared;
}
else
{
localShared = stm_PointerStorage[randID];
auto usedCount = localShared->pItem->UsedCount.fetch_add(1,std::memory_order_relaxed)+1;
if (usedCount > MAX_ACCESS_COUNT)
stm_PointerStorage[randID] = BR::SharedPointerT<SharedObjectType>();
}
}
// Pretend to use
Sleep(0);
}
return true;
}
void rspfPixelFlipper::setClampValue(rspf_float64 clamp_value, bool clamp_max_value)
{
OpenThreads::ScopedLock<OpenThreads::Mutex> scopeLock(theMutex);
if (inRange(clamp_value))
{
// Stupid MS compiler complains if we do an |= on enum type. (OLK 1/11)
int temp_int = (int) theClampingMode;
if (clamp_max_value)
{
theClampValueHi = clamp_value;
temp_int |= (int) CLAMPING_HI;
}
else
{
theClampValueLo = clamp_value;
temp_int |= (int) CLAMPING_LO;
}
theClampingMode = (ClampingMode) temp_int;
}
}
ILogger* GetLogger(ELoggerType const& type) override {
if (m_pLoggers.find(type) == m_pLoggers.end()) {
if (!m_pLoggers[type]) {
lock_guard<mutex> scopeLock(m_CreateMutex);
if (!m_pLoggers[type]) {
switch (type) {
case ELoggerType::EConsole:
m_pLoggers[type] = new ConsoleLogger;
break;
case ELoggerType::EFile:
m_pLoggers[type] = new FileLogger;
break;
case ELoggerType::EWebsocket:
m_pLoggers[type] = new WebSocketLogger;
break;
}
}
}
}
return m_pLoggers[type];
}
bool rspfPixelFlipper::setReplacementMode(const rspfString& modeString)
{
OpenThreads::ScopedLock<OpenThreads::Mutex> scopeLock(theMutex);
rspfString mode = modeString;
mode.upcase();
if (mode == "REPLACE_BAND_IF_TARGET")
{
theReplacementMode = REPLACE_BAND_IF_TARGET;
}
else if (mode == "REPLACE_BAND_IF_PARTIAL_TARGET")
{
theReplacementMode = REPLACE_BAND_IF_PARTIAL_TARGET;
}
else if (mode == "REPLACE_ALL_BANDS_IF_PARTIAL_TARGET")
{
theReplacementMode = REPLACE_ALL_BANDS_IF_PARTIAL_TARGET;
}
else if (mode == "REPLACE_ONLY_FULL_TARGETS")
{
theReplacementMode = REPLACE_ONLY_FULL_TARGETS;
}
else if (mode == "REPLACE_ALL_BANDS_IF_ANY_TARGET")
{
theReplacementMode = REPLACE_ALL_BANDS_IF_ANY_TARGET;
}
else
{
rspfNotify(rspfNotifyLevel_WARN)
<< "rspfPixelFlipper::setReplacementMode warning:\n"
<< "Invalid mode: " << modeString
<< endl;
return false;
}
return true;
}
bool OsmAnd::TileDB::obtainTileData( const TileId& tileId, const ZoomLevel& zoom, QByteArray& data )
{
QMutexLocker scopeLock(&_indexMutex);
// Check that index is available
if(!_indexDb.isOpen())
{
if(!openIndex())
return false;
}
QSqlQuery query(_indexDb);
query.prepare("SELECT filename FROM tiledb_files WHERE id IN (SELECT id FROM tiledb_index WHERE xMin<=? AND xMax>=? AND yMin<=? AND yMax>=? AND zoom=?)");
query.addBindValue(tileId.x);
query.addBindValue(tileId.x);
query.addBindValue(tileId.y);
query.addBindValue(tileId.y);
query.addBindValue(zoom);
if(!query.exec())
return false;
bool hit = false;
while(!hit && query.next())
{
const auto dbFilename = query.value(0).toString();
//TODO: manage access to database
//QMutexLocker scopeLock(&dbEntry->mutex);
// Open database
const auto connectionName = QString::fromLatin1("tiledb-sqlite:") + dbFilename;
QSqlDatabase db;
if(!QSqlDatabase::contains(connectionName))
db = QSqlDatabase::addDatabase("QSQLITE", connectionName);
else
db = QSqlDatabase::database(connectionName);
db.setDatabaseName(dbFilename);
if(!db.open())
{
LogPrintf(LogSeverityLevel::Error, "Failed to open TileDB index from '%s': %s", qPrintable(dbFilename), qPrintable(db.lastError().text()));
return false;
}
// Get tile from
QSqlQuery query(db);
query.prepare("SELECT data FROM tiles WHERE x=? AND y=? AND zoom=?");
query.addBindValue(tileId.x);
query.addBindValue(tileId.y);
query.addBindValue(static_cast<int>(zoom));
if(query.exec() && query.next())
{
data = query.value(0).toByteArray();
hit = true;
}
// Close database
db.close();
}
return hit;
}
// this should not throw exception since it is called often in a
// catch block to log an error resulting from an exception.
void Log::vlog(ModuleId module, Level level, const char *format,
std::va_list args)
throw()
{
if (initialized) {
if (module >= moduleList->size()) {
module = ALL_MODULES;
}
char *logMsg = PR_vsmprintf(format, args);
// call user defined logger if any.
if (loggerFunc != NULL) {
loggerFunc((*moduleList)[module].name.c_str(),
static_cast<am_log_level_t>(static_cast<int>(level)),
logMsg);
}
// do default log.
if ((*moduleList)[module].level >= level) {
// format:
// year-month-day hour:min:sec.usec level pid:thread module: msg
// get level string
std::size_t levelLabelIndex = getLevelString(level);
char levelStr[50];
PRUint32 llen;
if (levelLabelIndex < numLabels) {
llen = PR_snprintf(levelStr, sizeof(levelStr),
"%s", levelLabels[levelLabelIndex]);
} else {
llen = PR_snprintf(levelStr, sizeof(levelStr), "%d", level);
}
if (llen > 0) {
// get time.
PRExplodedTime now;
PR_ExplodeTime(PR_Now(), PR_LocalTimeParameters, &now);
// format header and msg.
PRUint32 len;
char hdr[100];
len = PR_snprintf(hdr, sizeof(hdr),
"%d-%02d-%02d %02d:%02d:%02d.%03d"
"%8s %u:%p %s: %%s\n",
now.tm_year, now.tm_month+1, now.tm_mday,
now.tm_hour, now.tm_min, now.tm_sec,
now.tm_usec / 1000,
levelStr,
getpid(), PR_GetCurrentThread(),
(*moduleList)[module].name.c_str());
if (len > 0) {
if (logRotation) {
if ((currentLogFileSize + 1000) < maxLogFileSize) {
std::fprintf(logFile, hdr, logMsg);
std::fflush(logFile);
} else {
ScopeLock scopeLock(*lockPtr);
currentLogFileSize = ftell(logFile);
if ((currentLogFileSize + 1000) > maxLogFileSize) {
// Open a new log file
if (!pSetLogFile(logFileName)) {
log(ALL_MODULES, LOG_ERROR,
"Unable to open log file: '%s', errno = %d",
logFileName.c_str(), errno);
}
}
std::fprintf(logFile, hdr, logMsg);
std::fflush(logFile);
}
currentLogFileSize = ftell(logFile);
} else {
std::fprintf(logFile, hdr, logMsg);
std::fflush(logFile);
}
}
}
}
// Remote Logging starts here.
if (module == remoteModule) {
if (remoteInitialized) {
bool doLogRemotely =
((*moduleList)[module].level >= Log::LOG_AUTH_REMOTE) &&
((*moduleList)[module].level & level);
if (doLogRemotely) {
am_status_t status;
status = rmtLogSvc->logMessage(logMsg);
if(status != AM_SUCCESS) {
Log::log(Log::ALL_MODULES, Log::LOG_ERROR,
"Log::vlog(): Error logging message [%s] "
"to remote server. Error: %s.",
logMsg, am_status_to_string(status));
}
}
} else {
Log::log(Log::ALL_MODULES, Log::LOG_ERROR,
"Log::vlog(): Remote logging service not initialized. "
"Cannot log message to remote server.");
}
}
PR_smprintf_free(logMsg);
}
return;
}
inline void forceQuit() // makes all waiting and subsequent calls to dequeue methods return failure,
{ // effectively forcing all waiting threads to quit if a program abort is received
std::lock_guard<std::mutex> scopeLock(queueAccessMutex);
shouldQuit = true;
queueAccessCondVar.notify_all();
}
inline uint size()
{
std::lock_guard<std::mutex> scopeLock(queueAccessMutex);
return q.size();
}
inline void pop()
{
std::lock_guard<std::mutex> scopeLock(queueAccessMutex);
q.pop();
}
inline const T & peek()
{
std::lock_guard<std::mutex> scopeLock(queueAccessMutex);
return q.front();
}
inline void enqueue(const T & elem)
{
std::lock_guard<std::mutex> scopeLock(queueAccessMutex);
q.push(elem);
queueAccessCondVar.notify_one();
}
void rspfPixelFlipper::setClipMode(rspfPixelFlipper::ClipMode mode)
{
OpenThreads::ScopedLock<OpenThreads::Mutex> scopeLock(theMutex);
theClipMode = mode;
}
am_status_t LogService::flushBuffer()
throw()
{
ScopeLock scopeLock(mLock);
if(bufferCount <= 0 || !remoteBodyChunkListInitialized) {
return AM_SUCCESS;
}
am_status_t status = AM_FAILURE;
remoteBodyChunkList.push_back(requestSetSuffixChunk);
Http::Response response;
status = doHttpPost(serviceInfo, std::string(), cookieList,
remoteBodyChunkList, response);
if (status == AM_SUCCESS) {
try {
std::vector<std::string> loggingResponses;
if (remoteRequest != NULL) {
loggingResponses =
parseGenericResponse(response,
remoteRequest->getGlobalId());
}
status = AM_ERROR_PARSING_XML;
if (loggingResponses.empty()) {
status = AM_ERROR_PARSING_XML;
// logging response is empty
} else {
status = AM_SUCCESS;
// What if there are more than one logging response ?
// status is success only if all responses are success.
// otherwise set status to the first error encountered.
for (std::size_t i = 0; i < loggingResponses.size(); ++i) {
if (strstr(loggingResponses[i].c_str(), "OK")) {
continue;
}
else if (strstr(loggingResponses[i].c_str(), "ERROR")) {
status = AM_REMOTE_LOG_FAILURE;
break;
}
else if (strstr(loggingResponses[i].c_str(),
"INVALID_SESSION")) {
status = AM_ACCESS_DENIED;
break;
}
else if (strstr(loggingResponses[i].c_str(),
"UNAUTHORIZED")) {
status = AM_ACCESS_DENIED;
break;
}
else {
// unknown response.
status = AM_ERROR_PARSING_XML;
break;
}
}
}
} catch (const XMLTree::ParseException& exc) {
Log::log(logModule, Log::LOG_ERROR,
"LogService::flushBuffer() caught exception: %s",
exc.getMessage().c_str());
status = AM_ERROR_PARSING_XML;
} catch (std::exception& exs) {
Log::log(logModule, Log::LOG_ERROR,
"LogService::flushBuffer() caught exception: %s",
exs.what());
status = AM_ERROR_PARSING_XML;
} catch (...) {
Log::log(logModule, Log::LOG_ERROR,
"LogService::flushBuffer() caught unknown exception.");
status = AM_ERROR_PARSING_XML;
}
}
bufferCount = 0;
remoteBodyChunkListInitialized = false;
remoteBodyChunkList.clear();
if (remoteRequest != NULL) {
delete remoteRequest;
remoteRequest = NULL;
}
return status;
}
rspfRefPtr<rspfProperty> rspfPixelFlipper::getProperty(
const rspfString& name)const
{
// Lock for the length of this method.
OpenThreads::ScopedLock<OpenThreads::Mutex> scopeLock(theMutex);
if (name == PF_TARGET_VALUE_KW)
{
rspfProperty* p =
new rspfNumericProperty(name,
rspfString::toString(theTargetValueLo));
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
if (name == PF_TARGET_RANGE_KW)
{
rspfProperty* p =
new rspfNumericProperty(name,
rspfString::toString(theTargetValueLo));
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
if (name == TARGET_UPPER_LIMIT_PROP_NAME)
{
rspfProperty* p =
new rspfNumericProperty(name,
rspfString::toString(theTargetValueHi));
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
else if (name == PF_REPLACEMENT_VALUE_KW)
{
rspfProperty* p =
new rspfNumericProperty(name,
rspfString::toString(theReplacementValue));
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
else if (name == PF_CLAMP_VALUE_LO_KW)
{
rspfProperty* p =
new rspfNumericProperty(name, rspfString::toString(theClampValueLo));
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
else if (name == PF_CLAMP_VALUE_HI_KW)
{
rspfProperty* p =
new rspfNumericProperty(name, rspfString::toString(theClampValueHi));
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
else if (name == PF_REPLACEMENT_MODE_KW)
{
vector<rspfString> constraintList(5);
constraintList[0] = "REPLACE_BAND_IF_TARGET";
constraintList[1] = "REPLACE_BAND_IF_PARTIAL_TARGET";
constraintList[2] = "REPLACE_ALL_BANDS_IF_PARTIAL_TARGET";
constraintList[3] = "REPLACE_ONLY_FULL_TARGETS";
constraintList[4] = "REPLACE_ALL_BANDS_IF_ANY_TARGET";
rspfStringProperty* p =
new rspfStringProperty(name,
getReplacementModeString(),
false,
constraintList);
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
else if (name == PF_CLIP_MODE_KW)
{
vector<rspfString> constraintList(3);
constraintList[0] = "none";
constraintList[1] = "bounding_rect";
constraintList[2] = "valid_vertices";
rspfStringProperty* p =
new rspfStringProperty(name,
getClipModeString(),
false,
constraintList);
p->setCacheRefreshBit();
return rspfRefPtr<rspfProperty>(p);
}
rspfRefPtr<rspfProperty> result = rspfSource::getProperty(name);
if(result.valid())
{
if(result->getName() == rspfKeywordNames::ENABLED_KW)
{
result->clearChangeType();
// we will at least say its a radiometric change
//
result->setCacheRefreshBit();
}
}
return result;
}
bool OsmAnd::TileDB::rebuildIndex()
{
QMutexLocker scopeLock(&_indexMutex);
bool ok;
// Open index database if it's not yet
if(!_indexDb.isOpen())
{
if(!openIndex())
return false;
}
QSqlQuery q(_indexDb);
LogPrintf(LogSeverityLevel::Info, "Rebuilding index of '%s' tiledb...", dataPath.absolutePath().toStdString().c_str());
auto beginTimestamp = std::chrono::steady_clock::now();
// Recreate index db structure
if(!indexFilename.isEmpty())
{
ok = q.exec("DROP TABLE IF EXISTS tiledb_files");
assert(ok);
ok = q.exec("DROP TABLE IF EXISTS tiledb_index");
assert(ok);
ok = q.exec("DROP INDEX IF EXISTS _tiledb_index");
assert(ok);
}
ok = q.exec(
"CREATE TABLE tiledb_files ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" filename TEXT"
")");
assert(ok);
ok = q.exec(
"CREATE TABLE tiledb_index ("
" xMin INTEGER,"
" yMin INTEGER,"
" xMax INTEGER,"
" yMax INTEGER,"
" zoom INTEGER,"
" id INTEGER"
")");
assert(ok);
ok = q.exec(
"CREATE INDEX _tiledb_index"
" ON tiledb_index(xMin, yMin, xMax, yMax, zoom)");
assert(ok);
_indexDb.commit();
QSqlQuery registerFileQuery(_indexDb);
ok = registerFileQuery.prepare("INSERT INTO tiledb_files (filename) VALUES ( ? )");
assert(ok);
QSqlQuery insertTileQuery(_indexDb);
ok = insertTileQuery.prepare("INSERT INTO tiledb_index (xMin, yMin, xMax, yMax, zoom, id) VALUES ( ?, ?, ?, ?, ?, ? )");
assert(ok);
// Index TileDBs
QFileInfoList files;
Utilities::findFiles(dataPath, QStringList() << "*", files);
for(auto itFile = files.cbegin(); itFile != files.cend(); ++itFile)
{
const auto& file = *itFile;
const auto dbFilename = file.absoluteFilePath();
const auto connectionName = QString::fromLatin1("tiledb-sqlite:") + dbFilename;
QSqlDatabase db;
if(!QSqlDatabase::contains(connectionName))
db = QSqlDatabase::addDatabase("QSQLITE", connectionName);
else
db = QSqlDatabase::database(connectionName);
db.setDatabaseName(dbFilename);
if(!db.open())
{
LogPrintf(LogSeverityLevel::Error, "Failed to open TileDB from '%s': %s", qPrintable(dbFilename), qPrintable(db.lastError().text()));
continue;
}
// Register new file
registerFileQuery.addBindValue(dbFilename);
ok = registerFileQuery.exec();
assert(ok);
auto fileId = registerFileQuery.lastInsertId();
// For each zoom, query min-max of tile coordinates
QSqlQuery minMaxQuery("SELECT zoom, xMin, yMin, xMax, yMax FROM bounds", db);
ok = minMaxQuery.exec();
assert(ok);
while(minMaxQuery.next())
{
int32_t zoom = minMaxQuery.value(0).toInt();
int32_t xMin = minMaxQuery.value(1).toInt();
int32_t yMin = minMaxQuery.value(2).toInt();
int32_t xMax = minMaxQuery.value(3).toInt();
int32_t yMax = minMaxQuery.value(4).toInt();
insertTileQuery.addBindValue(xMin);
insertTileQuery.addBindValue(yMin);
insertTileQuery.addBindValue(xMax);
insertTileQuery.addBindValue(yMax);
insertTileQuery.addBindValue(zoom);
insertTileQuery.addBindValue(fileId);
ok = insertTileQuery.exec();
assert(ok);
}
db.close();
}
_indexDb.commit();
auto endTimestamp = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast< std::chrono::duration<uint64_t, std::milli> >(endTimestamp - beginTimestamp).count();
LogPrintf(LogSeverityLevel::Info, "Finished indexing '%s', took %lldms, average %lldms/db", dataPath.absolutePath().toStdString().c_str(), duration, duration / files.length());
return true;
}
am_status_t LogService::addLogDetails(const std::string& logName,
const LogRecord& record,
const std::string& loggedByTokenID)
{
ScopeLock scopeLock(mLock);
char logLevel[32];
am_status_t status = AM_SUCCESS;
char *msg = NULL;
std::string message = record.getLogMessage();
//The encoded log message needs to be in multiple of 4 bytes.
msg = (char *)malloc(((message.size() * 4/3 + 1)/4+1)*4 + 4);
if(msg != NULL) {
encode_base64(message.c_str(), message.size(), msg);
} else {
status = AM_NO_MEMORY;
}
if (status == AM_SUCCESS) {
if(!remoteBodyChunkListInitialized) {
const std::size_t NUM_EXTRA_CHUNKS = 50;
remoteRequest = new Request(*this, requestPrefixChunk,
logRequestPrefixChunk, NUM_EXTRA_CHUNKS);
if (remoteRequest != NULL) {
remoteBodyChunkList = remoteRequest->getBodyChunkList();
remoteBodyChunkListInitialized = true;
}
}
if(bufferCount >= 1) {
remoteBodyChunkList.push_back(additionalRequestPrefixChunk);
BodyChunk temp;
char serviceIdBuf[1024];
if (remoteRequest != NULL) {
remoteRequest->getNextServiceRequestIdAsString(
serviceIdBuf, sizeof(serviceIdBuf));
}
temp.data = serviceIdBuf;
remoteBodyChunkList.push_back(temp);
}
sprintf(logLevel, "%d", record.getLogLevel());
remoteBodyChunkList.push_back(logLogPrefixChunk);
remoteBodyChunkList.push_back(BodyChunk(logName));
remoteBodyChunkList.push_back(logSidPrefixChunk);
remoteBodyChunkList.push_back(BodyChunk(loggedByTokenID));
remoteBodyChunkList.push_back(logLogSuffixChunk);
remoteBodyChunkList.push_back(logRecordPrefixChunk);
remoteBodyChunkList.push_back(logLevelPrefixChunk);
remoteBodyChunkList.push_back(BodyChunk(std::string(logLevel)));
remoteBodyChunkList.push_back(logLevelSuffixChunk);
remoteBodyChunkList.push_back(logRecMsgPrefixChunk);
std::string t_mesg(msg);
free(msg);
Utils::expandEntityRefs(t_mesg);
remoteBodyChunkList.push_back(BodyChunk(t_mesg));
remoteBodyChunkList.push_back(logRecMsgSuffixChunk);
remoteBodyChunkList.push_back(logInfoMapPrefixChunk);
const Properties &properties = record.getLogInfo();
Properties::const_iterator iter = properties.begin();
for(; iter != properties.end(); iter++) {
const Properties::key_type &k_iter = iter->first;
const Properties::mapped_type &v_iter = iter->second;
std::string keyStr("");
keyStr = k_iter.c_str();
std::string valueStr("");
valueStr = v_iter.c_str();
remoteBodyChunkList.push_back(logInfoKeyPrefixChunk);
remoteBodyChunkList.push_back(BodyChunk(keyStr));
remoteBodyChunkList.push_back(logInfoKeySuffixChunk);
remoteBodyChunkList.push_back(logInfoValuePrefixChunk);
remoteBodyChunkList.push_back(BodyChunk(valueStr));
remoteBodyChunkList.push_back(logInfoValueSuffixChunk);
}
remoteBodyChunkList.push_back(logInfoMapSuffixChunk);
remoteBodyChunkList.push_back(logRecordSuffixChunk);
remoteBodyChunkList.push_back(requestSuffixChunk);
bufferCount++;
} else {
status = AM_NO_MEMORY;
}
return status;
}
