bool slcPluginServer::RunPlugin( long a_lIndex, const slcPluginArgs* a_pPluginArgs )
{
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
const slcPluginArgs* pPluginArgs = a_pPluginArgs;
if( NULL == pPluginArgs )
pPluginArgs = &this->m_nullArgs;
if( ( a_lIndex < 0 ) ||
( a_lIndex > ( long )this->m_mapPluginInfos.size() ) )
{
return false;
}
this->itIds = this->m_mapPluginIds.find( a_lIndex );
if( this->itIds != this->m_mapPluginIds.end() )
{
SPL_RUN_FUNC pRunFunc =( SPL_RUN_FUNC )::GetProcAddress( ( HINSTANCE )this->itIds->second,
SPL_RUN_NAME );
if( NULL != pRunFunc )
{
return pRunFunc( pPluginArgs );
}
}
return false;
}
long slcPluginServer::RunAllPlugins( const slcPluginArgs* a_pPluginArgs )
{
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
long lPluginsRunned = 0;
const slcPluginArgs* pPluginArgs = a_pPluginArgs;
if( NULL == pPluginArgs )
pPluginArgs = &this->m_nullArgs;
if( this->m_mapPluginIds.empty() )
return 0;
for( long l = 0; l < ( long )this->m_mapPluginIds.size(); l++ )
{
if( this->RunPlugin( l, pPluginArgs ) )
lPluginsRunned++;
}
return lPluginsRunned;
}
void slcPluginServer::GetAllPluginInfos( void )
{
long l = 0;
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
for( this->itIds = this->m_mapPluginIds.begin();
this->itIds != this->m_mapPluginIds.end(); this->itIds++ )
{
SPL_INFO_FUNC pInfoFunc = ( SPL_INFO_FUNC )::GetProcAddress( ( HMODULE )( SPL_BITTYPE )this->itIds->second,
SPL_GETINFO_NAME );
if( NULL != pInfoFunc )
{
slcPluginInfo plugInfo = *pInfoFunc();
this->m_mapPluginInfos.insert( pair<long, slcPluginInfo>( l++, plugInfo ) );
}
else
{
//
// INFO FUNCTION NOT LOADED, INSERT DUMMY ENTRY TO AVOID ASYNCRON MAPS
//
slcPluginInfo nullInfo;
this->m_mapPluginInfos.insert( pair<long, slcPluginInfo>( l++, nullInfo ) );
}
}
}
void cDirCrawler::Action()
{
cThreadLock threadLock(this);
const char* DirName = pwd.path.c_str();
int LinkLevel = pwd.linkLevel;
isyslog("thread %d, Dir '%s'", ThreadId(), DirName);
cReadDir d(DirName);
struct dirent *e;
while (Running() && (e = d.Next()) != NULL)
{
// Ignore NULL or empty names and all hidden directories and files
if (!e->d_name || !strlen(e->d_name) || e->d_name[0] == '.')
continue;
// construct full path
cString buffer(AddDirectory(DirName, e->d_name));
struct stat st;
// check if path is a link
if (lstat(buffer, &st) != 0)
continue; // error
if (S_ISLNK(st.st_mode))
{
++LinkLevel;
// too many symlink in this path, maybe cyclic
if (LinkLevel > MAX_LINK_LEVEL)
{
isyslog("max link level exceeded - not scanning %s", *buffer);
continue;
}
if (stat(buffer, &st) != 0)
continue; // error
}
cPath nextPath;
nextPath.path = *buffer;
nextPath.linkLevel = LinkLevel;
// if subdirectory, put it in the directory Queue for other threads
if (S_ISDIR(st.st_mode))
{
isyslog("pwd: %s got '%s' , push '%s'", DirName, e->d_name, *buffer);
dirQ->AddPath(nextPath);
}
else if (S_ISREG(st.st_mode))
{
nextPath.filename = e->d_name;
// a file
AddPathToDatabase(nextPath);
}
} // while
} // Action()
void
Thread::Run()
{
boost::mutex::scoped_lock threadLock(m_threadObjMutex);
// Create the boost thread object.
if (!m_threadObj.get()) {
m_threadObj.reset(new boost::thread(ThreadStarter(*this)));
}
}
long slcPluginServer::LoadAllPlugins( const string& a_strPath,
const string& a_strWildcard, bool init_plugin,
const slcPluginArgs* a_pPluginArgs )
{
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
WIN32_FIND_DATA FindData;
string strWildcard;
if( a_strWildcard.empty() )
strWildcard = "*.dll";
else
strWildcard = a_strWildcard;
const slcPluginArgs* pPluginArgs = a_pPluginArgs;
if( NULL == pPluginArgs )
pPluginArgs = &this->m_nullArgs;
::SetCurrentDirectory( a_strPath.c_str() );
HANDLE hDirectory = ::FindFirstFileEx( strWildcard.c_str(), FindExInfoStandard,
&FindData, FindExSearchNameMatch, NULL, 0 );
if( hDirectory != INVALID_HANDLE_VALUE )
{
do {
if( ( strcmp( FindData.cFileName, "." ) != 0 ) &&
( strcmp( FindData.cFileName, ".." ) != 0 ) )
{
HMODULE hModule = ::LoadLibrary( FindData.cFileName );
if ( NULL != hModule )
{
SPL_INIT_FUNC pInitFunc =( SPL_INIT_FUNC )::GetProcAddress( ( HINSTANCE )hModule,
SPL_INIT_NAME );
if( NULL != pInitFunc )
{
if( init_plugin ) {
pInitFunc( pPluginArgs );
}
this->m_mapPluginIds.insert( pair< long, SPL_BITTYPE >( ( long )this->m_mapPluginIds.size(), ( SPL_BITTYPE )hModule ) );
}
}
}
}
while( ::FindNextFile( hDirectory, &FindData ) );
::FindClose( hDirectory );
}
return ( long )this->m_mapPluginIds.size();
}
HouseKeeper::~HouseKeeper(void)
{
gCoreContext->removeListener(this);
if (m_timer)
{
m_timer->stop();
disconnect(m_timer);
delete m_timer;
m_timer = NULL;
}
{
// remove anything from the queue first, so it does not start
QMutexLocker queueLock(&m_queueLock);
while (!m_taskQueue.isEmpty())
m_taskQueue.takeFirst()->DecrRef();
}
{
// issue a terminate call to any long-running tasks
// this is just a noop unless overwritten by a subclass
QMutexLocker mapLock(&m_mapLock);
QMap<QString,HouseKeeperTask*>::iterator it = m_taskMap.begin();
for (; it != m_taskMap.end(); ++it)
(*it)->Terminate();
}
if (!m_threadList.isEmpty())
{
QMutexLocker threadLock(&m_threadLock);
// tell primary thread to self-terminate and wake it
m_threadList.first()->Discard();
m_threadList.first()->Wake();
// wait for any remaining threads to self-terminate and close
while (!m_threadList.isEmpty())
{
HouseKeepingThread *thread = m_threadList.takeFirst();
thread->wait();
delete thread;
}
}
{
// unload any registered tasks
QMutexLocker mapLock(&m_mapLock);
QMap<QString,HouseKeeperTask*>::iterator it = m_taskMap.begin();
while (it != m_taskMap.end())
{
(*it)->DecrRef();
it = m_taskMap.erase(it);
}
}
}
void HouseKeeper::Run(void)
{
LOG(VB_GENERAL, LOG_DEBUG, "Running HouseKeeper.");
QDateTime now = MythDate::current();
QMutexLocker mapLock(&m_mapLock);
QMap<QString,HouseKeeperTask*>::const_iterator it;
for (it = m_taskMap.begin(); it != m_taskMap.end(); ++it)
{
if ((*it)->CheckRun(now))
{
// check if any tasks are ready to run, and add to queue
LOG(VB_GENERAL, LOG_INFO,
QString("Queueing HouseKeeperTask '%1'.").arg(it.key()));
QMutexLocker queueLock(&m_queueLock);
(*it)->IncrRef();
m_taskQueue.enqueue(*it);
}
}
if (!m_taskQueue.isEmpty())
StartThread();
if (m_threadList.size() > 1)
{
// spent threads exist in the thread list
// check to see if any have finished up their task and terminated
QMutexLocker threadLock(&m_threadLock);
int count1 = m_threadList.size();
QList<HouseKeepingThread*>::iterator it = m_threadList.begin();
++it; // skip the primary thread
while (it != m_threadList.end())
{
if ((*it)->isRunning())
++it;
else
it = m_threadList.erase(it);
}
int count2 = m_threadList.size();
if (count1 > count2)
LOG(VB_GENERAL, LOG_DEBUG,
QString("Discarded HouseKeepingThreads have completed and "
"been deleted. Current count %1 -> %2.")
.arg(count1).arg(count2));
}
}
void slcPluginServer::UnloadAllPlugins( const slcPluginArgs* a_pPluginArgs )
{
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
long lCount = ( long )this->m_mapPluginIds.size();
for( long l = 0; l < lCount; l++ )
{
this->UnloadPlugin( 0, true, a_pPluginArgs );
}
this->Clear();
}
void cAnalyzer::Action()
{
cThreadLock threadLock(this);
if (access(path.path.c_str(), R_OK) != 0)
return;
cFields field;
cRow props;
bool ok = MetaData(path.path.c_str(), field, props);
// XXX metadata extraction failed... what to do ?
if (true || ok)
{
std::string where = "path=\"" + path.path + "\"";
writer->Update(METADATA_TABLE, field, props, where);
}
}
long slcPluginServer::Refresh( const string& a_strPath,
const string& a_strWildcard,
const slcPluginArgs* a_pPluginShutdownArgs,
const slcPluginArgs* a_pPluginInitArgs )
{
this->UnloadAllPlugins( a_pPluginShutdownArgs );
//
// Lock this object for exclusive use.
// Explicitly usage.
//
slcThreadLock threadLock( this );
threadLock.Lock();
this->Clear();
threadLock.Unlock();
return( this->LoadAllPlugins( a_strPath, a_strWildcard, true, a_pPluginInitArgs ) > 0 );
}
long slcPluginServer::LoadPlugin( const string& a_strFileName, bool init_plugin, const slcPluginArgs* a_pPluginArgs )
{
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
if( a_strFileName.empty() )
return -2;
const slcPluginArgs* pPluginArgs = a_pPluginArgs;
if( NULL == pPluginArgs )
pPluginArgs = &this->m_nullArgs;
HMODULE hModule = ::LoadLibrary( a_strFileName.c_str() );
if ( NULL != hModule )
{
SPL_INIT_FUNC pInitFunc =( SPL_INIT_FUNC )::GetProcAddress( ( HINSTANCE )hModule,
SPL_INIT_NAME );
this->m_mapPluginIds.insert( pair< long, SPL_BITTYPE >( ( long )this->m_mapPluginIds.size(), ( SPL_BITTYPE )hModule ) );
if( NULL != pInitFunc )
{
if( init_plugin ) {
return pInitFunc( pPluginArgs ) ? 1 : 0;
} else {
return 1;
}
}
return -1;
}
return -2;
}
void* S3KeyWriter::UploadThreadFunc(void* data) {
MaskThreadSignals();
ThreadParams* params = (ThreadParams*)data;
S3KeyWriter* writer = params->keyWriter;
try {
S3DEBUG("Upload thread start: %" PRIX64 ", part number: %" PRIu64 ", data size: %zu",
(uint64_t) pthread_self(), params->currentNumber, params->data.size());
string etag = writer->s3Interface->uploadPartOfData(
params->data, writer->params.getS3Url(), params->currentNumber, writer->uploadId);
// when unique_lock destructs it will automatically unlock the mutex.
UniqueLock threadLock(&writer->mutex);
// etag is empty if the query is cancelled by user.
if (!etag.empty()) {
writer->etagList[params->currentNumber] = etag;
}
writer->activeThreads--;
pthread_cond_broadcast(&writer->cv);
S3DEBUG("Upload part finish: %" PRIX64 ", eTag: %s, part number: %" PRIu64, (uint64_t) pthread_self(),
etag.c_str(), params->currentNumber);
} catch (S3Exception& e) {
S3ERROR("Upload thread error: %s", e.getMessage().c_str());
UniqueLock exceptLock(&writer->exceptionMutex);
writer->sharedError = true;
writer->sharedException = std::current_exception();
// notify the flushBuffer, otherwise it will be locked when trying to create a new thread.
writer->activeThreads--;
pthread_cond_broadcast(&writer->cv);
}
delete params;
return NULL;
}
/** \brief Wake the primary run thread, or create a new one
*
* The HouseKeeper maintains one primary thread, but allows for multiple to
* allow task processing to continue should one long-running thread stall.
*
* The HouseKeeper scans to queue tasks once a minute. If any tasks are in
* the queue at the end of the scan, this method will be run. If the primary
* thread is still in use from the previous scan, it will be marked for
* destruction and a replacement thread will be started to pick up the
* remainder of the queue. The in-progress thread will terminate itself once
* its current task has finished, and will be eventually cleaned up by this
* method.
*/
void HouseKeeper::StartThread(void)
{
QMutexLocker threadLock(&m_threadLock);
if (m_threadList.isEmpty())
{
// we're running for the first time
// start up a new thread
LOG(VB_GENERAL, LOG_DEBUG, "Running initial HouseKeepingThread.");
HouseKeepingThread *thread = new HouseKeepingThread(this);
m_threadList.append(thread);
thread->start();
}
else if (!m_threadList.first()->isIdle())
{
// the old thread is still off processing something
// discard it and start a new one because we have more stuff
// that wants to run
LOG(VB_GENERAL, LOG_DEBUG,
QString("Current HouseKeepingThread is delayed on task, "
"spawning replacement. Current count %1.")
.arg(m_threadList.size()));
m_threadList.first()->Discard();
HouseKeepingThread *thread = new HouseKeepingThread(this);
m_threadList.prepend(thread);
thread->start();
}
else
{
// the old thread is idle, so just wake it for processing
LOG(VB_GENERAL, LOG_DEBUG, "Waking HouseKeepingThread.");
m_threadList.first()->Wake();
}
}
slcPluginInfo slcPluginServer::GetPluginInfo( const long a_lIndex )
{
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
if( ( a_lIndex < 0 ) ||
( a_lIndex > ( long )this->m_mapPluginInfos.size() ) )
{
slcPluginInfo invalidInfo;
return invalidInfo;
}
this->itIds = this->m_mapPluginIds.find( a_lIndex );
if( this->itIds != this->m_mapPluginIds.end() )
{
SPL_INFO_FUNC pInfoFunc = ( SPL_INFO_FUNC )::GetProcAddress( ( HMODULE )( SPL_BITTYPE )this->itIds->second, SPL_GETINFO_NAME );
if( NULL != pInfoFunc )
{
slcPluginInfo plugInfo = *pInfoFunc();
this->m_mapPluginInfos.insert( pair<long, slcPluginInfo>( this->m_mapPluginInfos.size() , plugInfo ) );
}
else
{
//
// INFO FUNCTION NOT LOADED, INSERT DUMMY ENTRY TO AVOID ASYNCRON MAPS
//
slcPluginInfo nullInfo;
this->m_mapPluginInfos.insert( pair<long, slcPluginInfo>( this->m_mapPluginInfos.size(), nullInfo ) );
}
}
return this->m_mapPluginInfos[ a_lIndex ];
}
bool slcPluginServer::UnloadPlugin( const long a_lIndex, bool shutdown_plugin, const slcPluginArgs* a_pPluginArgs )
{
//
// LOCKS OBJECT FOR EXCLUSIVE USE, UNLOCKS AUTOMATICALLY AT THE END OF FUNCTION.
// IMPLICIT USAGE.
//
slcThreadLock threadLock( this, true );
if( ( a_lIndex < 0 ) ||
( a_lIndex > ( long )this->m_mapPluginInfos.size() ) )
{
return false;
}
const slcPluginArgs* pPluginArgs = a_pPluginArgs;
if( NULL == pPluginArgs )
pPluginArgs = &this->m_nullArgs;
this->itIds = this->m_mapPluginIds.find( a_lIndex );
if( this->itIds != this->m_mapPluginIds.end() )
{
SPL_SHUTDOWN_FUNC pShutdownFunc =( SPL_SHUTDOWN_FUNC )::GetProcAddress( ( HINSTANCE )this->itIds->second,
SPL_SHUTDOWN_NAME );
if( NULL != pShutdownFunc )
{
if( shutdown_plugin ) {
pShutdownFunc( pPluginArgs );
}
}
//
// UNLOAD LIBRARY
//
::FreeLibrary( ( HMODULE )this->itIds->second );
//
// REMOVE LIBRARY FROM ID-MAP AND INFO-MAP
//
this->m_mapPluginInfos.erase( this->itIds->first );
this->m_mapPluginIds.erase( this->itIds->first );
//
// REFRESH ALL MAP IDS (CREATE NEW KEYS FOR ALL VALUES)
//
int i = 0;
long lSize = this->m_mapPluginIds.size();
map< long, SPL_BITTYPE > tempIds;
map< long, SPL_BITTYPE >::iterator itTempIds;
tempIds = this->m_mapPluginIds;
this->m_mapPluginIds.clear();
itTempIds = tempIds.begin();
for( i = 0 ; i < lSize; i++, itTempIds++ )
this->m_mapPluginIds.insert( pair<long, SPL_BITTYPE> (i, itTempIds->second ) );
lSize = this->m_mapPluginInfos.size();
map< long, slcPluginInfo > tempInfos;
map< long, slcPluginInfo >::iterator itTempInfos;
tempInfos = this->m_mapPluginInfos;
this->m_mapPluginInfos.clear();
itTempInfos = tempInfos.begin();
for( i = 0 ; i < lSize; i++, itTempInfos++ )
this->m_mapPluginInfos.insert( pair<long, slcPluginInfo> (i, itTempInfos->second ) );
return true;
}
return false;
}
bool
Thread::IsRunning() const
{
boost::mutex::scoped_lock threadLock(m_threadObjMutex);
return (m_threadObj.get() != NULL);
}
MemoryMapNode *MemoryMap::addChild(
const Instance &origInst, const InstanceList &candidates,
MemoryMapNode *parent, int threadIndex, quint64 addrInParent,
bool addToQueue)
{
MemoryMapNode *child = 0;
// Only proceed if we don't have more than one instance to process
if (candidates.size() > 1 || (!origInst.isValid() && candidates.isEmpty()))
return 0;
Instance i(origInst);
// Do we have a candidate?
if (!candidates.isEmpty()) {
float o_prob, c_prob;
Instance cand(candidates.first());
// Does one instance embed the other?
ObjectRelation orel = BaseType::embeds(
cand.type(), cand.address(),
origInst.type(), origInst.address());
switch(orel) {
case orSecondEmbedsFirst:
// Use the original type
break;
// Use the candidate type
case orFirstEmbedsSecond:
// Use the candidate type (it might hold properties from the
// JavaScript rule evaluation
case orEqual:
i = cand;
break;
case orCover:
case orOverlap:
case orNoOverlap:
// Calculate probability for both
o_prob = calculateNodeProbability(origInst, 1.0);
c_prob = calculateNodeProbability(cand, 1.0);
// If probs are significantly different (> 0.1), and add the one
// with higher prob., otherwise add both
if (qAbs(c_prob - o_prob) > 0.1) {
if (c_prob > o_prob)
i = cand;
}
else {
addChild(cand, InstanceList(), parent, threadIndex,
addrInParent, addToQueue);
}
break;
}
}
// Dereference, if required
if (i.type() && (i.type()->type() & BaseType::trLexical))
i = i.dereference(BaseType::trLexical);
if (!i.isNull() && i.type() && (i.type()->type() & interestingTypes)) {
// Don't add nodes if the probability is below the threshold
float prob = calculateNodeProbability(i, parent ? parent->probability() : 1.0);
if (prob <= _shared->minProbability)
return 0;
// Is any other thread currently searching for the same address?
_shared->currAddressesLock.lockForWrite();
_shared->currAddresses[threadIndex] = 0;
int idx = 0;
while (idx < _shared->threadCount) {
if (_shared->currAddresses[idx] == i.address()) {
// Another thread searches for the same address, so release the
// currAddressesLock and wait for it to finish
_shared->currAddressesLock.unlock();
_shared->perThreadLock[idx].lock();
// We are blocked until the other thread has finished its node
_shared->perThreadLock[idx].unlock();
_shared->currAddressesLock.lockForWrite();
idx = 0;
}
else
++idx;
}
// No conflicts anymore, so we lock our current address
_shared->currAddresses[threadIndex] = i.address();
QMutexLocker threadLock(&_shared->perThreadLock[threadIndex]);
_shared->currAddressesLock.unlock();
// Check if object conflicts previously given objects
if (objectIsSane(i) && parent) {
try {
MemoryMapNodeSV* parent_sv = _buildType == btSibi ?
dynamic_cast<MemoryMapNodeSV*>(parent) : 0;
if (parent_sv)
child = parent_sv->addChild(i, addrInParent);
else
child = parent->addChild(i);
}
catch(GenericException& ge) {
// The address of the instance is invalid
// Release locks and return
_shared->currAddressesLock.lockForWrite();
_shared->currAddresses[threadIndex] = 0;
_shared->currAddressesLock.unlock();
return child;
}
addNodeToHashes(child);
// Insert the new node into the queue
if (addToQueue && shouldEnqueue(i, child)) {
_shared->queueLock.lock();
_shared->queue.insert(child->probability(), child);
_shared->queueLock.unlock();
}
}
// Done, release our current address (no need to hold currAddressesLock)
_shared->currAddressesLock.lockForWrite();
_shared->currAddresses[threadIndex] = 0;
_shared->currAddressesLock.unlock();
// // Wake up a waiting thread (if it exists)
// _shared->threadDone[threadIndex].wakeOne();
}
return child;
}
void cDirCrawler::SetDirectory(const cPath& path)
{
cThreadLock threadLock(this);
pwd = path;
}
