void StatMeter::Init()
{
m_tStartServer = time(NULL);
m_tLastCheck = m_tStartServer;
AdjustTimeOffset();
m_ServerVolumes.resize(1 + g_pServerPool->GetServers()->size());
m_ServerVolumes[0] = new ServerVolume();
for (Servers::iterator it = g_pServerPool->GetServers()->begin(); it != g_pServerPool->GetServers()->end(); it++)
{
NewsServer* pServer = *it;
m_ServerVolumes[pServer->GetID()] = new ServerVolume();
}
}
void ServerPool::InitConnections()
{
debug("Initializing connections in ServerPool");
m_iMaxLevel = 0;
for (Servers::iterator it = m_Servers.begin(); it != m_Servers.end(); it++)
{
NewsServer* pNewsServer = *it;
if (m_iMaxLevel < pNewsServer->GetLevel())
{
m_iMaxLevel = pNewsServer->GetLevel();
}
for (int i = 0; i < pNewsServer->GetMaxConnections(); i++)
{
PooledConnection* pConnection = new PooledConnection(pNewsServer);
pConnection->SetTimeout(m_iTimeout);
m_Connections.push_back(pConnection);
}
}
for (int iLevel = 0; iLevel <= m_iMaxLevel; iLevel++)
{
int iMaxConnectionsForLevel = 0;
for (Servers::iterator it = m_Servers.begin(); it != m_Servers.end(); it++)
{
NewsServer* pNewsServer = *it;
if (iLevel == pNewsServer->GetLevel())
{
iMaxConnectionsForLevel += pNewsServer->GetMaxConnections();
}
}
m_Levels.push_back(iMaxConnectionsForLevel);
}
}
int main(int argc, char** argv)
{
NewsServer* server = new NewsServer();
ConcreteNewsListener* firstListener = new ConcreteNewsListener();
server->attach(firstListener);
ConcreteNewsListener* secondListener = new ConcreteNewsListener();
server->attach(secondListener);
server->propagateNews("First News");
printf("%s\n", firstListener->getData().c_str());
printf("%s\n", secondListener->getData().c_str());
server->detach(secondListener);
server->propagateNews("Second News");
printf("%s\n", firstListener->getData().c_str());
printf("%s\n", secondListener->getData().c_str());
void (*pPropagateNews)(std::string) = server->propagateNews;
pPropagateNews("Third News");
printf("%s\n", firstListener->getData().c_str());
printf("%s\n", secondListener->getData().c_str());
printf("asdf");
}
/*
* Compute maximum number of allowed download threads
**/
void QueueCoordinator::AdjustDownloadsLimit()
{
if (m_iServerConfigGeneration == g_pServerPool->GetGeneration())
{
return;
}
// two extra threads for completing files (when connections are not needed)
int iDownloadsLimit = 2;
// allow one thread per 0-level (main) and 1-level (backup) server connection
for (Servers::iterator it = g_pServerPool->GetServers()->begin(); it != g_pServerPool->GetServers()->end(); it++)
{
NewsServer* pNewsServer = *it;
if ((pNewsServer->GetNormLevel() == 0 || pNewsServer->GetNormLevel() == 1) && pNewsServer->GetActive())
{
iDownloadsLimit += pNewsServer->GetMaxConnections();
}
}
m_iDownloadsLimit = iDownloadsLimit;
}
int main() {
NewsServer<string> srv;
cout << "expect nothing" << endl;
srv.propergate("nothing");
cout << endl << "expect only notify 1" << endl;
srv.reg(notifyMeth1);
srv.propergate("test");
cout << endl << "expect notify 1 + 2" << endl;
srv.reg(notifyMeth2);
srv.propergate("test 2");
cout << endl << "expect notify 1 + 2 + 3" << endl;
srv.reg(notifyMeth3);
srv.propergate("test 3");
cout << endl <<"expect notify 2 + 3" << endl;
srv.unreg(notifyMeth1);
srv.propergate("test 4");
return 0;
}
void ServerPool::InitConnections()
{
debug("Initializing connections in ServerPool");
m_mutexConnections.Lock();
NormalizeLevels();
m_Levels.clear();
for (Servers::iterator it = m_SortedServers.begin(); it != m_SortedServers.end(); it++)
{
NewsServer* pNewsServer = *it;
pNewsServer->SetBlockTime(0);
int iNormLevel = pNewsServer->GetNormLevel();
if (pNewsServer->GetNormLevel() > -1)
{
if ((int)m_Levels.size() <= iNormLevel)
{
m_Levels.push_back(0);
}
if (pNewsServer->GetActive())
{
int iConnections = 0;
for (Connections::iterator it = m_Connections.begin(); it != m_Connections.end(); it++)
{
PooledConnection* pConnection = *it;
if (pConnection->GetNewsServer() == pNewsServer)
{
iConnections++;
}
}
for (int i = iConnections; i < pNewsServer->GetMaxConnections(); i++)
{
PooledConnection* pConnection = new PooledConnection(pNewsServer);
pConnection->SetTimeout(m_iTimeout);
m_Connections.push_back(pConnection);
iConnections++;
}
m_Levels[iNormLevel] += iConnections;
}
}
}
m_iGeneration++;
m_mutexConnections.Unlock();
}
void ServerPool::LogDebugInfo()
{
info(" ---------- ServerPool");
info(" Max-Level: %i", m_iMaxNormLevel);
m_mutexConnections.Lock();
time_t tCurTime = time(NULL);
info(" Servers: %i", m_Servers.size());
for (Servers::iterator it = m_Servers.begin(); it != m_Servers.end(); it++)
{
NewsServer* pNewsServer = *it;
info(" %i) %s (%s): Level=%i, NormLevel=%i, BlockSec=%i", pNewsServer->GetID(), pNewsServer->GetName(),
pNewsServer->GetHost(), pNewsServer->GetLevel(), pNewsServer->GetNormLevel(),
pNewsServer->GetBlockTime() && pNewsServer->GetBlockTime() + m_iRetryInterval > tCurTime ?
pNewsServer->GetBlockTime() + m_iRetryInterval - tCurTime : 0);
}
info(" Levels: %i", m_Levels.size());
int index = 0;
for (Levels::iterator it = m_Levels.begin(); it != m_Levels.end(); it++, index++)
{
int iSize = *it;
info(" %i: Free connections=%i", index, iSize);
}
info(" Connections: %i", m_Connections.size());
for (Connections::iterator it = m_Connections.begin(); it != m_Connections.end(); it++)
{
PooledConnection* pConnection = *it;
info(" %i) %s (%s): Level=%i, NormLevel=%i, InUse:%i", pConnection->GetNewsServer()->GetID(),
pConnection->GetNewsServer()->GetName(), pConnection->GetNewsServer()->GetHost(),
pConnection->GetNewsServer()->GetLevel(), pConnection->GetNewsServer()->GetNormLevel(),
(int)pConnection->GetInUse());
}
m_mutexConnections.Unlock();
}
NNTPConnection* ServerPool::GetConnection(int iLevel, NewsServer* pWantServer, Servers* pIgnoreServers)
{
PooledConnection* pConnection = NULL;
m_mutexConnections.Lock();
time_t tCurTime = time(NULL);
if (iLevel < (int)m_Levels.size() && m_Levels[iLevel] > 0)
{
Connections candidates;
candidates.reserve(m_Connections.size());
for (Connections::iterator it = m_Connections.begin(); it != m_Connections.end(); it++)
{
PooledConnection* pCandidateConnection = *it;
NewsServer* pCandidateServer = pCandidateConnection->GetNewsServer();
if (!pCandidateConnection->GetInUse() && pCandidateServer->GetActive() &&
pCandidateServer->GetNormLevel() == iLevel &&
(!pWantServer || pCandidateServer == pWantServer ||
(pWantServer->GetGroup() > 0 && pWantServer->GetGroup() == pCandidateServer->GetGroup())) &&
(pCandidateConnection->GetStatus() == Connection::csConnected ||
!pCandidateServer->GetBlockTime() ||
pCandidateServer->GetBlockTime() + m_iRetryInterval <= tCurTime ||
pCandidateServer->GetBlockTime() > tCurTime))
{
// free connection found, check if it's not from the server which should be ignored
bool bUseConnection = true;
if (pIgnoreServers && !pWantServer)
{
for (Servers::iterator it = pIgnoreServers->begin(); it != pIgnoreServers->end(); it++)
{
NewsServer* pIgnoreServer = *it;
if (pIgnoreServer == pCandidateServer ||
(pIgnoreServer->GetGroup() > 0 && pIgnoreServer->GetGroup() == pCandidateServer->GetGroup() &&
pIgnoreServer->GetNormLevel() == pCandidateServer->GetNormLevel()))
{
bUseConnection = false;
break;
}
}
}
pCandidateServer->SetBlockTime(0);
if (bUseConnection)
{
candidates.push_back(pCandidateConnection);
}
}
}
if (!candidates.empty())
{
// Peeking a random free connection. This is better than taking the first
// available connection because provides better distribution across news servers,
// especially when one of servers becomes unavailable or doesn't have requested articles.
int iRandomIndex = rand() % candidates.size();
pConnection = candidates[iRandomIndex];
pConnection->SetInUse(true);
}
if (pConnection)
{
m_Levels[iLevel]--;
}
}
m_mutexConnections.Unlock();
return pConnection;
}
/*
* Calculate normalized levels for all servers.
* Normalized Level means: starting from 0 with step 1.
* The servers of minimum Level must be always used even if they are not active;
* this is to prevent backup servers to act as main servers.
**/
void ServerPool::NormalizeLevels()
{
if (m_Servers.empty())
{
return;
}
std::sort(m_SortedServers.begin(), m_SortedServers.end(), CompareServers);
// find minimum level
int iMinLevel = m_SortedServers.front()->GetLevel();
for (Servers::iterator it = m_SortedServers.begin(); it != m_SortedServers.end(); it++)
{
NewsServer* pNewsServer = *it;
if (pNewsServer->GetLevel() < iMinLevel)
{
iMinLevel = pNewsServer->GetLevel();
}
}
m_iMaxNormLevel = 0;
int iLastLevel = iMinLevel;
for (Servers::iterator it = m_SortedServers.begin(); it != m_SortedServers.end(); it++)
{
NewsServer* pNewsServer = *it;
if ((pNewsServer->GetActive() && pNewsServer->GetMaxConnections() > 0) ||
(pNewsServer->GetLevel() == iMinLevel))
{
if (pNewsServer->GetLevel() != iLastLevel)
{
m_iMaxNormLevel++;
}
pNewsServer->SetNormLevel(m_iMaxNormLevel);
iLastLevel = pNewsServer->GetLevel();
}
else
{
pNewsServer->SetNormLevel(-1);
}
}
}
/*
* How server management (for one particular article) works:
- there is a list of failed servers which is initially empty;
- level is initially 0;
<loop>
- request a connection from server pool for current level;
Exception: this step is skipped for the very first download attempt, because a
level-0 connection is initially passed from queue manager;
- try to download from server;
- if connection to server cannot be established or download fails due to interrupted connection,
try again (as many times as needed without limit) the same server until connection is OK;
- if download fails with error "Not-Found" (article or group not found) or with CRC error,
add the server to failed server list;
- if download fails with general failure error (article incomplete, other unknown error
codes), try the same server again as many times as defined by option <Retries>; if all attempts
fail, add the server to failed server list;
- if all servers from current level were tried, increase level;
- if all servers from all levels were tried, break the loop with failure status.
<end-loop>
*/
void ArticleDownloader::Run()
{
debug("Entering ArticleDownloader-loop");
SetStatus(adRunning);
BuildOutputFilename();
m_szResultFilename = m_pArticleInfo->GetResultFilename();
if (g_pOptions->GetContinuePartial())
{
if (Util::FileExists(m_szResultFilename))
{
// file exists from previous program's start
detail("Article %s already downloaded, skipping", m_szInfoName);
FreeConnection(true);
SetStatus(adFinished);
Notify(NULL);
return;
}
}
EStatus Status = adFailed;
int iRetries = g_pOptions->GetRetries() > 0 ? g_pOptions->GetRetries() : 1;
int iRemainedRetries = iRetries;
ServerPool::Servers failedServers;
failedServers.reserve(g_pServerPool->GetServers()->size());
NewsServer* pWantServer = NULL;
NewsServer* pLastServer = NULL;
int iLevel = 0;
int iServerConfigGeneration = g_pServerPool->GetGeneration();
while (!IsStopped())
{
Status = adFailed;
SetStatus(adWaiting);
while (!m_pConnection && !(IsStopped() || iServerConfigGeneration != g_pServerPool->GetGeneration()))
{
m_pConnection = g_pServerPool->GetConnection(iLevel, pWantServer, &failedServers);
usleep(5 * 1000);
}
SetLastUpdateTimeNow();
SetStatus(adRunning);
if (IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration())
{
Status = adRetry;
break;
}
pLastServer = m_pConnection->GetNewsServer();
m_pConnection->SetSuppressErrors(false);
// test connection
bool bConnected = m_pConnection && m_pConnection->Connect();
if (bConnected && !IsStopped())
{
// Okay, we got a Connection. Now start downloading.
detail("Downloading %s @ %s (%s)", m_szInfoName,
m_pConnection->GetNewsServer()->GetName(), m_pConnection->GetHost());
Status = Download();
}
if (bConnected)
{
if (Status == adConnectError)
{
m_pConnection->Disconnect();
bConnected = false;
Status = adFailed;
}
else
{
// freeing connection allows other threads to start.
// we doing this only if the problem was with article or group.
// if the problem occurs by connecting or authorization we do not
// free the connection, to prevent starting of thousands of threads
// (cause each of them will also free it's connection after the
// same connect-error).
FreeConnection(Status == adFinished || Status == adNotFound);
}
}
if (Status == adFinished || Status == adFatalError)
{
break;
}
pWantServer = NULL;
if (bConnected && Status == adFailed)
{
iRemainedRetries--;
}
if (!bConnected || (Status == adFailed && iRemainedRetries > 0))
{
pWantServer = pLastServer;
}
if (pWantServer &&
!(IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration()))
{
detail("Waiting %i sec to retry", g_pOptions->GetRetryInterval());
SetStatus(adWaiting);
int msec = 0;
while (!(IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration()) &&
msec < g_pOptions->GetRetryInterval() * 1000)
{
usleep(100 * 1000);
msec += 100;
}
SetLastUpdateTimeNow();
SetStatus(adRunning);
}
if (IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration())
{
Status = adRetry;
break;
}
if (!pWantServer)
{
failedServers.push_back(pLastServer);
// if all servers from current level were tried, increase level
// if all servers from all levels were tried, break the loop with failure status
bool bAllServersOnLevelFailed = true;
for (ServerPool::Servers::iterator it = g_pServerPool->GetServers()->begin(); it != g_pServerPool->GetServers()->end(); it++)
{
NewsServer* pCandidateServer = *it;
if (pCandidateServer->GetNormLevel() == iLevel)
{
bool bServerFailed = !pCandidateServer->GetActive();
if (!bServerFailed)
{
for (ServerPool::Servers::iterator it = failedServers.begin(); it != failedServers.end(); it++)
{
NewsServer* pIgnoreServer = *it;
if (pIgnoreServer == pCandidateServer ||
(pIgnoreServer->GetGroup() > 0 && pIgnoreServer->GetGroup() == pCandidateServer->GetGroup() &&
pIgnoreServer->GetNormLevel() == pCandidateServer->GetNormLevel()))
{
bServerFailed = true;
break;
}
}
}
if (!bServerFailed)
{
bAllServersOnLevelFailed = false;
break;
}
}
}
if (bAllServersOnLevelFailed)
{
if (iLevel < g_pServerPool->GetMaxNormLevel())
{
detail("Article %s @ all level %i servers failed, increasing level", m_szInfoName, iLevel);
iLevel++;
}
else
{
warn("Article %s @ all servers failed", m_szInfoName);
Status = adFailed;
break;
}
}
iRemainedRetries = iRetries;
}
}
FreeConnection(Status == adFinished);
if (m_bDuplicate)
{
Status = adFinished;
}
if (Status != adFinished && Status != adRetry)
{
Status = adFailed;
}
if (IsStopped())
{
detail("Download %s cancelled", m_szInfoName);
Status = adRetry;
}
if (Status == adFailed)
{
warn("Download %s failed", m_szInfoName);
}
SetStatus(Status);
Notify(NULL);
debug("Exiting ArticleDownloader-loop");
}
