static void Imo2sproxy_Exit(IMO2SPROXY *hInst)
{
IMO2SPROXY_INST *hProxy = (IMO2SPROXY_INST*)hInst;
CONNINST *pInst;
if (hProxy->pCfg->bVerbose && hProxy->pCfg->fpLog)
fprintf (hProxy->pCfg->fpLog, "W32SkypeEmu:Exit()\n");
if (hProxy->hWndDispatch) DestroyWindow (hProxy->hWndDispatch);
if (hProxy->dwThreadID) PostThreadMessage (hProxy->dwThreadID, WM_QUIT, 0, 0);
LockMutex(hProxy->loopmutex);
// Kill 'em all!
if (hProxy->hClients)
{
while (pInst=List_Pop(hProxy->hClients))
{
FreeConnection(pInst);
free (pInst);
}
List_Exit (hProxy->hClients);
}
UnregisterClass ("Imo2SProxyDispatchWindow", GetModuleHandle(NULL));
UnlockMutex(hProxy->loopmutex);
ExitMutex(hProxy->loopmutex);
free (hProxy);
}
static void _WorkerCloseHandler(Event event, Connection* source, Worker* worker) {
if (worker->queue == NULL || !EnqueueEventItem(worker->queue, event, (EventSource)source)) {
if (ConnectionNoMoreUsed(source, AIO4C_CONNECTION_OWNER_WORKER)) {
FreeConnection(&source);
}
}
}
void ChildProcessLauncher::InnerDeathCallback::OnChildProcessDied(
/* [in] */ ChildProcessConnection* connection)
{
if (connection->GetPid() != 0) {
Stop(connection->GetPid());
}
else {
FreeConnection(connection);
}
}
static void _ReaderEventHandler(Event event, Connection* connection, Reader* reader) {
if (reader->queue == NULL || !EnqueueEventItem(reader->queue, event, (EventSource)connection)) {
if (ConnectionNoMoreUsed(connection, AIO4C_CONNECTION_OWNER_READER)) {
FreeConnection(&connection);
}
return;
}
SelectorWakeUp(reader->selector);
}
static void _serverExit(ThreadData _server) {
Server* server = (Server*)_server;
if (server->acceptor != NULL) {
AcceptorEnd(server->acceptor);
}
FreeAddress(&server->address);
FreeBufferPool(&server->pool);
FreeConnection(&server->factory);
FreeQueue(&server->queue);
}
static void Imo2sproxy_Loop(IMO2SPROXY *hInst)
{
struct sockaddr_in sock;
int socklen;
SOCKET new_fd;
TYP_LIST *hConns = List_Init(32);
CONNINST *pInst;
IMO2SPROXY_INST *hProxy = (IMO2SPROXY_INST*)hInst;
fd_set fdListen;
if (hProxy->pCfg->bVerbose && hProxy->pCfg->fpLog)
fprintf (hProxy->pCfg->fpLog, "Socksproxy:Loop(Start)\n");
hProxy->iRunning = 1;
LockMutex(hProxy->loopmutex);
while (hProxy->iRunning)
{
FD_ZERO(&fdListen);
FD_SET(hProxy->listen_fd, &fdListen);
socklen = sizeof(sock);
if (select (0, &fdListen, NULL, NULL, NULL) != SOCKET_ERROR && FD_ISSET(hProxy->listen_fd, &fdListen))
{
new_fd = accept(hProxy->listen_fd, (struct sockaddr *) &sock, &socklen);
if (hProxy->pCfg->bVerbose && hProxy->pCfg->fpLog)
{
fprintf (hProxy->pCfg->fpLog, "Connection from %s:%d -> Connection: %d\n", inet_ntoa(sock.sin_addr),
ntohs(sock.sin_port), new_fd);
fflush (hProxy->pCfg->fpLog);
}
if (new_fd != INVALID_SOCKET && (pInst = calloc (1, sizeof(CONNINST))))
{
CleanConnections (hConns);
List_Push(hConns, pInst);
pInst->hSock = new_fd;
pInst->hProxy = hProxy;
InitMutex(pInst->connected);
LockMutex(pInst->connected);
InitMutex(pInst->sendmutex);
Dispatcher_Start(pInst);
}
}
}
if (hProxy->pCfg->bVerbose && hProxy->pCfg->fpLog)
fprintf (hProxy->pCfg->fpLog, "Socksproxy:Loop(End)\n");
CleanConnections (hConns);
while (pInst=List_Pop(hConns))
{
Dispatcher_Stop(pInst);
FreeConnection(pInst);
free (pInst);
}
List_Exit(hConns);
UnlockMutex(hProxy->loopmutex);
}
/* post an asynchounous recv request */
static int PostRecvRequest(connection_t* conn)
{
DWORD flags = 0;
DWORD recv_bytes = 0;
int error = WSARecv(conn->socket, &conn->buf, 1, &recv_bytes, &flags,
&conn->overlap, OnRecvComplete);
if (error == SOCKET_ERROR && WSAGetLastError() != WSA_IO_PENDING)
{
fprintf(stderr, ("WSARecv() failed [%d], %s"), conn->socket, LAST_ERROR_MSG);
FreeConnection(conn);
return -1;
}
return 0;
}
void CALLBACK OnSendComplete(DWORD error,
DWORD bytes_transferred,
WSAOVERLAPPED* overlap,
DWORD flags)
{
connection_t* conn = (connection_t*)overlap->hEvent;
if (error || bytes_transferred == 0)
{
FreeConnection(conn);
return ;
}
shutdown(conn->socket, SD_BOTH);
PostRecvRequest(conn);
}
void CALLBACK OnRecvComplete(DWORD error,
DWORD bytes_transferred,
WSAOVERLAPPED* overlap,
DWORD flags)
{
DWORD bytes_send = 0;
connection_t* conn = (connection_t*)overlap->hEvent;
if (error || bytes_transferred == 0)
{
FreeConnection(conn);
return ;
}
/* send back */
memset(&conn->overlap, 0, sizeof(conn->overlap));
conn->overlap.hEvent = (WSAEVENT)conn;
conn->buf.len = bytes_transferred;
error = WSASend(conn->socket, &conn->buf, 1, &bytes_send, flags,
&conn->overlap, OnSendComplete);
if (error == SOCKET_ERROR && WSAGetLastError() != WSA_IO_PENDING)
{
FreeConnection(conn);
}
}
//@CalledByNative
void ChildProcessLauncher::Stop(
/* [in] */ Int32 pid)
{
assert(0);
#if 0
// Log.d(TAG, "stopping child connection: pid=" + pid);
AutoPtr<ChildProcessConnection> connection = sServiceMap->Remove(pid);
if (connection == NULL) {
LogPidWarning(pid, "Tried to stop non-existent connection");
return;
}
sBindingManager->ClearConnection(pid);
connection->Stop();
FreeConnection(connection);
#endif
}
static void CleanConnections (TYP_LIST *hList)
{
unsigned int i;
CONNINST *pInst;
for (i=0; i<List_Count(hList); i++)
{
pInst = List_ElementAt (hList, i);
if (!IsWindow (pInst->hWnd))
{
if (pInst->hInst) FreeConnection(pInst);
free (List_RemoveElementAt(hList, i));
i--;
}
}
}
void HttpManager::HandleError(HttpResponse& response, const string_t& error) {
HttpClient& client = *FindClient(response.uid);
switch (client.mode()) {
case kHttpServiceAuthenticateUser:
case kHttpServiceGetMetadataById:
case kHttpServiceGetMetadataByIdV2:
case kHttpServiceSearchTitle:
case kHttpServiceAddLibraryEntry:
case kHttpServiceDeleteLibraryEntry:
case kHttpServiceGetLibraryEntries:
case kHttpServiceUpdateLibraryEntry:
ServiceManager.HandleHttpError(client.response_, error);
break;
}
FreeConnection(client.request_.url.host);
ProcessQueue();
}
Server* NewServer(AddressType type, char* host, aio4c_port_t port, int bufferSize, int nbPipes, void (*handler)(Event,Connection*,void*), void* handlerArg, void* (*dataFactory)(Connection*,void*)) {
Server* server = NULL;
ErrorCode code = AIO4C_ERROR_CODE_INITIALIZER;
if ((server = aio4c_malloc(sizeof(Server))) == NULL) {
#ifndef AIO4C_WIN32
code.error = errno;
#else /* AIO4C_WIN32 */
code.source = AIO4C_ERRNO_SOURCE_SYS;
#endif /* AIO4C_WIN32 */
code.size = sizeof(Server);
code.type = "Server";
Raise(AIO4C_LOG_LEVEL_ERROR, AIO4C_ALLOC_ERROR_TYPE, AIO4C_ALLOC_ERROR, &code);
return NULL;
}
server->address = NewAddress(type, host, port);
server->pool = NewBufferPool(bufferSize);
server->factory = NewConnectionFactory(server->pool, dataFactory, handlerArg);
server->acceptor = NULL;
server->thread = NULL;
server->handler = handler;
server->queue = NewQueue();
server->nbPipes = nbPipes;
server->thread = NewThread(
"server",
_serverInit,
_serverRun,
_serverExit,
(ThreadData)server);
if (server->thread == NULL) {
FreeAddress(&server->address);
FreeBufferPool(&server->pool);
FreeConnection(&server->factory);
FreeQueue(&server->queue);
aio4c_free(server);
return NULL;
}
return server;
}
static bool _WorkerRun(ThreadData _worker) {
Worker* worker = (Worker*)_worker;
QueueItem* item = NewQueueItem();
Connection* connection = NULL;
Buffer* buffer = NULL;
while (Dequeue(worker->queue, item, true)) {
switch (QueueItemGetType(item)) {
case AIO4C_QUEUE_ITEM_EXIT:
FreeQueueItem(&item);
return false;
case AIO4C_QUEUE_ITEM_TASK:
connection = QueueTaskItemGetConnection(item);
Log(AIO4C_LOG_LEVEL_DEBUG, "dequeued task for connection %s", connection->string);
ProbeTimeStart(AIO4C_TIME_PROBE_DATA_PROCESS);
buffer = QueueTaskItemGetBuffer(item);
connection->dataBuffer = buffer;
ConnectionProcessData(connection);
connection->dataBuffer = NULL;
ProbeSize(AIO4C_PROBE_PROCESSED_DATA_SIZE,BufferGetPosition(buffer));
ReleaseBuffer(&buffer);
ProbeTimeEnd(AIO4C_TIME_PROBE_DATA_PROCESS);
break;
case AIO4C_QUEUE_ITEM_EVENT:
connection = (Connection*)QueueEventItemGetSource(item);
Log(AIO4C_LOG_LEVEL_DEBUG, "close received for connection %s", connection->string);
RemoveAll(worker->queue, _removeCallback, (QueueDiscriminant)connection);
if (ConnectionNoMoreUsed(connection, AIO4C_CONNECTION_OWNER_WORKER)) {
Log(AIO4C_LOG_LEVEL_DEBUG, "freeing connection %s", connection->string);
FreeConnection(&connection);
}
break;
default:
break;
}
}
FreeQueueItem(&item);
return true;
}
//-------------------------------------------------------
OdbcPersistor::~OdbcPersistor()
{
try
{
DisconnectOdbcConnection();
FreeConnection(m_odbcConnection);
const SQLRETURN ret = ::SQLFreeHandle(SQL_HANDLE_ENV, m_environment);
if (!SQL_SUCCEEDED(ret))
{
OdbcHelper::ThrowException(SQL_HANDLE_ENV, m_environment);
}
}
catch(const OdbcException& ex)
{
Safir::Logging::SendSystemLog
(Safir::Logging::Error,
Safir::Dob::Typesystem::Utilities::ToWstring(ex.what()));
}
}
static void DispatcherThread(void *pUser)
{
CONNINST *pInst = (CONNINST*)pUser;
char *pszUser, *pszPass, *pszError, *pszMsgBuf=NULL;
unsigned int uiLength, cbMsgBuf=0, bAuthenticated = 0, iConnected=0, iLogin=1;
char command=0, reply=0;
if (pInst->hProxy->pCfg->bVerbose && pInst->hProxy->pCfg->fpLog)
fprintf (pInst->hProxy->pCfg->fpLog, "Imo2sproxy::DispatcherThread()\n");
if (!(pInst->hInst = Imo2S_Init(EventHandler, pInst, pInst->hProxy->pCfg->iFlags)))
{
pInst->hProxy->pCfg->logerror (stderr, "Connection %d: Cannot start Imo2Skype instance.\n", pInst->hSock);
FreeConnection (pInst);
return;
}
// FIXME: Static user+pass from cmdline, until there is a possibility for
// a client to authenticate
pszUser = pInst->hProxy->pCfg->pszUser;
pszPass = pInst->hProxy->pCfg->pszPass;
// FIXME: We should enable logging dependent on a loglevel rather than just enabling it
if (pInst->hProxy->pCfg->bVerbose)
Imo2S_SetLog (pInst->hInst, pInst->hProxy->pCfg->fpLog);
while (pInst->hProxy->iRunning)
{
if (RcvPacket(pInst, &uiLength, sizeof(uiLength))<=0) break;
LockMutex(pInst->sendmutex);
if (uiLength == 0)
{
if (RcvPacket(pInst, &command, 1)<=0)
{
UnlockMutex(pInst->sendmutex);
break;
}
switch (command)
{
case AUTHENTICATE:
if (pInst->hProxy->pMyCfg->pszAuthPass) reply=1;
break;
case CAPABILITIES:
if (pInst->hProxy->pMyCfg->pszAuthPass) reply=USE_AUTHENTICATION;
break;
}
if (SendPacket (pInst, &reply, 1)<=0)
{
UnlockMutex(pInst->sendmutex);
break;
}
UnlockMutex(pInst->sendmutex);
continue;
}
if (uiLength >= cbMsgBuf)
{
pszMsgBuf = realloc (pszMsgBuf, uiLength+1);
if (!pszMsgBuf)
{
UnlockMutex(pInst->sendmutex);
break;
}
cbMsgBuf=uiLength+1;
}
if (RcvPacket(pInst, pszMsgBuf, uiLength)<=0)
{
UnlockMutex(pInst->sendmutex);
break;
}
if (command)
{
if (command == AUTHENTICATE)
{
bAuthenticated = pInst->hProxy->pMyCfg->pszAuthPass && strcmp(pInst->hProxy->pMyCfg->pszAuthPass, pszMsgBuf) == 0;
if (SendPacket (pInst, &bAuthenticated, 1)<=0)
{
UnlockMutex(pInst->sendmutex);
break;
}
}
command = 0;
}
UnlockMutex(pInst->sendmutex);
if (iLogin)
{
if (Imo2S_Login (pInst->hInst, pszUser, pszPass, &pszError) != 1)
{
pInst->hProxy->pCfg->logerror (stderr, "Connection %d: Cannot login with (%s/****): %s\n",
pInst->hSock, pszUser, pszError);
FreeConnection (pInst);
return;
}
iLogin = 0;
}
if (pInst->hProxy->pMyCfg->pszAuthPass && !bAuthenticated)continue;
pszMsgBuf[uiLength]=0;
if (pInst->hProxy->pCfg->bVerbose && pInst->hProxy->pCfg->fpLog)
{
fprintf (pInst->hProxy->pCfg->fpLog, "%03d< [%s]\n", pInst->hSock, pszMsgBuf);
fflush (pInst->hProxy->pCfg->fpLog);
}
if (!iConnected)
{
LockMutex (pInst->connected);
iConnected = pInst->iConnectionStat;
if (!iConnected)
{
pInst->hProxy->pCfg->logerror(stderr, "Invalid username / password");
}
}
Imo2S_Send (pInst->hInst, pszMsgBuf);
}
FreeConnection (pInst);
if (pszMsgBuf) free (pszMsgBuf);
return;
}
void MainProc() {
for (;;) {
int MaxFd;
fd_set ReadQ;
fd_set WriteQ;
if (Reconnect) {
close(connection->Fd);
FreeConnection(connection);
// To prevent floods. I know, this should be rather handled by ewrecv, but that would be nontrivial.
sleep(2);
AttachConnection();
Reconnect = 0;
}
MaxFd = 0;
FD_ZERO(&ReadQ);
FD_ZERO(&WriteQ);
// stdin
if (read_from_stdin
&& !want_quit
&& strlen(Commands) < COMMANDS_MAXLEN) {
FD_SET(0, &ReadQ);
}
if (connection) {
FD_SET(connection->Fd, &ReadQ);
if (connection->Fd > MaxFd) MaxFd = connection->Fd;
if (connection->WriteBuffer) FD_SET(connection->Fd, &WriteQ);
}
struct timeval to;
to.tv_sec = timeout_denominator;
to.tv_usec = 0;
struct timeval *top = &to;
if (timeout_denominator == 0) top = NULL;
int s = select(MaxFd + 1, &ReadQ, &WriteQ, 0, top);
if (s < 0) {
if (errno == EINTR) continue;
Done(1);
} else if (s == 0) {
// timeout
} else {
// stdin
if (read_from_stdin
&& strlen(Commands) < COMMANDS_MAXLEN
&& FD_ISSET(0, &ReadQ)) {
char buf[256+1] = "";
int to_read = COMMANDS_MAXLEN - strlen(Commands);
if (to_read > 256) to_read = 256;
int r = read(0, buf, to_read);
buf[r] = 0;
if (r == 0) {
want_quit = 1;
TryQuit();
} else {
strncat(Commands, buf, r);
SendNextCommand();
}
}
// Exchange input
if (connection && FD_ISSET(connection->Fd, &ReadQ)) {
errno = 0;
if (DoRead(connection) <= 0) {
if (detaching) {
Done(0);
} else {
Done(1);
}
} else {
int Chr;
while (Read(connection, &Chr, 1)) {
TestIProtoChar(connection, Chr);
}
}
}
// Exchange output
if (connection && FD_ISSET(connection->Fd, &WriteQ)) {
if (DoWrite(connection) < 0) Done(1);
}
}
if (login_timeout && login_start && !logged_in) {
if (time(NULL) - login_start > login_timeout) Done(103);
}
if (command_timeout && command_start && jobs) {
if (time(NULL) - command_start > command_timeout) Done(104);
}
}
}
void HttpManager::HandleResponse(HttpResponse& response) {
HttpClient& client = *FindClient(response.uid);
switch (client.mode()) {
case kHttpServiceAuthenticateUser:
case kHttpServiceGetMetadataById:
case kHttpServiceGetMetadataByIdV2:
case kHttpServiceSearchTitle:
case kHttpServiceAddLibraryEntry:
case kHttpServiceDeleteLibraryEntry:
case kHttpServiceGetLibraryEntries:
case kHttpServiceUpdateLibraryEntry:
ServiceManager.HandleHttpResponse(response);
break;
case kHttpGetLibraryEntryImage: {
int anime_id = static_cast<int>(response.parameter);
SaveToFile(client.write_buffer_, anime::GetImagePath(anime_id));
if (ImageDatabase.Load(anime_id, true, false))
ui::OnLibraryEntryImageChange(anime_id);
break;
}
case kHttpFeedCheck:
case kHttpFeedCheckAuto: {
Feed* feed = reinterpret_cast<Feed*>(response.parameter);
if (feed) {
bool automatic = client.mode() == kHttpFeedCheckAuto;
Aggregator.HandleFeedCheck(*feed, client.write_buffer_, automatic);
}
break;
}
case kHttpFeedDownload:
case kHttpFeedDownloadAll: {
if (Aggregator.ValidateFeedDownload(client.request(), response)) {
auto feed = reinterpret_cast<Feed*>(response.parameter);
if (feed) {
bool download_all = client.mode() == kHttpFeedDownloadAll;
Aggregator.HandleFeedDownload(*feed, client.write_buffer_, download_all);
}
}
break;
}
case kHttpTwitterRequest:
case kHttpTwitterAuth:
case kHttpTwitterPost:
::Twitter.HandleHttpResponse(client.mode(), response);
break;
case kHttpTaigaUpdateCheck:
if (Taiga.Updater.ParseData(response.body))
if (Taiga.Updater.IsDownloadAllowed())
break;
ui::OnUpdateFinished();
break;
case kHttpTaigaUpdateDownload:
SaveToFile(client.write_buffer_, Taiga.Updater.GetDownloadPath());
Taiga.Updater.RunInstaller();
ui::OnUpdateFinished();
break;
}
FreeConnection(client.request_.url.host);
ProcessQueue();
}
ArticleDownloader::EStatus ArticleDownloader::Download()
{
const char* szResponse = NULL;
EStatus Status = adRunning;
if (m_pConnection->GetNewsServer()->GetJoinGroup())
{
// change group
for (FileInfo::Groups::iterator it = m_pFileInfo->GetGroups()->begin(); it != m_pFileInfo->GetGroups()->end(); it++)
{
szResponse = m_pConnection->JoinGroup(*it);
if (szResponse && !strncmp(szResponse, "2", 1))
{
break;
}
}
Status = CheckResponse(szResponse, "could not join group");
if (Status != adFinished)
{
return Status;
}
}
// retrieve article
char tmp[1024];
snprintf(tmp, 1024, "ARTICLE %s\r\n", m_pArticleInfo->GetMessageID());
tmp[1024-1] = '\0';
for (int retry = 3; retry > 0; retry--)
{
szResponse = m_pConnection->Request(tmp);
if ((szResponse && !strncmp(szResponse, "2", 1)) || m_pConnection->GetAuthError())
{
break;
}
}
Status = CheckResponse(szResponse, "could not fetch article");
if (Status != adFinished)
{
return Status;
}
// positive answer!
if (g_pOptions->GetDecode())
{
m_YDecoder.Clear();
m_YDecoder.SetAutoSeek(g_pOptions->GetDirectWrite());
m_YDecoder.SetCrcCheck(g_pOptions->GetCrcCheck());
m_UDecoder.Clear();
}
m_pOutFile = NULL;
bool bBody = false;
bool bEnd = false;
const int LineBufSize = 1024*10;
char* szLineBuf = (char*)malloc(LineBufSize);
Status = adRunning;
while (!IsStopped())
{
SetLastUpdateTimeNow();
// Throttle the bandwidth
while (!IsStopped() && (g_pOptions->GetDownloadRate() > 0.0f) &&
(g_pDownloadSpeedMeter->CalcCurrentDownloadSpeed() > g_pOptions->GetDownloadRate()))
{
SetLastUpdateTimeNow();
usleep(10 * 1000);
}
int iLen = 0;
char* line = m_pConnection->ReadLine(szLineBuf, LineBufSize, &iLen);
g_pDownloadSpeedMeter->AddSpeedReading(iLen);
// Have we encountered a timeout?
if (!line)
{
if (!IsStopped())
{
warn("Article %s @ %s (%s) failed: Unexpected end of article", m_szInfoName,
m_pConnection->GetNewsServer()->GetName(), m_pConnection->GetHost());
}
Status = adFailed;
break;
}
//detect end of article
if (!strcmp(line, ".\r\n") || !strcmp(line, ".\n"))
{
bEnd = true;
break;
}
//detect lines starting with "." (marked as "..")
if (!strncmp(line, "..", 2))
{
line++;
iLen--;
}
if (!bBody)
{
// detect body of article
if (*line == '\r' || *line == '\n')
{
bBody = true;
}
// check id of returned article
else if (!strncmp(line, "Message-ID: ", 12))
{
char* p = line + 12;
if (strncmp(p, m_pArticleInfo->GetMessageID(), strlen(m_pArticleInfo->GetMessageID())))
{
if (char* e = strrchr(p, '\r')) *e = '\0'; // remove trailing CR-character
warn("Article %s @ %s (%s) failed: Wrong message-id, expected %s, returned %s", m_szInfoName,
m_pConnection->GetNewsServer()->GetName(), m_pConnection->GetHost(), m_pArticleInfo->GetMessageID(), p);
Status = adFailed;
break;
}
}
}
else if (m_eFormat == Decoder::efUnknown && g_pOptions->GetDecode())
{
m_eFormat = Decoder::DetectFormat(line, iLen);
}
// write to output file
if (((bBody && m_eFormat != Decoder::efUnknown) || !g_pOptions->GetDecode()) && !Write(line, iLen))
{
Status = adFatalError;
break;
}
}
free(szLineBuf);
if (m_pOutFile)
{
fclose(m_pOutFile);
}
if (!bEnd && Status == adRunning && !IsStopped())
{
warn("Article %s @ %s (%s) failed: article incomplete", m_szInfoName,
m_pConnection->GetNewsServer()->GetName(), m_pConnection->GetHost());
Status = adFailed;
}
if (IsStopped())
{
Status = adFailed;
}
if (Status == adRunning)
{
FreeConnection(true);
return DecodeCheck();
}
else
{
remove(m_szTempFilename);
return Status;
}
}
/*
* How server management (for one particular article) works:
- there is a list of failed servers which is initially empty;
- level is initially 0;
<loop>
- request a connection from server pool for current level;
Exception: this step is skipped for the very first download attempt, because a
level-0 connection is initially passed from queue manager;
- try to download from server;
- if connection to server cannot be established or download fails due to interrupted connection,
try again (as many times as needed without limit) the same server until connection is OK;
- if download fails with error "Not-Found" (article or group not found) or with CRC error,
add the server to failed server list;
- if download fails with general failure error (article incomplete, other unknown error
codes), try the same server again as many times as defined by option <Retries>; if all attempts
fail, add the server to failed server list;
- if all servers from current level were tried, increase level;
- if all servers from all levels were tried, break the loop with failure status.
<end-loop>
*/
void ArticleDownloader::Run()
{
debug("Entering ArticleDownloader-loop");
SetStatus(adRunning);
BuildOutputFilename();
m_szResultFilename = m_pArticleInfo->GetResultFilename();
if (g_pOptions->GetContinuePartial())
{
if (Util::FileExists(m_szResultFilename))
{
// file exists from previous program's start
detail("Article %s already downloaded, skipping", m_szInfoName);
FreeConnection(true);
SetStatus(adFinished);
Notify(NULL);
return;
}
}
EStatus Status = adFailed;
int iRetries = g_pOptions->GetRetries() > 0 ? g_pOptions->GetRetries() : 1;
int iRemainedRetries = iRetries;
ServerPool::Servers failedServers;
failedServers.reserve(g_pServerPool->GetServers()->size());
NewsServer* pWantServer = NULL;
NewsServer* pLastServer = NULL;
int iLevel = 0;
int iServerConfigGeneration = g_pServerPool->GetGeneration();
while (!IsStopped())
{
Status = adFailed;
SetStatus(adWaiting);
while (!m_pConnection && !(IsStopped() || iServerConfigGeneration != g_pServerPool->GetGeneration()))
{
m_pConnection = g_pServerPool->GetConnection(iLevel, pWantServer, &failedServers);
usleep(5 * 1000);
}
SetLastUpdateTimeNow();
SetStatus(adRunning);
if (IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration())
{
Status = adRetry;
break;
}
pLastServer = m_pConnection->GetNewsServer();
m_pConnection->SetSuppressErrors(false);
// test connection
bool bConnected = m_pConnection && m_pConnection->Connect();
if (bConnected && !IsStopped())
{
// Okay, we got a Connection. Now start downloading.
detail("Downloading %s @ %s (%s)", m_szInfoName,
m_pConnection->GetNewsServer()->GetName(), m_pConnection->GetHost());
Status = Download();
}
if (bConnected)
{
if (Status == adConnectError)
{
m_pConnection->Disconnect();
bConnected = false;
Status = adFailed;
}
else
{
// freeing connection allows other threads to start.
// we doing this only if the problem was with article or group.
// if the problem occurs by connecting or authorization we do not
// free the connection, to prevent starting of thousands of threads
// (cause each of them will also free it's connection after the
// same connect-error).
FreeConnection(Status == adFinished || Status == adNotFound);
}
}
if (Status == adFinished || Status == adFatalError)
{
break;
}
pWantServer = NULL;
if (bConnected && Status == adFailed)
{
iRemainedRetries--;
}
if (!bConnected || (Status == adFailed && iRemainedRetries > 0))
{
pWantServer = pLastServer;
}
if (pWantServer &&
!(IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration()))
{
detail("Waiting %i sec to retry", g_pOptions->GetRetryInterval());
SetStatus(adWaiting);
int msec = 0;
while (!(IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration()) &&
msec < g_pOptions->GetRetryInterval() * 1000)
{
usleep(100 * 1000);
msec += 100;
}
SetLastUpdateTimeNow();
SetStatus(adRunning);
}
if (IsStopped() || g_pOptions->GetPauseDownload() || g_pOptions->GetPauseDownload2() ||
iServerConfigGeneration != g_pServerPool->GetGeneration())
{
Status = adRetry;
break;
}
if (!pWantServer)
{
failedServers.push_back(pLastServer);
// if all servers from current level were tried, increase level
// if all servers from all levels were tried, break the loop with failure status
bool bAllServersOnLevelFailed = true;
for (ServerPool::Servers::iterator it = g_pServerPool->GetServers()->begin(); it != g_pServerPool->GetServers()->end(); it++)
{
NewsServer* pCandidateServer = *it;
if (pCandidateServer->GetNormLevel() == iLevel)
{
bool bServerFailed = !pCandidateServer->GetActive();
if (!bServerFailed)
{
for (ServerPool::Servers::iterator it = failedServers.begin(); it != failedServers.end(); it++)
{
NewsServer* pIgnoreServer = *it;
if (pIgnoreServer == pCandidateServer ||
(pIgnoreServer->GetGroup() > 0 && pIgnoreServer->GetGroup() == pCandidateServer->GetGroup() &&
pIgnoreServer->GetNormLevel() == pCandidateServer->GetNormLevel()))
{
bServerFailed = true;
break;
}
}
}
if (!bServerFailed)
{
bAllServersOnLevelFailed = false;
break;
}
}
}
if (bAllServersOnLevelFailed)
{
if (iLevel < g_pServerPool->GetMaxNormLevel())
{
detail("Article %s @ all level %i servers failed, increasing level", m_szInfoName, iLevel);
iLevel++;
}
else
{
warn("Article %s @ all servers failed", m_szInfoName);
Status = adFailed;
break;
}
}
iRemainedRetries = iRetries;
}
}
FreeConnection(Status == adFinished);
if (m_bDuplicate)
{
Status = adFinished;
}
if (Status != adFinished && Status != adRetry)
{
Status = adFailed;
}
if (IsStopped())
{
detail("Download %s cancelled", m_szInfoName);
Status = adRetry;
}
if (Status == adFailed)
{
warn("Download %s failed", m_szInfoName);
}
SetStatus(Status);
Notify(NULL);
debug("Exiting ArticleDownloader-loop");
}
static bool _ReaderRun(ThreadData _reader) {
Reader* reader = (Reader*)_reader;
QueueItem* item = NewQueueItem();
Connection* connection = NULL;
SelectionKey* key = NULL;
int numConnectionsReady = 0;
while(Dequeue(reader->queue, item, false)) {
switch(QueueItemGetType(item)) {
case AIO4C_QUEUE_ITEM_EXIT:
FreeQueueItem(&item);
return false;
case AIO4C_QUEUE_ITEM_DATA:
connection = (Connection*)QueueDataItemGet(item);
connection->readKey = Register(reader->selector, AIO4C_OP_READ, connection->socket, (void*)connection);
Log(AIO4C_LOG_LEVEL_DEBUG, "managing connection %s", connection->string);
ConnectionManagedBy(connection, AIO4C_CONNECTION_OWNER_READER);
reader->load++;
break;
case AIO4C_QUEUE_ITEM_EVENT:
connection = (Connection*)QueueEventItemGetSource(item);
if (QueueEventItemGetEvent(item) == AIO4C_CLOSE_EVENT) {
if (connection->readKey != NULL) {
Unregister(reader->selector, connection->readKey, true, NULL);
connection->readKey = NULL;
}
Log(AIO4C_LOG_LEVEL_DEBUG, "close received for connection %s", connection->string);
reader->load--;
if (ConnectionNoMoreUsed(connection, AIO4C_CONNECTION_OWNER_READER)) {
Log(AIO4C_LOG_LEVEL_DEBUG, "freeing connection %s", connection->string);
FreeConnection(&connection);
}
} else if (QueueEventItemGetEvent(item) == AIO4C_PENDING_CLOSE_EVENT) {
Log(AIO4C_LOG_LEVEL_DEBUG, "pending close received for connection %s", connection->string);
}
break;
default:
break;
}
}
ProbeTimeStart(AIO4C_TIME_PROBE_IDLE);
numConnectionsReady = Select(reader->selector);
ProbeTimeEnd(AIO4C_TIME_PROBE_IDLE);
if (numConnectionsReady > 0) {
ProbeTimeStart(AIO4C_TIME_PROBE_NETWORK_READ);
while (SelectionKeyReady(reader->selector, &key)) {
if (SelectionKeyIsOperationSuccessful(key)) {
connection = ConnectionRead(SelectionKeyGetAttachment(key));
} else {
Log(AIO4C_LOG_LEVEL_WARN, "select operation unsuccessful for connection %s", ((Connection*)SelectionKeyGetAttachment(key))->string);
}
}
ProbeTimeEnd(AIO4C_TIME_PROBE_NETWORK_READ);
}
FreeQueueItem(&item);
return true;
}
static LONG APIENTRY WndProc(HWND hWnd, UINT message, UINT wParam, LONG lParam)
{
switch (message)
{
case WM_CREATE:
{
LPCREATESTRUCT lpCr = (LPCREATESTRUCT)lParam;
SetWindowLongPtr (hWnd, GWLP_USERDATA, (LONG_PTR)lpCr->lpCreateParams);
SetTimer (hWnd, 0, 60000, NULL);
break;
}
case WM_COPYDATA:
{
PCOPYDATASTRUCT pCopyData = (PCOPYDATASTRUCT)lParam;
CONNINST *pInst;
IMO2SPROXY_INST *hProxy = (IMO2SPROXY_INST*)GetWindowLongPtr(hWnd, GWLP_USERDATA);
if (pInst = FindClient (hProxy, (HWND)wParam))
{
if (pInst->hProxy->pMyCfg->bDelayLogin && pInst->iConnectionStat < 1)
{
char *pszError;
if ((pInst->iConnectionStat = Imo2S_Login (pInst->hInst, hProxy->pCfg->pszUser, hProxy->pCfg->pszPass, &pszError)) != 1)
{
pInst->hProxy->pCfg->logerror (stderr, "Connection %08X: Cannot login with (%s/****): %s\n",
pInst->hWnd, hProxy->pCfg->pszUser, pszError);
FreeConnection(pInst);
free (List_Pop(hProxy->hClients));
PostMessage ((HWND)wParam, m_ControlAPIAttach, (WPARAM)hWnd, SKYPECONTROLAPI_ATTACH_REFUSED);
return 0;
}
}
LockMutex(pInst->sendmutex);
if (pInst->hProxy->pCfg->bVerbose && pInst->hProxy->pCfg->fpLog)
{
fprintf (pInst->hProxy->pCfg->fpLog, "%08X< [%s]\n", pInst->hWnd, pCopyData->lpData);
fflush (pInst->hProxy->pCfg->fpLog);
}
Imo2S_Send (pInst->hInst, pCopyData->lpData);
UnlockMutex(pInst->sendmutex);
}
return 1;
}
case WM_TIMER:
// Housekeeping timer
CleanConnections (((IMO2SPROXY_INST*)GetWindowLongPtr(hWnd, GWLP_USERDATA))->hClients);
break;
case WM_DESTROY:
KillTimer (hWnd, 0);
break;
default:
if (message == m_ControlAPIDiscover)
{
CONNINST *pInst;
IMO2SPROXY_INST *hProxy = (IMO2SPROXY_INST*)GetWindowLongPtr(hWnd, GWLP_USERDATA);
char *pszError;
if (!(pInst = FindClient (hProxy, (HWND)wParam)))
{
pInst = (CONNINST*)calloc (1, sizeof(CONNINST));
if (!pInst) break;
List_Push(hProxy->hClients, pInst);
pInst->hProxy = hProxy;
pInst->hWnd = (HWND)wParam;
if (hProxy->pCfg->bVerbose && hProxy->pCfg->fpLog)
fprintf (hProxy->pCfg->fpLog, "Imo2sproxy::SkypeControlAPIDiscover\n");
if (!(pInst->hInst = Imo2S_Init(EventHandler, pInst, hProxy->pCfg->iFlags)))
{
hProxy->pCfg->logerror (stderr, "Connection %08X: Cannot start Imo2Skype instance.\n", pInst->hWnd);
free (List_Pop(hProxy->hClients));
PostMessage ((HWND)wParam, m_ControlAPIAttach, (WPARAM)hWnd, SKYPECONTROLAPI_ATTACH_REFUSED);
return 0;
}
// FIXME: We should enable logging dependent on a loglevel rather than just enabling it
if (hProxy->pCfg->bVerbose)
Imo2S_SetLog (pInst->hInst, hProxy->pCfg->fpLog);
InitMutex(pInst->sendmutex);
if (!pInst->hProxy->pMyCfg->bDelayLogin)
{
PostMessage ((HWND)wParam, m_ControlAPIAttach, (WPARAM)hWnd, SKYPECONTROLAPI_ATTACH_NOT_AVAILABLE);
if ((pInst->iConnectionStat = Imo2S_Login (pInst->hInst, hProxy->pCfg->pszUser, hProxy->pCfg->pszPass, &pszError)) != 1)
{
pInst->hProxy->pCfg->logerror (stderr, "Connection %08X: Cannot login with (%s/****): %s\n",
pInst->hWnd, hProxy->pCfg->pszUser, pszError);
FreeConnection(pInst);
free (List_Pop(hProxy->hClients));
PostMessage ((HWND)wParam, m_ControlAPIAttach, (WPARAM)hWnd, SKYPECONTROLAPI_ATTACH_REFUSED);
return 0;
}
PostMessage ((HWND)wParam, m_ControlAPIAttach, (WPARAM)hWnd, SKYPECONTROLAPI_ATTACH_API_AVAILABLE);
}
else
{
SendMessage ((HWND)wParam, m_ControlAPIAttach, (WPARAM)hWnd, SKYPECONTROLAPI_ATTACH_SUCCESS);
}
return 0;
}
else
SendMessage ((HWND)wParam, m_ControlAPIAttach, (WPARAM)hWnd, SKYPECONTROLAPI_ATTACH_SUCCESS);
return 0;
}
break;
}
return (DefWindowProc(hWnd, message, wParam, lParam));
}
//-------------------------------------------------------
void OdbcPersistor::RestoreAll()
{
int connectionAttempts = 0;
bool errorReported = false;
EntityIdSet restoredObjects;
const int binaryLargeSize = Safir::Dob::PersistenceParameters::BinaryDataColumnSize();
const int binarySmallSize = Safir::Dob::PersistenceParameters::BinarySmallDataColumnSize();
const int xmlSize = Safir::Dob::PersistenceParameters::XmlDataColumnSize();
SQLHDBC getAllConnection = SQL_NULL_HANDLE;
bool isConnected = false;
bool getAllIsValid = false;
SQLHSTMT getAllStatement = SQL_NULL_HANDLE;
Safir::Dob::Typesystem::Int64 typeId = 0;
Safir::Dob::Typesystem::Int64 instance = 0;
Safir::Dob::Typesystem::Int64 handlerId = 0;
boost::scoped_array<unsigned char> storeBinarySmallData(new unsigned char[binarySmallSize]);
SQLLEN currentSmallDataSize = 0;
boost::scoped_array<unsigned char> storeBinaryLargeData(new unsigned char[binaryLargeSize]);
SQLLEN currentLargeDataSize = 0;
boost::scoped_array<char> xmlBuffer(new char[xmlSize]);
boost::scoped_array<wchar_t> xmlBufferW(new wchar_t[xmlSize / sizeof(wchar_t)]);
SQLLEN currentXmlSize = 0;
const boost::chrono::steady_clock::time_point startTime = boost::chrono::steady_clock::now();
const SQLRETURN ret = ::SQLAllocHandle(SQL_HANDLE_DBC, m_environment, &getAllConnection);
if (!SQL_SUCCEEDED(ret))
{
OdbcHelper::ThrowException(SQL_HANDLE_ENV, m_environment);
}
bool done = false;
while (!done)
{
try
{
ConnectIfNeeded(getAllConnection, isConnected, connectionAttempts);
for (;;)
{
//getAll statement execution (will loop here until we successfully execute)
if (!getAllIsValid)
{
m_helper.AllocStatement(&getAllStatement, getAllConnection);
m_helper.Prepare(
getAllStatement,
"SELECT typeId, instance, handlerid, xmlData, binaryData, binarySmallData "
"FROM PersistentEntity");
m_helper.BindColumnInt64(getAllStatement, 1, &typeId);
m_helper.BindColumnInt64(getAllStatement, 2, &instance);
m_helper.BindColumnInt64(getAllStatement, 3, &handlerId);
if (Safir::Dob::PersistenceParameters::TextColumnsAreUtf8())
{
BindColumnString(getAllStatement, 4, xmlSize, xmlBuffer.get(), ¤tXmlSize);
}
else
{
BindColumnStringW(getAllStatement,
4,
xmlSize / sizeof(wchar_t),
xmlBufferW.get(),
¤tXmlSize);
}
m_helper.BindColumnBinary(getAllStatement, 5, binaryLargeSize, storeBinaryLargeData.get(), ¤tLargeDataSize);
m_helper.BindColumnBinary(getAllStatement, 6, binarySmallSize, storeBinarySmallData.get(), ¤tSmallDataSize);
}
try
{
m_helper.Execute(getAllStatement);
break;
}
catch(const OdbcException& e)
{
const std::wstring err = Safir::Dob::Typesystem::Utilities::ToWstring(e.what());
m_debug << "Caught a RetryException in GetAll:\n" << err << std::endl;
boost::this_thread::sleep_for(RETRY_EXCEPTION_DELAY);
}
}
for (;;)
{
if (!m_helper.Fetch(getAllStatement))
{//we've got all rows!
done = true;
break;
}
const Safir::Dob::Typesystem::EntityId entityId
(typeId,
Safir::Dob::Typesystem::InstanceId(instance));
const Safir::Dob::Typesystem::HandlerId handler(handlerId);
auto findIt = GetPersistentTypes().find(entityId.GetTypeId());
if (findIt == GetPersistentTypes().end())
{ //not to be persisted!
Remove(entityId);
continue;
}
if (restoredObjects.find(entityId) != restoredObjects.end())
{ //already restored this object
continue;
}
try
{
Safir::Dob::ConnectionAspectInjector injector(m_dobConnection);
if (currentXmlSize != SQL_NULL_DATA)
{ //some xml persistent data set
std::wstring xml;
if (Safir::Dob::PersistenceParameters::TextColumnsAreUtf8())
{
xml = Safir::Dob::Typesystem::Utilities::ToWstring(xmlBuffer.get());
}
else
{
xml = xmlBufferW.get();
}
m_debug
<< "Restoring from xml" << entityId
<< ", size = " << xml.size()
<< ". First 100 chars of the data: " << xml.substr(0,100)
<< std::endl;
const Safir::Dob::Typesystem::ObjectPtr object =
Safir::Dob::Typesystem::Serialization::ToObject(xml);
const Safir::Dob::EntityPtr entity =
boost::dynamic_pointer_cast<Safir::Dob::Entity>(object);
m_debug << "Successfully deserialized" <<std::endl;
injector.InitialSet(entity, entityId.GetInstanceId(), handler);
m_debug << "InitialSet successful"<<std::endl;
Safir::Dob::Typesystem::BinarySerialization bin;
Safir::Dob::Typesystem::Serialization::ToBinary(entity, bin);
Store(entityId, handler, bin, true);
m_debug << "Stored it as binary" << std::endl;
}
else if (currentSmallDataSize != SQL_NULL_DATA)
{
const char * const data = reinterpret_cast<const char * const>(storeBinarySmallData.get());
m_debug << "Restoring " << entityId << " from binary " <<std::endl;
Safir::Dob::EntityPtr entity =
boost::dynamic_pointer_cast<Safir::Dob::Entity>
(Safir::Dob::Typesystem::ObjectFactory::Instance().CreateObject(data));
m_debug << "Successfully deserialized" <<std::endl;
injector.InitialSet(entity, entityId.GetInstanceId(), handler);
m_debug << "InitialSet successful"<<std::endl;
}
else
{
if (currentLargeDataSize != SQL_NULL_DATA)
{ //some binarypersistent data set
const char * const data = reinterpret_cast<const char * const>(storeBinaryLargeData.get());
m_debug << "Restoring " << entityId << " from binary " <<std::endl;
Safir::Dob::EntityPtr entity =
boost::dynamic_pointer_cast<Safir::Dob::Entity>
(Safir::Dob::Typesystem::ObjectFactory::Instance().CreateObject(data));
m_debug << "Successfully deserialized" <<std::endl;
injector.InitialSet(entity, entityId.GetInstanceId(), handler);
m_debug << "InitialSet successful"<<std::endl;
}
else
{
m_debug << "No data set for " << entityId <<std::endl;
}
}
//add the objectid to the list of objectids that we've done, so that we can resume from where
//we were in case the db goes down during restore
restoredObjects.insert(entityId);
}
catch(const Safir::Dob::Typesystem::IllegalValueException &)
{
m_debug << "Could not restore "
<< entityId.ToString()
<< ", removing it" << std::endl;
Safir::Logging::SendSystemLog(Safir::Logging::Error,
L"Failed to restore entity" +
entityId.ToString() +
L", will remove persisted data.");
//we don't want to try it again if the connection fails later.
restoredObjects.insert(entityId);
//remove the row from the db
Remove(entityId);
//since we did not remove the objectid from persistentObjects an empty row will be inserted below.
}
}
}
catch(const OdbcException& e)
{
const std::wstring err = Safir::Dob::Typesystem::Utilities::ToWstring(e.what());
m_debug << "Caught a ReconnectException in RestoreAll:\n" << err << std::endl;
if (connectionAttempts > REPORT_AFTER_RECONNECTS && !errorReported)
{
Safir::Logging::SendSystemLog(Safir::Logging::Error,
L"RestoreAll: Failed to connect to the database, will keep trying. Exception info: " +
err);
errorReported = true;
}
if (isConnected)
{
Disconnect(getAllConnection);
isConnected = false;
}
getAllIsValid = false;
boost::this_thread::sleep_for(RECONNECT_EXCEPTION_DELAY);
}
}
if (errorReported)
{
Safir::Logging::SendSystemLog(Safir::Logging::Informational,
L"Successfully connected to the database");
errorReported = false;
connectionAttempts = 0;
}
try
{
if (isConnected)
{
Disconnect(getAllConnection);
}
FreeConnection(getAllConnection);
}
catch(const OdbcException& e)
{
const std::wstring err = Safir::Dob::Typesystem::Utilities::ToWstring(e.what());
Safir::Logging::SendSystemLog(Safir::Logging::Error,
L"Whoops. Error while disconnecting the getAllConnection. Ignoring this and moving on. Exception info: " +
err);
}
m_debug << "RestoreAll completed: "
<< restoredObjects.size()
<< " objects restored in time "
<< boost::chrono::steady_clock::now() - startTime << std::endl;
}
void HttpManager::HandleRedirect(const std::wstring& current_host,
const std::wstring& next_host) {
FreeConnection(current_host);
AddConnection(next_host);
}
