void NetworkManager::cleanUpLinks(Links & links, Assignment * assignment)
{
for ( Links::iterator l = links.begin(); l != links.end(); l++ )
{
NetPath path = assignment->GetAssignment(*l);
if ( !path.empty() )
removeAssignment(*l, path);
assignment->RemoveAssignment(*l);
}
}
void Page::ProcessLinks(OutputDev *pOut, Catalog *pCatalog)
{
Links *pLinks = GetLinks(pCatalog);
if (pLinks)
{
for (int nIndex = 0; nIndex < pLinks->GetLinksCount(); ++nIndex)
{
pOut->ProcessLink(pLinks->GetLink(nIndex), pCatalog);
}
delete pLinks;
}
}
void Print(std::ostream& stream)
{
std::vector<Objects::const_iterator> referencedObjects;
referencedObjects.reserve(links.size());
for(Links::const_iterator i = links.begin(); i != links.end(); ++i)
referencedObjects.push_back(i->second);
std::sort(referencedObjects.begin(), referencedObjects.end());
referencedObjects.resize(std::unique(referencedObjects.begin(), referencedObjects.end()) - referencedObjects.begin());
// пока просто вывести
for(Objects::const_iterator object = objects.begin(); object != objects.end(); ++object)
{
if(std::binary_search(referencedObjects.begin(), referencedObjects.end(), object))
continue;
Print(stream, object);
}
}
/** Gets the results of the trace, then returns the first detector
* (that is NOT a monitor) found in the results.
* @return sptr to IDetector, or an invalid sptr if not found
*/
IDetector_const_sptr InstrumentRayTracer::getDetectorResult() const {
Links results = this->getResults();
// Go through all results
Links::const_iterator resultItr = results.begin();
for (; resultItr != results.end(); ++resultItr) {
IComponent_const_sptr component =
m_instrument->getComponentByID(resultItr->componentID);
IDetector_const_sptr det =
boost::dynamic_pointer_cast<const IDetector>(component);
if (det) {
if (!det->isMonitor()) {
return det;
}
} // (is a detector)
} // each ray tracer result
return IDetector_const_sptr();
}
void Print(std::ostream& stream, Objects::const_iterator object, bool last = false)
{
for(size_t i = 0; i < levels.size(); ++i)
if(i == levels.size() - 1)
stream << "|_";
else if(levels[i])
stream << "| ";
else
stream << " ";
bool cyclicReference = std::find(levelObjects.begin(), levelObjects.end(), object) != levelObjects.end();
if(cyclicReference)
stream << "CYCLIC ";
stream << object->data << ", " << object->info << "\n";
if(cyclicReference)
return;
if(last)
levels[levels.size() - 1] = false;
levelObjects.push_back(object);
Links::const_iterator link = links.lower_bound(object);
Links::const_iterator endLink = links.upper_bound(object);
if(link != endLink)
{
levels.push_back(true);
Links::const_iterator nextLink;
for(; link != links.end() && link->first == object; link = nextLink)
{
nextLink = link;
++nextLink;
Print(stream, link->second, nextLink == links.end() || nextLink->first != object);
}
levels.pop_back();
}
levelObjects.pop_back();
}
PtrTracer(const ManagedHeap& heap)
{
// сформировать список объектов
objects.reserve(heap.allocations.size());
for(Allocations::const_iterator i = heap.allocations.begin(); i != heap.allocations.end(); ++i)
objects.push_back(Object(i->first, i->second.size, i->second.info));
std::sort(objects.begin(), objects.end(), sorter);
// сформировать карту ссылок
for(Ptrs::const_iterator i = heap.ptrs.begin(); i != heap.ptrs.end(); ++i)
{
// найти объект, в котором содержится указатель
Objects::const_iterator j = std::upper_bound(objects.begin(), objects.end(), i->first, sorter);
if(j > objects.begin())
{
--j;
if((size_t)((char*)i->first - (char*)j->data) < j->size)
// да, указатель содержится в этом объекте
// получить объект, на который указывает указатель, и добавить ссылку
links.insert(std::make_pair(j, std::lower_bound(objects.begin(), objects.end(), i->second, sorter)));
}
}
}
NetPath VirtualLinkRouter::routeKShortestPathsALL(VirtualLink * virtualLink, Network * network, std::vector<NetPath> * pathStorage)
{
// links and switches that would be removed from the network,
// they should be restored after algorithm's finish
Links removedLinks;
Switches removedSwitches;
// first, create the graph with decreased capacities
decreaseCapacities(virtualLink, network, &removedLinks, &removedSwitches);
// Yen's algorithm
NetPath shortest = searchPathDejkstra(virtualLink, network, K_SHORTEST_PATHS);
if ( shortest.size() == 0 )
{
restoreCapacities(virtualLink, network, &removedLinks, &removedSwitches);
return NetPath(); // no path found!
}
pathStorage->push_back(shortest);
Links& links = network->getLinks();
unsigned pathsFound = 1;
long pathWeight = calculateKShortestPathWeight(shortest);
bool isNewPathFound = true;
while ( isNewPathFound && pathsFound < Criteria::kShortestPathDepth() )
{
NetPath candidate = shortest;
NetPath::iterator it = shortest.begin();
NetPath::iterator itEnd = shortest.end();
isNewPathFound = false;
bool isNewCandidateFound = false;
Link * linkToRemove = NULL;
for ( ; it != itEnd; ++it )
{
if ( (*it)->isLink() && links.find(static_cast<Link*>(*it)) != links.end() )
{
// removing link and trying dejkstra
links.erase(links.find(static_cast<Link*>(*it)));
NetPath newPath = searchPathDejkstra(virtualLink, network, K_SHORTEST_PATHS);
if ( newPath.size() == shortest.size() )
{
isNewPathFound = true;
pathStorage->push_back(newPath);
++pathsFound;
long weight = calculateKShortestPathWeight(newPath);
if ( weight > pathWeight )
{
isNewCandidateFound = true;
linkToRemove = static_cast<Link*>(*it);
pathWeight = weight;
candidate = newPath;
if ( pathsFound == Criteria::kShortestPathDepth() )
break;
}
if ( linkToRemove == NULL )
linkToRemove = static_cast<Link*>(*it); // to avoid the situation with removing NULL-link
}
links.insert(static_cast<Link*>(*it)); // inserting link again
}
}
if ( isNewCandidateFound )
shortest = candidate;
if ( isNewPathFound && pathsFound != Criteria::kShortestPathDepth() )
{
// restoring the capacity of link being removed
linkToRemove->RemoveAssignment(virtualLink);
links.erase(links.find(linkToRemove));
removedLinks.insert(linkToRemove);
}
}
// restoring removed capacities
restoreCapacities(virtualLink, network, &removedLinks, &removedSwitches);
return shortest;
}
inline void storeLinkImpl(Links<A> & cont) const
{
cont.add(links_);
}
inline void storeLinkImpl(Links<A> & cont) const
{
cont.push_back(ConditionJumpLink<A, true>(pred_, r_.getContentsSize(), r_.getLinksSize() + 1));
r_.storeLink(cont);
cont.push_back(JumpLink(-r_.getContentsSize(), -r_.getLinksSize() - 2));
}
void Blaster::gotDoc4 ( void *state, TcpSocket *s){
StateBD *st=(StateBD *)state;
st->m_numUrlDocsReceived++;
if (!s) {
//Shouldn't happen, but still putting a checkpoint
log (LOG_WARN,"blaster: Got a null s in gotDoc4."
"Happened because ip could not be found for gigablast"
"server");
if (st->m_numUrlDocsReceived==st->m_numUrlDocsSent){
m_launched--;
// Free stateBD
freeStateBD(st);
}
return;
}
// bail if got cut off
if ( s->m_readOffset == 0 ) {
log("blasterDiff : lost the Request in gotDoc4");
if (st->m_numUrlDocsReceived==st->m_numUrlDocsSent){
m_launched--;
freeStateBD(st);
}
return;
}
char *reply = s->m_readBuf ;
long size = s->m_readOffset;
HttpMime mime;
mime.set ( reply , size , NULL );
char *content = reply + mime.getMimeLen();
long contentLen = size - mime.getMimeLen();
//short csEnum = get_iana_charset(mime.getCharset(),
// mime.getCharsetLen());
/* if (csEnum == csUnknown)
log(LOG_DEBUG, "blaster: Unknown charset : %s", mime.getCharset());*/
Xml xml;
if (!xml.set(
content,
contentLen,
false,
0,
false,
TITLEREC_CURRENT_VERSION)){
log(LOG_WARN,"blaster: Couldn't set XML Class in gotDoc4");
}
Links links;
Url *url=mime.getLocationUrl();
if (!links.set(0,//siterec xml
&xml,
url,
false,
NULL,
TITLEREC_CURRENT_VERSION,
0,
false,
NULL)){
log(LOG_WARN, "blaster: Coudn't set Links class in gotDoc4");
}
for (long i=0;i<links.getNumLinks();i++){
char *ss=links.getLink(i);
char *p;
// This page *should* always be a gigablast page. So not adding
// checks for msn or yahoo or google page.
p=strstr(ss,"google.");
if(p) continue;
p=strstr(ss,"cache:"); //googles cache page
if(p) continue;
p= strstr(ss,"gigablast.");
if(p) continue;
p= strstr(ss,"web.archive.org");//older copies on gigablast
if(p) continue;
p= strstr(ss,"search.yahoo.com");//from gigablast search
if(p) continue;
p= strstr(ss,"search.msn.com");//from gigablast search
if(p) continue;
p= strstr(ss,"s.teoma.com");//from gigablast search
if(p) continue;
p= strstr(ss,"search.dmoz.org");//from gigablast search
if(p) continue;
p= strstr(ss,"www.answers.com");//from gigablast search
if(p) continue;
if (m_verbose)
log(LOG_WARN,"blaster: Link Present on server2=%s",ss);
}
// So if one of the links that is returned is the exact url,
// then we know that the url is present.So get the url from the
// mime, search for it in the links that are returned.
char tmp[1024];
char *sendBuf=s->m_sendBuf;
char *p1,*p2;
// First get the Host, which is the domain. Since socket s is going to
// be useless after this function, changing m_sendBuf instead of using
// more space
p1=strstr(sendBuf,"%3A");
if(p1){
p1+=3;
p2=strstr(p1," HTTP");
if (p2){
//Since I do not care about the sendbuf anymore
*p2='\0';
}
}
if (!p1 || !p2){
log(LOG_WARN,"blasterdiff: Could not find search link"
"from m_sendBuf in gotdoc4");
}
else{
sprintf(tmp,"%s",p1);
//log(LOG_WARN,"blaster: tmp in gotDoc4 = %s",tmp);
bool isFound=false;
// So now we search for tmp in the links
for (long i=0;i<links.getNumLinks();i++){
if(strstr(links.getLink(i),tmp) &&
links.getLinkLen(i)==(int)gbstrlen(tmp)){
isFound=true;
log(LOG_WARN,"blaster: %s in results1 but not"
" in results2 for query %s but does exist"
" in server2",tmp,st->m_u1);//->getQuery()
}
}
if (!isFound)
log(LOG_WARN,"blaster: %s in results1 but not"
" in results2 for query %s and does NOT exist"
" in server2",tmp,st->m_u1); // ->getQuery()
}
if (st->m_numUrlDocsReceived==st->m_numUrlDocsSent){
m_launched--;
// Free stateBD
freeStateBD(st);
}
return;
}
int main(int argc, char **argv) {
if (argc < 3) {
print_usage(argv[0]);
return 1;
}
if (strcmp(argv[1], "--h") == 0 || strcmp(argv[1], "--help") == 0 ) {
print_usage(argv[0]);
return 1;
}
g_log.m_disabled = true;
// initialize library
g_mem.init();
hashinit();
// current dir
char path[PATH_MAX];
realpath(argv[1], path);
size_t pathLen = strlen(path);
if (path[pathLen] != '/') {
strcat(path, "/");
}
g_hostdb.init(-1, NULL, false, false, path);
g_conf.init(path);
ucInit();
// initialize rdbs
g_loop.init();
g_collectiondb.loadAllCollRecs();
g_statsdb.init();
g_posdb.init();
g_titledb.init();
g_tagdb.init();
g_spiderdb.init();
g_doledb.init();
g_spiderCache.init();
g_clusterdb.init();
g_linkdb.init();
g_collectiondb.addRdbBaseToAllRdbsForEachCollRec();
g_log.m_disabled = false;
g_log.m_logPrefix = false;
uint64_t docId = strtoul(argv[2], NULL, 10);
logf(LOG_TRACE, "Getting titlerec for docId=%" PRIu64, docId);
Msg5 msg5;
RdbList list;
key96_t startKey = Titledb::makeFirstKey(docId);
key96_t endKey = Titledb::makeLastKey(docId);
msg5.getList(RDB_TITLEDB, 0, &list, startKey, endKey, 500000000, true, 0, 0, -1, NULL, NULL, 0, true, NULL, 0, -1, -1LL, false, true);
if (list.getNumRecs() != 1) {
logf(LOG_TRACE, "Unable to find titlerec for docId=%" PRIu64, docId);
cleanup();
exit(1);
}
XmlDoc xmlDoc;
if (!xmlDoc.set2(list.getCurrentRec(), list.getCurrentRecSize(), "main", NULL, 0)) {
logf(LOG_TRACE, "Unable to set XmlDoc for docId=%" PRIu64, docId);
cleanup();
exit(1);
}
logf(LOG_TRACE, "XmlDoc info");
logf(LOG_TRACE, "\tfirstUrl : %.*s", xmlDoc.size_firstUrl, xmlDoc.ptr_firstUrl);
logf(LOG_TRACE, "\tredirUrl : %.*s", xmlDoc.size_redirUrl, xmlDoc.ptr_redirUrl);
logf(LOG_TRACE, "\trootTitle : %.*s", xmlDoc.size_rootTitleBuf, xmlDoc.ptr_rootTitleBuf);
// logf(LOG_TRACE, "\timageData :");
logf(LOG_TRACE, "\t");
loghex(LOG_TRACE, xmlDoc.ptr_utf8Content, xmlDoc.size_utf8Content, "\tutf8Content:");
logf(LOG_TRACE, "\tsite : %.*s", xmlDoc.size_site, xmlDoc.ptr_site);
logf(LOG_TRACE, "\tlinkInfo");
LinkInfo* linkInfo = xmlDoc.getLinkInfo1();
logf(LOG_TRACE, "\t\tm_numGoodInlinks : %d", linkInfo->m_numGoodInlinks);
logf(LOG_TRACE, "\t\tm_numInlinksInternal : %d", linkInfo->m_numInlinksInternal);
logf(LOG_TRACE, "\t\tm_numStoredInlinks : %d", linkInfo->m_numStoredInlinks);
int i = 0;
for (Inlink *inlink = linkInfo->getNextInlink(NULL); inlink; inlink = linkInfo->getNextInlink(inlink)) {
logf(LOG_TRACE, "\t\tinlink #%d", i++);
logf(LOG_TRACE, "\t\t\tdocId : %" PRIu64, inlink->m_docId);
logf(LOG_TRACE, "\t\t\turl : %s", inlink->getUrl());
logf(LOG_TRACE, "\t\t\tlinktext : %s", inlink->getLinkText());
logf(LOG_TRACE, "\t\t\tcountry : %s", getCountryCode(inlink->m_country));
logf(LOG_TRACE, "\t\t\tlanguage : %s", getLanguageAbbr(inlink->m_language));
}
loghex(LOG_TRACE, xmlDoc.ptr_linkdbData, xmlDoc.size_linkdbData, "\tlinkdbData");
logf(LOG_TRACE, "\ttagRec");
TagRec *tagRec = xmlDoc.getTagRec();
for (Tag *tag = tagRec->getFirstTag(); tag; tag = tagRec->getNextTag(tag)) {
SafeBuf sb;
tag->printDataToBuf(&sb);
logf(LOG_TRACE, "\t\t%-12s: %s", getTagStrFromType(tag->m_type), sb.getBufStart());
}
logf(LOG_TRACE, "\t");
logf(LOG_TRACE, "Links info");
g_log.m_disabled = true;
Links *links = xmlDoc.getLinks();
g_log.m_disabled = false;
for (int i = 0; i < links->getNumLinks(); ++i) {
logf(LOG_TRACE, "\tlink : %.*s", links->getLinkLen(i), links->getLinkPtr(i));
}
cleanup();
return 0;
}
