boost::filesystem::path ApplicationPathProvider::getResourcePath(const std::string& resource) const { std::vector<boost::filesystem::path> resourcePaths = getResourceDirs(); foreach(const boost::filesystem::path& resourcePath, resourcePaths) { boost::filesystem::path r(resourcePath / resource); if (boost::filesystem::exists(r)) { return r; } }
void HttpManager::run() { unsigned int i; deque<pair<string,int> > linksQueue; //queue that is used for BFS exploring // We create another instance of UrlLocalResourceKey UrlLocalResourceKey * keygen = new UrlLocalResourceKey(); linksQueue.push_back(make_pair(link,1)); while(!linksQueue.empty()) { string url=linksQueue[0].first; int currentDeep=linksQueue[0].second; linksQueue.pop_front(); cout<<"Deep of search:"<<currentDeep<<" trying to get "<<url<<endl; WebResourceInfo webResource=downloader->getWebResourceInfo(url); //call all the validators and if not valid skip this resource bool valid=true; for(i=0;i<validators.size();i++)valid=valid && validators[i]->isValid(webResource); if(!valid) { cout<<"But validators say not to download!"<<endl; continue; } // oldbam LocalResourceInfo storedLocalResource = registry->getRecord(url); if (storedLocalResource.getUrl() == webResource.getUrl()) { cout<<"INFO:"<<"Resource exists in the registry"<<endl; cout<<"URL:"<<storedLocalResource.getUrl()<<endl; cout<<"DATE:"<<storedLocalResource.getDate()<<endl; cout<<"LOCALPATH:"<<storedLocalResource.getLocalPath()<<endl; continue; } // end oldbam string hostName=getHostName(webResource); //virtual path of the resource on the server string resourceHostPath=getResourceDirs(webResource); //change last '/' to '\' if(resourceHostPath!="") resourceHostPath[resourceHostPath.length()-1]='\\'; //local resource's path string localResourceDir=localDir+hostName+"\\"+resourceHostPath; //full resource's local file name(path + name) string fullLocalFileName=localResourceDir+getLocalFileName(webResource); downloader->makeDir(localResourceDir); bool result = downloader->download(webResource,localResourceDir); if (result == true) cout<<"Ok. Placed to: "<<fullLocalFileName<<endl; else cout<<"ERROR : Downloader returned 'false' after download."<<endl; //create and register LocalResource char timeChar [19]; _strdate( timeChar ); timeChar[8]='#'; _strtime( timeChar+9 ); LocalResourceInfo localResource(webResource.getUrl(),(string)timeChar,fullLocalFileName); registry->addRecord(keygen->getKey(localResource),localResource); //parse and add new links to the queue if(webResource.getMimeValue("Content-Type").find("text/html")!=string::npos && currentDeep<maxDeep) { string content,tmp; ifstream file(fullLocalFileName.c_str()); while (getline(file, tmp)) { content += tmp; } content = encodeUrl(content, ' '); vector<string> links=parser->getLinks(content); //deleting duplicates sort(links.begin(),links.end()); links.erase(unique(links.begin(),links.end()),links.end()); parser->relative2Absolute(links,getHostName(webResource)+"/"+getResourceDirs(webResource)); // modified by oldbam, add cout for links cout<<"INFO : New links are found by manager:"<<endl; for(i=0;i<links.size();i++) { linksQueue.push_back(make_pair(links[i],currentDeep+1)); cout<<"LINK : "<<links[i]<<endl<<"DEPTH : "<<currentDeep+1<<endl; } cout<<endl; // end modified by oldbam } cout<<endl; } delete keygen; }