static HRESULT WINAPI HTMLLocation_get_pathname(IHTMLLocation *iface, BSTR *p) { HTMLLocation *This = impl_from_IHTMLLocation(iface); URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)}; HRESULT hres; TRACE("(%p)->(%p)\n", This, p); if(!p) return E_POINTER; url.dwUrlPathLength = 1; url.dwExtraInfoLength = 1; hres = get_url_components(This, &url); if(FAILED(hres)) return hres; if(url.dwUrlPathLength && url.lpszUrlPath[0] == '/') *p = SysAllocStringLen(url.lpszUrlPath + 1, url.dwUrlPathLength - 1); else *p = SysAllocStringLen(url.lpszUrlPath, url.dwUrlPathLength); if(!*p) return E_OUTOFMEMORY; return S_OK; }
static HRESULT WINAPI HTMLLocation_get_hash(IHTMLLocation *iface, BSTR *p) { HTMLLocation *This = impl_from_IHTMLLocation(iface); URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)}; const WCHAR hash[] = {'#',0}; DWORD hash_pos = 0; HRESULT hres; TRACE("(%p)->(%p)\n", This, p); if(!p) return E_POINTER; url.dwExtraInfoLength = 1; hres = get_url_components(This, &url); if(FAILED(hres)) return hres; if(!url.dwExtraInfoLength){ *p = NULL; return S_OK; } hash_pos = strcspnW(url.lpszExtraInfo, hash); url.dwExtraInfoLength -= hash_pos; *p = SysAllocStringLen(url.lpszExtraInfo + hash_pos, url.dwExtraInfoLength); if(!*p) return E_OUTOFMEMORY; return S_OK; }
static HRESULT WINAPI HTMLLocation_get_port(IHTMLLocation *iface, BSTR *p) { HTMLLocation *This = impl_from_IHTMLLocation(iface); URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)}; HRESULT hres; TRACE("(%p)->(%p)\n", This, p); if(!p) return E_POINTER; hres = get_url_components(This, &url); if(FAILED(hres)) return hres; if(url.nPort) { const WCHAR format[] = {'%','u',0}; WCHAR buf[6]; snprintfW(buf, 6, format, url.nPort); *p = SysAllocString(buf); }else { const WCHAR empty[] = {0}; *p = SysAllocString(empty); } if(!*p) return E_OUTOFMEMORY; return S_OK; }
static HRESULT WINAPI HTMLLocation_get_protocol(IHTMLLocation *iface, BSTR *p) { HTMLLocation *This = impl_from_IHTMLLocation(iface); URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)}; HRESULT hres; TRACE("(%p)->(%p)\n", This, p); if(!p) return E_POINTER; url.dwSchemeLength = 1; hres = get_url_components(This, &url); if(FAILED(hres)) return hres; if(!url.dwSchemeLength) { FIXME("Unexpected blank protocol\n"); return E_NOTIMPL; }else { WCHAR *buf; buf = *p = SysAllocStringLen(NULL, url.dwSchemeLength + 1); memcpy(buf, url.lpszScheme, url.dwSchemeLength * sizeof(WCHAR)); buf[url.dwSchemeLength] = ':'; } if(!*p) return E_OUTOFMEMORY; return S_OK; }
static HRESULT WINAPI HTMLLocation_get_hostname(IHTMLLocation *iface, BSTR *p) { HTMLLocation *This = impl_from_IHTMLLocation(iface); URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)}; HRESULT hres; TRACE("(%p)->(%p)\n", This, p); if(!p) return E_POINTER; url.dwHostNameLength = 1; hres = get_url_components(This, &url); if(FAILED(hres)) return hres; if(!url.dwHostNameLength){ *p = NULL; return S_OK; } *p = SysAllocStringLen(url.lpszHostName, url.dwHostNameLength); if(!*p) return E_OUTOFMEMORY; return S_OK; }
static HRESULT WINAPI HTMLLocation_get_host(IHTMLLocation *iface, BSTR *p) { HTMLLocation *This = impl_from_IHTMLLocation(iface); URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)}; HRESULT hres; TRACE("(%p)->(%p)\n", This, p); if(!p) return E_POINTER; url.dwHostNameLength = 1; hres = get_url_components(This, &url); if(FAILED(hres)) return hres; if(!url.dwHostNameLength){ *p = NULL; return S_OK; } if(url.nPort) { /* <hostname>:<port> */ const WCHAR format[] = {'%','u',0}; DWORD len = url.dwHostNameLength + 1 + 5; WCHAR *buf; buf = *p = SysAllocStringLen(NULL, len); memcpy(buf, url.lpszHostName, url.dwHostNameLength * sizeof(WCHAR)); buf[url.dwHostNameLength] = ':'; snprintfW(buf + url.dwHostNameLength + 1, 6, format, url.nPort); }else *p = SysAllocStringLen(url.lpszHostName, url.dwHostNameLength); if(!*p) return E_OUTOFMEMORY; return S_OK; }
static HRESULT WINAPI HTMLLocation_get_href(IHTMLLocation *iface, BSTR *p) { HTMLLocation *This = impl_from_IHTMLLocation(iface); URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)}; WCHAR *buf = NULL, *url_path = NULL; HRESULT hres, ret; DWORD len = 0; int i; TRACE("(%p)->(%p)\n", This, p); if(!p) return E_POINTER; url.dwSchemeLength = 1; url.dwHostNameLength = 1; url.dwUrlPathLength = 1; url.dwExtraInfoLength = 1; hres = get_url_components(This, &url); if(FAILED(hres)) return hres; switch(url.nScheme) { case INTERNET_SCHEME_FILE: { /* prepend a slash */ url_path = HeapAlloc(GetProcessHeap(), 0, (url.dwUrlPathLength + 1) * sizeof(WCHAR)); if(!url_path) return E_OUTOFMEMORY; url_path[0] = '/'; memcpy(url_path + 1, url.lpszUrlPath, url.dwUrlPathLength * sizeof(WCHAR)); url.lpszUrlPath = url_path; url.dwUrlPathLength = url.dwUrlPathLength + 1; } break; case INTERNET_SCHEME_HTTP: case INTERNET_SCHEME_HTTPS: case INTERNET_SCHEME_FTP: if(!url.dwUrlPathLength) { /* add a slash if it's blank */ url_path = url.lpszUrlPath = HeapAlloc(GetProcessHeap(), 0, 1 * sizeof(WCHAR)); if(!url.lpszUrlPath) return E_OUTOFMEMORY; url.lpszUrlPath[0] = '/'; url.dwUrlPathLength = 1; } break; default: break; } /* replace \ with / */ for(i = 0; i < url.dwUrlPathLength; ++i) if(url.lpszUrlPath[i] == '\\') url.lpszUrlPath[i] = '/'; if(InternetCreateUrlW(&url, ICU_ESCAPE, NULL, &len)) { FIXME("InternetCreateUrl succeeded with NULL buffer?\n"); ret = E_FAIL; goto cleanup; } if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) { FIXME("InternetCreateUrl failed with error: %08x\n", GetLastError()); SetLastError(0); ret = E_FAIL; goto cleanup; } SetLastError(0); buf = HeapAlloc(GetProcessHeap(), 0, len * sizeof(WCHAR)); if(!buf) { ret = E_OUTOFMEMORY; goto cleanup; } if(!InternetCreateUrlW(&url, ICU_ESCAPE, buf, &len)) { FIXME("InternetCreateUrl failed with error: %08x\n", GetLastError()); SetLastError(0); ret = E_FAIL; goto cleanup; } *p = SysAllocStringLen(buf, len); if(!*p) { ret = E_OUTOFMEMORY; goto cleanup; } ret = S_OK; cleanup: HeapFree(GetProcessHeap(), 0, buf); HeapFree(GetProcessHeap(), 0, url_path); return ret; }
bool FeSettings::thegamesdb_scraper( FeImporterContext &c ) { #ifndef NO_NET const char *HOSTNAME = "http://thegamesdb.net"; const char *PLATFORM_LIST_REQ = "api/GetPlatformsList.php"; const char *PLAT_REQ = "api/GetPlatform.php?id=$1"; const char *GAME_REQ = "api/GetGame.php?name=$1"; const char *FLYER = "flyer/"; const char *WHEEL = "wheel/"; const char *MARQUEE = "marquee/"; const char *SNAP = "snap/"; const char *FANART = "fanart/"; // // Get a list of valid platforms // FeNetQueue q; q.add_buffer_task( HOSTNAME, PLATFORM_LIST_REQ, 0 ); sf::Http::Response::Status status; std::string err_req; q.do_next_task( status, err_req ); if ( status != sf::Http::Response::Ok ) { get_resource( "Error getting platform list from thegamesdb.net. Code: $1", as_str( status ), c.user_message ); std::cerr << " ! " << c.user_message << " (" << err_req << ")" << std::endl; return true; } int temp; std::string body; q.pop_completed_task( temp, body ); FeGameDBPlatformListParser gdbplp; gdbplp.parse( body ); std::vector<std::string> system_list; std::vector<int> system_ids; const std::vector<std::string> &sl_temp = c.emulator.get_systems(); for ( std::vector<std::string>::const_iterator itr = sl_temp.begin(); itr != sl_temp.end(); ++itr ) { std::string comp_fuzz = get_fuzzy( *itr ); for ( size_t i=0; i<gdbplp.m_names.size(); i++ ) { ASSERT( gdbplp.m_names.size() == gdbplp.m_ids.size() ); std::string &n = gdbplp.m_names[i]; int id = ( i < gdbplp.m_ids.size() ) ? i : 0; if ( comp_fuzz.compare( get_fuzzy( n ) ) == 0 ) { system_list.push_back( n ); system_ids.push_back( id ); break; } else { size_t pos = n.find_first_of( "(" ); if (( pos != std::string::npos ) && (( comp_fuzz.compare( get_fuzzy( n.substr(0,pos-1))) == 0 ) || ( comp_fuzz.compare(get_fuzzy( n.substr(pos+1,n.size()-pos-1 ))) == 0 ))) { system_list.push_back( n ); system_ids.push_back( id ); break; } } } } if ( system_list.size() < 1 ) { // Correct if we can based on the configured info source, // otherwise we error out switch( c.emulator.get_info_source() ) { case FeEmulatorInfo::Listxml: system_list.push_back( "Arcade" ); break; case FeEmulatorInfo::Steam: system_list.push_back( "PC" ); break; default: get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.", c.user_message ); std::cerr << " ! " << c.user_message << std::endl; return true; } } std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name ); std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR; base_path += emu_name + "/"; if ( c.scrape_art ) { // // Get emulator-specific images // for ( std::vector<int>::iterator iti=system_ids.begin(); iti != system_ids.end(); ++iti ) { std::string plat_string = PLAT_REQ; perform_substitution( plat_string, "$1", as_str( *iti ) ); q.add_buffer_task( HOSTNAME, plat_string, 0 ); q.do_next_task( status, err_req ); if ( status != sf::Http::Response::Ok ) { std::cout << " * Unable to get platform information. Status code: " << status << " (" << err_req << ")" << std::endl; continue; } body.clear(); q.pop_completed_task( temp, body ); FeGameDBArt my_art; FeGameDBPlatformParser gdbpp( my_art ); gdbpp.parse( body ); std::string hostn = HOSTNAME; std::string base_req = "banners/"; get_url_components( my_art.base, hostn, base_req ); std::string path = base_path + FLYER; if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && ( !art_exists( path, emu_name ) )) { confirm_directory( base_path, FLYER ); q.add_file_task( hostn, base_req + my_art.flyer, path + emu_name ); } path = base_path + WHEEL; if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && ( !art_exists( path, emu_name ) )) { confirm_directory( base_path, WHEEL ); q.add_file_task( hostn, base_req + my_art.wheel, path + emu_name ); } path = base_path + MARQUEE; if ( m_scrape_marquees && ( !my_art.marquee.empty() ) && ( !art_exists( path, emu_name ) )) { confirm_directory( base_path, MARQUEE ); q.add_file_task( hostn, base_req + my_art.marquee, path + emu_name ); } if ( m_scrape_fanart && !my_art.fanart.empty() ) { std::string path_base = base_path + FANART + emu_name + "/"; confirm_directory( base_path, "" ); confirm_directory( base_path + FANART, emu_name ); for ( std::vector<std::string>::iterator itr = my_art.fanart.begin(); itr != my_art.fanart.end(); ++itr ) { size_t start_pos = (*itr).find_last_of( "/\\" ); size_t end_pos = (*itr).find_last_of( '.' ); if (( start_pos != std::string::npos ) && ( !file_exists( path_base + (*itr).substr( start_pos+1 ) ) )) { if (( end_pos != std::string::npos ) && ( end_pos > start_pos )) { q.add_file_task( hostn, base_req + (*itr), path_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ) ); } } } } } } bool prefer_alt_filename = c.emulator.is_mess(); // // Build a map for looking up parents // ParentMapType parent_map; build_parent_map( parent_map, c.romlist, prefer_alt_filename ); // // Build a worklist of the roms where we need to lookup // std::vector<FeRomInfo *> worklist; worklist.reserve( c.romlist.size() ); for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr ) { (*itr).set_info( FeRomInfo::Emulator, emu_name ); // Don't scrape for a clone if its parent has the same name // if ( has_same_name_as_parent( *itr, parent_map ) ) continue; if ( !c.scrape_art || m_scrape_fanart || ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) ) || ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) ) || ( m_scrape_snaps && (!has_image_artwork( *itr, "snap" ) ) ) || ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) ) worklist.push_back( &(*itr) ); } const int NUM_ARTS=5; // the number of scrape-able artwork types int done_count( 0 ); // // Set up our initial queue of network tasks // for ( unsigned int i=0; i<worklist.size(); i++ ) { std::string req_string = GAME_REQ; std::string game = url_escape( name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) ); perform_substitution( req_string, "$1", game ); // // If we don't need to scrape a wheel artwork, then add the specific platform to our request // If we are scraping a wheel, we want to be able to grab them where the game name (but // not the system) matches, so we don't limit ourselves by system... // if (( system_list.size() == 1 ) && ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) )) { req_string += "&platform="; req_string += url_escape( system_list.front() ); } q.add_buffer_task( HOSTNAME, req_string, i ); } // // Create worker threads to process the queue, adding new tasks to download // artwork files where appropriate // FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q ); std::string aux; // // Process the output queue from our worker threads // while ( !( q.input_done() && q.output_done() ) ) { int id; std::string result; if ( q.pop_completed_task( id, result ) ) { if ( id < 0 ) { if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask )) { std::cout << " + Downloaded: " << result << std::endl; c.download_count++; // find second last forward slash in filename // we assume that there will always be at least two size_t pos = result.find_last_of( "\\/" ); if ( pos != std::string::npos ) { pos = result.find_last_of( "\\/", pos-1 ); if ( pos != std::string::npos ) aux = result.substr( pos ); } } if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask done_count++; } else { FeGameDBArt my_art; FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) ); gdbp.parse( result ); if ( c.scrape_art && !my_art.base.empty() ) { std::string hostn = HOSTNAME; std::string base_req = "banners/"; get_url_components( my_art.base, hostn, base_req ); const FeRomInfo &rom = *(worklist[id]); if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) ) { std::string fname = base_path + FLYER; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, FLYER ); q.add_file_task( hostn, base_req + my_art.flyer, fname ); } else done_count++; if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) ) { std::string fname = base_path + WHEEL; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, WHEEL ); q.add_file_task( hostn, base_req + my_art.wheel, fname ); } else done_count++; if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) ) { std::string fname = base_path + MARQUEE; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, MARQUEE ); q.add_file_task( hostn, base_req + my_art.marquee, fname ); } else done_count++; if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_image_artwork( rom, "snap" )) ) { std::string fname = base_path + SNAP; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, SNAP ); q.add_file_task( hostn, base_req + my_art.snap, fname ); } else done_count++; if ( m_scrape_fanart && !my_art.fanart.empty() ) { std::string fname_base = base_path + FANART; confirm_directory( base_path, "" ); const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) { fname_base += rom.get_info( FeRomInfo::AltRomname ); confirm_directory( base_path + FANART, rom.get_info( FeRomInfo::AltRomname ) ); } else { fname_base += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path + FANART, rom.get_info( FeRomInfo::Romname ) ); } fname_base += "/"; bool done_first=false; // we only count the first fanart against our percentage completed... for ( std::vector<std::string>::iterator itr = my_art.fanart.begin(); itr != my_art.fanart.end(); ++itr ) { size_t start_pos = (*itr).find_last_of( "/\\" ); size_t end_pos = (*itr).find_last_of( '.' ); if (( start_pos != std::string::npos ) && ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) )) { if (( end_pos != std::string::npos ) && ( end_pos > start_pos )) { q.add_file_task( hostn, base_req + (*itr), fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ), done_first ); done_first=true; } } } } else done_count++; } else { aux = (worklist[id])->get_info( FeRomInfo::Title ); done_count+=NUM_ARTS; } } if (( c.uiupdate ) && !worklist.empty() ) { int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() ); if ( c.uiupdate( c.uiupdatedata, p, aux ) == false ) return false; } } else if ( q.output_done() ) { sf::Http::Response::Status status; std::string err_req; q.do_next_task( status, err_req ); if ( status != sf::Http::Response::Ok ) { std::cout << " * Error processing request. Status code: " << status << " (" << err_req << ")" << std::endl; } } else sf::sleep( sf::milliseconds( 10 ) ); } #endif return true; }