static HRESULT WINAPI HTMLLocation_get_pathname(IHTMLLocation *iface, BSTR *p)
{
HTMLLocation *This = impl_from_IHTMLLocation(iface);
URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)};
HRESULT hres;
TRACE("(%p)->(%p)\n", This, p);
if(!p)
return E_POINTER;
url.dwUrlPathLength = 1;
url.dwExtraInfoLength = 1;
hres = get_url_components(This, &url);
if(FAILED(hres))
return hres;
if(url.dwUrlPathLength && url.lpszUrlPath[0] == '/')
*p = SysAllocStringLen(url.lpszUrlPath + 1, url.dwUrlPathLength - 1);
else
*p = SysAllocStringLen(url.lpszUrlPath, url.dwUrlPathLength);
if(!*p)
return E_OUTOFMEMORY;
return S_OK;
}
static HRESULT WINAPI HTMLLocation_get_hash(IHTMLLocation *iface, BSTR *p)
{
HTMLLocation *This = impl_from_IHTMLLocation(iface);
URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)};
const WCHAR hash[] = {'#',0};
DWORD hash_pos = 0;
HRESULT hres;
TRACE("(%p)->(%p)\n", This, p);
if(!p)
return E_POINTER;
url.dwExtraInfoLength = 1;
hres = get_url_components(This, &url);
if(FAILED(hres))
return hres;
if(!url.dwExtraInfoLength){
*p = NULL;
return S_OK;
}
hash_pos = strcspnW(url.lpszExtraInfo, hash);
url.dwExtraInfoLength -= hash_pos;
*p = SysAllocStringLen(url.lpszExtraInfo + hash_pos, url.dwExtraInfoLength);
if(!*p)
return E_OUTOFMEMORY;
return S_OK;
}
static HRESULT WINAPI HTMLLocation_get_port(IHTMLLocation *iface, BSTR *p)
{
HTMLLocation *This = impl_from_IHTMLLocation(iface);
URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)};
HRESULT hres;
TRACE("(%p)->(%p)\n", This, p);
if(!p)
return E_POINTER;
hres = get_url_components(This, &url);
if(FAILED(hres))
return hres;
if(url.nPort) {
const WCHAR format[] = {'%','u',0};
WCHAR buf[6];
snprintfW(buf, 6, format, url.nPort);
*p = SysAllocString(buf);
}else {
const WCHAR empty[] = {0};
*p = SysAllocString(empty);
}
if(!*p)
return E_OUTOFMEMORY;
return S_OK;
}
static HRESULT WINAPI HTMLLocation_get_protocol(IHTMLLocation *iface, BSTR *p)
{
HTMLLocation *This = impl_from_IHTMLLocation(iface);
URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)};
HRESULT hres;
TRACE("(%p)->(%p)\n", This, p);
if(!p)
return E_POINTER;
url.dwSchemeLength = 1;
hres = get_url_components(This, &url);
if(FAILED(hres))
return hres;
if(!url.dwSchemeLength) {
FIXME("Unexpected blank protocol\n");
return E_NOTIMPL;
}else {
WCHAR *buf;
buf = *p = SysAllocStringLen(NULL, url.dwSchemeLength + 1);
memcpy(buf, url.lpszScheme, url.dwSchemeLength * sizeof(WCHAR));
buf[url.dwSchemeLength] = ':';
}
if(!*p)
return E_OUTOFMEMORY;
return S_OK;
}
static HRESULT WINAPI HTMLLocation_get_hostname(IHTMLLocation *iface, BSTR *p)
{
HTMLLocation *This = impl_from_IHTMLLocation(iface);
URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)};
HRESULT hres;
TRACE("(%p)->(%p)\n", This, p);
if(!p)
return E_POINTER;
url.dwHostNameLength = 1;
hres = get_url_components(This, &url);
if(FAILED(hres))
return hres;
if(!url.dwHostNameLength){
*p = NULL;
return S_OK;
}
*p = SysAllocStringLen(url.lpszHostName, url.dwHostNameLength);
if(!*p)
return E_OUTOFMEMORY;
return S_OK;
}
static HRESULT WINAPI HTMLLocation_get_host(IHTMLLocation *iface, BSTR *p)
{
HTMLLocation *This = impl_from_IHTMLLocation(iface);
URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)};
HRESULT hres;
TRACE("(%p)->(%p)\n", This, p);
if(!p)
return E_POINTER;
url.dwHostNameLength = 1;
hres = get_url_components(This, &url);
if(FAILED(hres))
return hres;
if(!url.dwHostNameLength){
*p = NULL;
return S_OK;
}
if(url.nPort) {
/* <hostname>:<port> */
const WCHAR format[] = {'%','u',0};
DWORD len = url.dwHostNameLength + 1 + 5;
WCHAR *buf;
buf = *p = SysAllocStringLen(NULL, len);
memcpy(buf, url.lpszHostName, url.dwHostNameLength * sizeof(WCHAR));
buf[url.dwHostNameLength] = ':';
snprintfW(buf + url.dwHostNameLength + 1, 6, format, url.nPort);
}else
*p = SysAllocStringLen(url.lpszHostName, url.dwHostNameLength);
if(!*p)
return E_OUTOFMEMORY;
return S_OK;
}
static HRESULT WINAPI HTMLLocation_get_href(IHTMLLocation *iface, BSTR *p)
{
HTMLLocation *This = impl_from_IHTMLLocation(iface);
URL_COMPONENTSW url = {sizeof(URL_COMPONENTSW)};
WCHAR *buf = NULL, *url_path = NULL;
HRESULT hres, ret;
DWORD len = 0;
int i;
TRACE("(%p)->(%p)\n", This, p);
if(!p)
return E_POINTER;
url.dwSchemeLength = 1;
url.dwHostNameLength = 1;
url.dwUrlPathLength = 1;
url.dwExtraInfoLength = 1;
hres = get_url_components(This, &url);
if(FAILED(hres))
return hres;
switch(url.nScheme) {
case INTERNET_SCHEME_FILE:
{
/* prepend a slash */
url_path = HeapAlloc(GetProcessHeap(), 0, (url.dwUrlPathLength + 1) * sizeof(WCHAR));
if(!url_path)
return E_OUTOFMEMORY;
url_path[0] = '/';
memcpy(url_path + 1, url.lpszUrlPath, url.dwUrlPathLength * sizeof(WCHAR));
url.lpszUrlPath = url_path;
url.dwUrlPathLength = url.dwUrlPathLength + 1;
}
break;
case INTERNET_SCHEME_HTTP:
case INTERNET_SCHEME_HTTPS:
case INTERNET_SCHEME_FTP:
if(!url.dwUrlPathLength) {
/* add a slash if it's blank */
url_path = url.lpszUrlPath = HeapAlloc(GetProcessHeap(), 0, 1 * sizeof(WCHAR));
if(!url.lpszUrlPath)
return E_OUTOFMEMORY;
url.lpszUrlPath[0] = '/';
url.dwUrlPathLength = 1;
}
break;
default:
break;
}
/* replace \ with / */
for(i = 0; i < url.dwUrlPathLength; ++i)
if(url.lpszUrlPath[i] == '\\')
url.lpszUrlPath[i] = '/';
if(InternetCreateUrlW(&url, ICU_ESCAPE, NULL, &len)) {
FIXME("InternetCreateUrl succeeded with NULL buffer?\n");
ret = E_FAIL;
goto cleanup;
}
if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
FIXME("InternetCreateUrl failed with error: %08x\n", GetLastError());
SetLastError(0);
ret = E_FAIL;
goto cleanup;
}
SetLastError(0);
buf = HeapAlloc(GetProcessHeap(), 0, len * sizeof(WCHAR));
if(!buf) {
ret = E_OUTOFMEMORY;
goto cleanup;
}
if(!InternetCreateUrlW(&url, ICU_ESCAPE, buf, &len)) {
FIXME("InternetCreateUrl failed with error: %08x\n", GetLastError());
SetLastError(0);
ret = E_FAIL;
goto cleanup;
}
*p = SysAllocStringLen(buf, len);
if(!*p) {
ret = E_OUTOFMEMORY;
goto cleanup;
}
ret = S_OK;
cleanup:
HeapFree(GetProcessHeap(), 0, buf);
HeapFree(GetProcessHeap(), 0, url_path);
return ret;
}
bool FeSettings::thegamesdb_scraper( FeImporterContext &c )
{
#ifndef NO_NET
const char *HOSTNAME = "http://thegamesdb.net";
const char *PLATFORM_LIST_REQ = "api/GetPlatformsList.php";
const char *PLAT_REQ = "api/GetPlatform.php?id=$1";
const char *GAME_REQ = "api/GetGame.php?name=$1";
const char *FLYER = "flyer/";
const char *WHEEL = "wheel/";
const char *MARQUEE = "marquee/";
const char *SNAP = "snap/";
const char *FANART = "fanart/";
//
// Get a list of valid platforms
//
FeNetQueue q;
q.add_buffer_task( HOSTNAME, PLATFORM_LIST_REQ, 0 );
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
get_resource( "Error getting platform list from thegamesdb.net. Code: $1",
as_str( status ), c.user_message );
std::cerr << " ! " << c.user_message << " (" << err_req << ")" << std::endl;
return true;
}
int temp;
std::string body;
q.pop_completed_task( temp, body );
FeGameDBPlatformListParser gdbplp;
gdbplp.parse( body );
std::vector<std::string> system_list;
std::vector<int> system_ids;
const std::vector<std::string> &sl_temp = c.emulator.get_systems();
for ( std::vector<std::string>::const_iterator itr = sl_temp.begin();
itr != sl_temp.end(); ++itr )
{
std::string comp_fuzz = get_fuzzy( *itr );
for ( size_t i=0; i<gdbplp.m_names.size(); i++ )
{
ASSERT( gdbplp.m_names.size() == gdbplp.m_ids.size() );
std::string &n = gdbplp.m_names[i];
int id = ( i < gdbplp.m_ids.size() ) ? i : 0;
if ( comp_fuzz.compare( get_fuzzy( n ) ) == 0 )
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
else
{
size_t pos = n.find_first_of( "(" );
if (( pos != std::string::npos ) &&
(( comp_fuzz.compare( get_fuzzy( n.substr(0,pos-1))) == 0 )
|| ( comp_fuzz.compare(get_fuzzy( n.substr(pos+1,n.size()-pos-1 ))) == 0 )))
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
}
}
}
if ( system_list.size() < 1 )
{
// Correct if we can based on the configured info source,
// otherwise we error out
switch( c.emulator.get_info_source() )
{
case FeEmulatorInfo::Listxml:
system_list.push_back( "Arcade" ); break;
case FeEmulatorInfo::Steam:
system_list.push_back( "PC" ); break;
default:
get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.",
c.user_message );
std::cerr << " ! " << c.user_message << std::endl;
return true;
}
}
std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name );
std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR;
base_path += emu_name + "/";
if ( c.scrape_art )
{
//
// Get emulator-specific images
//
for ( std::vector<int>::iterator iti=system_ids.begin();
iti != system_ids.end(); ++iti )
{
std::string plat_string = PLAT_REQ;
perform_substitution( plat_string, "$1", as_str( *iti ) );
q.add_buffer_task( HOSTNAME, plat_string, 0 );
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Unable to get platform information. Status code: "
<< status << " (" << err_req << ")" << std::endl;
continue;
}
body.clear();
q.pop_completed_task( temp, body );
FeGameDBArt my_art;
FeGameDBPlatformParser gdbpp( my_art );
gdbpp.parse( body );
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
std::string path = base_path + FLYER;
if ( m_scrape_flyers && ( !my_art.flyer.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer,
path + emu_name );
}
path = base_path + WHEEL;
if ( m_scrape_wheels && ( !my_art.wheel.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel,
path + emu_name );
}
path = base_path + MARQUEE;
if ( m_scrape_marquees && ( !my_art.marquee.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee,
path + emu_name );
}
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string path_base = base_path + FANART + emu_name + "/";
confirm_directory( base_path, "" );
confirm_directory( base_path + FANART, emu_name );
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( path_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
path_base + (*itr).substr( start_pos+1,
end_pos-start_pos-1 ) );
}
}
}
}
}
}
bool prefer_alt_filename = c.emulator.is_mess();
//
// Build a map for looking up parents
//
ParentMapType parent_map;
build_parent_map( parent_map, c.romlist, prefer_alt_filename );
//
// Build a worklist of the roms where we need to lookup
//
std::vector<FeRomInfo *> worklist;
worklist.reserve( c.romlist.size() );
for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr )
{
(*itr).set_info( FeRomInfo::Emulator, emu_name );
// Don't scrape for a clone if its parent has the same name
//
if ( has_same_name_as_parent( *itr, parent_map ) )
continue;
if ( !c.scrape_art || m_scrape_fanart
|| ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) )
|| ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) )
|| ( m_scrape_snaps && (!has_image_artwork( *itr, "snap" ) ) )
|| ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) )
worklist.push_back( &(*itr) );
}
const int NUM_ARTS=5; // the number of scrape-able artwork types
int done_count( 0 );
//
// Set up our initial queue of network tasks
//
for ( unsigned int i=0; i<worklist.size(); i++ )
{
std::string req_string = GAME_REQ;
std::string game = url_escape(
name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) );
perform_substitution( req_string, "$1", game );
//
// If we don't need to scrape a wheel artwork, then add the specific platform to our request
// If we are scraping a wheel, we want to be able to grab them where the game name (but
// not the system) matches, so we don't limit ourselves by system...
//
if (( system_list.size() == 1 )
&& ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) ))
{
req_string += "&platform=";
req_string += url_escape( system_list.front() );
}
q.add_buffer_task( HOSTNAME, req_string, i );
}
//
// Create worker threads to process the queue, adding new tasks to download
// artwork files where appropriate
//
FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q );
std::string aux;
//
// Process the output queue from our worker threads
//
while ( !( q.input_done() && q.output_done() ) )
{
int id;
std::string result;
if ( q.pop_completed_task( id, result ) )
{
if ( id < 0 )
{
if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask ))
{
std::cout << " + Downloaded: " << result << std::endl;
c.download_count++;
// find second last forward slash in filename
// we assume that there will always be at least two
size_t pos = result.find_last_of( "\\/" );
if ( pos != std::string::npos )
{
pos = result.find_last_of( "\\/", pos-1 );
if ( pos != std::string::npos )
aux = result.substr( pos );
}
}
if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask
done_count++;
}
else
{
FeGameDBArt my_art;
FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) );
gdbp.parse( result );
if ( c.scrape_art && !my_art.base.empty() )
{
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
const FeRomInfo &rom = *(worklist[id]);
if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) )
{
std::string fname = base_path + FLYER;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer, fname );
}
else
done_count++;
if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) )
{
std::string fname = base_path + WHEEL;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel, fname );
}
else
done_count++;
if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) )
{
std::string fname = base_path + MARQUEE;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee, fname );
}
else
done_count++;
if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_image_artwork( rom, "snap" )) )
{
std::string fname = base_path + SNAP;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, SNAP );
q.add_file_task( hostn, base_req + my_art.snap, fname );
}
else
done_count++;
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string fname_base = base_path + FANART;
confirm_directory( base_path, "" );
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
{
fname_base += rom.get_info( FeRomInfo::AltRomname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::AltRomname ) );
}
else
{
fname_base += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::Romname ) );
}
fname_base += "/";
bool done_first=false; // we only count the first fanart against our percentage completed...
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ),
done_first );
done_first=true;
}
}
}
}
else
done_count++;
}
else
{
aux = (worklist[id])->get_info( FeRomInfo::Title );
done_count+=NUM_ARTS;
}
}
if (( c.uiupdate ) && !worklist.empty() )
{
int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() );
if ( c.uiupdate( c.uiupdatedata, p, aux ) == false )
return false;
}
}
else if ( q.output_done() )
{
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Error processing request. Status code: "
<< status << " (" << err_req << ")" << std::endl;
}
}
else
sf::sleep( sf::milliseconds( 10 ) );
}
#endif
return true;
}
