bool FeSettings::thegamesdb_scraper( FeImporterContext &c )
{
#ifndef NO_NET
const char *HOSTNAME = "http://thegamesdb.net";
const char *PLATFORM_REQ = "api/GetPlatformsList.php";
const char *GAME_REQ = "api/GetGame.php?name=$1";
//
// Get a list of valid platforms
//
FeNetQueue q;
q.add_buffer_task( HOSTNAME, PLATFORM_REQ, 0 );
sf::Http::Response::Status status;
q.do_next_task( status );
if ( status != sf::Http::Response::Ok )
{
get_resource( "Error getting platform list from thegamesdb.net. Code: $1",
as_str( status ), c.user_message );
std::cout << " * " << c.user_message << std::endl;
return true;
}
int temp;
std::string body;
q.pop_completed_task( temp, body );
FeGameDBPlatformParser gdbpp;
gdbpp.parse( body );
const std::vector<std::string> &sl_temp = c.emulator.get_systems();
std::vector<std::string> system_list;
for ( std::vector<std::string>::const_iterator itr = sl_temp.begin(); itr!=sl_temp.end(); ++itr )
{
if ( gdbpp.m_set.find( *itr ) != gdbpp.m_set.end() )
system_list.push_back( *itr );
else
std::cout << " * System identifier '" << (*itr) << "' not recognized by "
<< HOSTNAME << std::endl;
}
if ( system_list.size() < 1 )
{
// Correct if we can based on the configured info source,
// otherwise we error out
const std::string source = c.emulator.get_info( FeEmulatorInfo::Info_source );
if ( source.compare( "mame" ) == 0 )
system_list.push_back( "Arcade" );
else if ( source.compare( "steam" ) == 0 )
system_list.push_back( "PC" );
else
{
get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.",
c.user_message );
std::cout << " * " << c.user_message << std::endl;
return true;
}
}
std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name );
//
// Build a map for looking up parents
//
std::map < std::string, FeRomInfo * > parent_map;
build_parent_map( parent_map, c.romlist );
//
// Build a worklist of the roms where we need to lookup
//
std::vector<FeRomInfo *> worklist;
worklist.reserve( c.romlist.size() );
for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr )
{
(*itr).set_info( FeRomInfo::Emulator, emu_name );
// Don't scrape for a clone if its parent has the same name
//
if ( has_same_name_as_parent( *itr, parent_map ) )
continue;
if ( !c.scrape_art || m_scrape_fanart
|| ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) )
|| ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) )
|| ( m_scrape_snaps && (!has_artwork( *itr, "snap" ) ) )
|| ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) )
worklist.push_back( &(*itr) );
}
const int NUM_ARTS=5; // the number of scrape-able artwork types
int done_count( 0 );
//
// Set up our initial queue of network tasks
//
for ( unsigned int i=0; i<worklist.size(); i++ )
{
std::string req_string = GAME_REQ;
std::string game = url_escape(
name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) );
perform_substitution( req_string, "$1", game );
//
// If we don't need to scrape a wheel artwork, then add the specific platform to our request
// If we are scraping a wheel, we want to be able to grab them where the game name (but
// not the system) matches, so we don't limit ourselves by system...
//
if (( system_list.size() == 1 )
&& ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) ))
{
req_string += "&platform=";
req_string += url_escape( system_list.front() );
}
q.add_buffer_task( HOSTNAME, req_string, i );
}
std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR;
base_path += emu_name + "/";
//
// Create worker threads to process the queue, adding new tasks to download
// artwork files where appropriate
//
FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q );
//
// Process the output queue from our worker threads
//
while ( !q.input_done() || !q.output_done() )
{
int id;
std::string result;
if ( q.pop_completed_task( id, result ) )
{
if ( id < 0 )
{
if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask ))
{
std::cout << " + Downloaded: " << result << std::endl;
c.download_count++;
}
if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask
done_count++;
}
else
{
FeGameDBArt my_art;
FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) );
gdbp.parse( result );
if ( c.scrape_art && !my_art.base.empty() )
{
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
size_t pos=0;
for ( int i=0; i<3 && ( pos != std::string::npos ); i++ )
pos = my_art.base.find_first_of( '/', pos+1 );
if (( pos != std::string::npos ) && ( pos < my_art.base.size()-1 ) )
{
hostn = my_art.base.substr( 0, pos+1 );
base_req = my_art.base.substr( pos+1 );
}
FeRomInfo &rom = *(worklist[id]);
if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) )
{
const char *FLYER = "flyer/";
std::string fname = base_path + FLYER + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer, fname );
}
else
done_count++;
if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) )
{
const char *WHEEL = "wheel/";
std::string fname = base_path + WHEEL + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel, fname );
}
else
done_count++;
if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) )
{
const char *MARQUEE = "marquee/";
std::string fname = base_path + MARQUEE + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee, fname );
}
else
done_count++;
if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_artwork( rom, "snap" )) )
{
const char *SNAP = "snap/";
std::string fname = base_path + SNAP + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, SNAP );
q.add_file_task( hostn, base_req + my_art.snap, fname );
}
else
done_count++;
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
const char *FANART = "fanart/";
std::string fname_base = base_path + FANART + rom.get_info( FeRomInfo::Romname ) + "/";
confirm_directory( base_path, "" );
confirm_directory( base_path + FANART, rom.get_info( FeRomInfo::Romname ) );
bool done_first=false; // we only count the first fanart against our percentage completed...
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ),
done_first );
done_first=true;
}
}
}
}
else
done_count++;
}
else
done_count+=NUM_ARTS;
}
if ( c.uiupdate )
{
int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() );
if ( c.uiupdate( c.uiupdatedata, p ) == false )
return false;
}
}
else if ( q.output_done() )
{
sf::Http::Response::Status status;
q.do_next_task( status );
}
else
sf::sleep( sf::milliseconds( 10 ) );
}
#endif
return true;
}
bool FeSettings::thegamesdb_scraper( FeImporterContext &c )
{
#ifndef NO_NET
const char *HOSTNAME = "http://thegamesdb.net";
const char *PLATFORM_LIST_REQ = "api/GetPlatformsList.php";
const char *PLAT_REQ = "api/GetPlatform.php?id=$1";
const char *GAME_REQ = "api/GetGame.php?name=$1";
const char *FLYER = "flyer/";
const char *WHEEL = "wheel/";
const char *MARQUEE = "marquee/";
const char *SNAP = "snap/";
const char *FANART = "fanart/";
//
// Get a list of valid platforms
//
FeNetQueue q;
q.add_buffer_task( HOSTNAME, PLATFORM_LIST_REQ, 0 );
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
get_resource( "Error getting platform list from thegamesdb.net. Code: $1",
as_str( status ), c.user_message );
std::cerr << " ! " << c.user_message << " (" << err_req << ")" << std::endl;
return true;
}
int temp;
std::string body;
q.pop_completed_task( temp, body );
FeGameDBPlatformListParser gdbplp;
gdbplp.parse( body );
std::vector<std::string> system_list;
std::vector<int> system_ids;
const std::vector<std::string> &sl_temp = c.emulator.get_systems();
for ( std::vector<std::string>::const_iterator itr = sl_temp.begin();
itr != sl_temp.end(); ++itr )
{
std::string comp_fuzz = get_fuzzy( *itr );
for ( size_t i=0; i<gdbplp.m_names.size(); i++ )
{
ASSERT( gdbplp.m_names.size() == gdbplp.m_ids.size() );
std::string &n = gdbplp.m_names[i];
int id = ( i < gdbplp.m_ids.size() ) ? i : 0;
if ( comp_fuzz.compare( get_fuzzy( n ) ) == 0 )
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
else
{
size_t pos = n.find_first_of( "(" );
if (( pos != std::string::npos ) &&
(( comp_fuzz.compare( get_fuzzy( n.substr(0,pos-1))) == 0 )
|| ( comp_fuzz.compare(get_fuzzy( n.substr(pos+1,n.size()-pos-1 ))) == 0 )))
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
}
}
}
if ( system_list.size() < 1 )
{
// Correct if we can based on the configured info source,
// otherwise we error out
switch( c.emulator.get_info_source() )
{
case FeEmulatorInfo::Listxml:
system_list.push_back( "Arcade" ); break;
case FeEmulatorInfo::Steam:
system_list.push_back( "PC" ); break;
default:
get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.",
c.user_message );
std::cerr << " ! " << c.user_message << std::endl;
return true;
}
}
std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name );
std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR;
base_path += emu_name + "/";
if ( c.scrape_art )
{
//
// Get emulator-specific images
//
for ( std::vector<int>::iterator iti=system_ids.begin();
iti != system_ids.end(); ++iti )
{
std::string plat_string = PLAT_REQ;
perform_substitution( plat_string, "$1", as_str( *iti ) );
q.add_buffer_task( HOSTNAME, plat_string, 0 );
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Unable to get platform information. Status code: "
<< status << " (" << err_req << ")" << std::endl;
continue;
}
body.clear();
q.pop_completed_task( temp, body );
FeGameDBArt my_art;
FeGameDBPlatformParser gdbpp( my_art );
gdbpp.parse( body );
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
std::string path = base_path + FLYER;
if ( m_scrape_flyers && ( !my_art.flyer.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer,
path + emu_name );
}
path = base_path + WHEEL;
if ( m_scrape_wheels && ( !my_art.wheel.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel,
path + emu_name );
}
path = base_path + MARQUEE;
if ( m_scrape_marquees && ( !my_art.marquee.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee,
path + emu_name );
}
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string path_base = base_path + FANART + emu_name + "/";
confirm_directory( base_path, "" );
confirm_directory( base_path + FANART, emu_name );
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( path_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
path_base + (*itr).substr( start_pos+1,
end_pos-start_pos-1 ) );
}
}
}
}
}
}
bool prefer_alt_filename = c.emulator.is_mess();
//
// Build a map for looking up parents
//
ParentMapType parent_map;
build_parent_map( parent_map, c.romlist, prefer_alt_filename );
//
// Build a worklist of the roms where we need to lookup
//
std::vector<FeRomInfo *> worklist;
worklist.reserve( c.romlist.size() );
for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr )
{
(*itr).set_info( FeRomInfo::Emulator, emu_name );
// Don't scrape for a clone if its parent has the same name
//
if ( has_same_name_as_parent( *itr, parent_map ) )
continue;
if ( !c.scrape_art || m_scrape_fanart
|| ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) )
|| ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) )
|| ( m_scrape_snaps && (!has_image_artwork( *itr, "snap" ) ) )
|| ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) )
worklist.push_back( &(*itr) );
}
const int NUM_ARTS=5; // the number of scrape-able artwork types
int done_count( 0 );
//
// Set up our initial queue of network tasks
//
for ( unsigned int i=0; i<worklist.size(); i++ )
{
std::string req_string = GAME_REQ;
std::string game = url_escape(
name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) );
perform_substitution( req_string, "$1", game );
//
// If we don't need to scrape a wheel artwork, then add the specific platform to our request
// If we are scraping a wheel, we want to be able to grab them where the game name (but
// not the system) matches, so we don't limit ourselves by system...
//
if (( system_list.size() == 1 )
&& ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) ))
{
req_string += "&platform=";
req_string += url_escape( system_list.front() );
}
q.add_buffer_task( HOSTNAME, req_string, i );
}
//
// Create worker threads to process the queue, adding new tasks to download
// artwork files where appropriate
//
FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q );
std::string aux;
//
// Process the output queue from our worker threads
//
while ( !( q.input_done() && q.output_done() ) )
{
int id;
std::string result;
if ( q.pop_completed_task( id, result ) )
{
if ( id < 0 )
{
if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask ))
{
std::cout << " + Downloaded: " << result << std::endl;
c.download_count++;
// find second last forward slash in filename
// we assume that there will always be at least two
size_t pos = result.find_last_of( "\\/" );
if ( pos != std::string::npos )
{
pos = result.find_last_of( "\\/", pos-1 );
if ( pos != std::string::npos )
aux = result.substr( pos );
}
}
if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask
done_count++;
}
else
{
FeGameDBArt my_art;
FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) );
gdbp.parse( result );
if ( c.scrape_art && !my_art.base.empty() )
{
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
const FeRomInfo &rom = *(worklist[id]);
if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) )
{
std::string fname = base_path + FLYER;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer, fname );
}
else
done_count++;
if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) )
{
std::string fname = base_path + WHEEL;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel, fname );
}
else
done_count++;
if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) )
{
std::string fname = base_path + MARQUEE;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee, fname );
}
else
done_count++;
if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_image_artwork( rom, "snap" )) )
{
std::string fname = base_path + SNAP;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, SNAP );
q.add_file_task( hostn, base_req + my_art.snap, fname );
}
else
done_count++;
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string fname_base = base_path + FANART;
confirm_directory( base_path, "" );
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
{
fname_base += rom.get_info( FeRomInfo::AltRomname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::AltRomname ) );
}
else
{
fname_base += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::Romname ) );
}
fname_base += "/";
bool done_first=false; // we only count the first fanart against our percentage completed...
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ),
done_first );
done_first=true;
}
}
}
}
else
done_count++;
}
else
{
aux = (worklist[id])->get_info( FeRomInfo::Title );
done_count+=NUM_ARTS;
}
}
if (( c.uiupdate ) && !worklist.empty() )
{
int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() );
if ( c.uiupdate( c.uiupdatedata, p, aux ) == false )
return false;
}
}
else if ( q.output_done() )
{
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Error processing request. Status code: "
<< status << " (" << err_req << ")" << std::endl;
}
}
else
sf::sleep( sf::milliseconds( 10 ) );
}
#endif
return true;
}