void FeMessXMLParser::end_element( const char *element )
{
if ( strcmp( element, "software" ) == 0 )
{
if ( m_keep_rom )
{
set_info_values( *m_itr );
}
else
{
// otherwise try matching based on description
for ( m_itr=m_ctx.romlist.begin(); m_itr!=m_ctx.romlist.end(); ++m_itr )
{
if ( m_description.compare(
(*m_itr).get_info( FeRomInfo::Romname ) ) == 0 )
{
set_info_values( *m_itr );
}
else if ( (*m_itr).get_info( FeRomInfo::AltRomname ).empty() )
{
// Try using a "fuzzy" match (ignores brackets).
//
const std::string &temp = (*m_itr).get_info( FeRomInfo::BuildScratchPad );
if ( temp.compare( m_fuzzydesc ) == 0 )
{
set_info_values( *m_itr );
}
}
}
}
clear_parse_state();
}
else if ( m_element_open )
{
if ( strcmp( element, "description" ) == 0 )
{
m_description = m_current_data;
m_fuzzydesc = get_fuzzy( m_current_data );
}
else if ( strcmp( element, "year" ) == 0 )
m_year = m_current_data;
else if ( strcmp( element, "publisher" ) == 0 )
m_man = m_current_data;
m_current_data.clear();
m_element_open=false;
}
}
void FeGameDBParser::end_element( const char *element )
{
if ( strcmp( element, "Similar" ) == 0 )
{
m_ignore=false;
return;
}
if ( m_ignore )
return;
if ( m_element_open )
{
if ( strcmp( element, "GameTitle" ) == 0 )
m_work.set_info( FeRomInfo::Title, m_current_data );
else if ( strcmp( element, "title" ) == 0 )
{
if ( m_work.get_info( FeRomInfo::AltTitle ).empty() )
m_work.set_info( FeRomInfo::AltTitle, m_current_data );
}
else if ( strcmp( element, "ReleaseDate" ) == 0 )
{
size_t cut = m_current_data.find_last_of( "/" );
if ( cut != std::string::npos )
m_work.set_info( FeRomInfo::Year, m_current_data.substr( cut+1 ) );
else
m_work.set_info( FeRomInfo::Year, m_current_data );
}
else if ( strcmp( element, "genre" ) == 0 )
{
std::string cat = m_work.get_info( FeRomInfo::Category );
if ( cat.size() == 0 )
cat = m_current_data;
else
{
cat += " / ";
cat += m_current_data;
}
m_work.set_info( FeRomInfo::Category, cat );
}
else if ( strcmp( element, "Players" ) == 0 )
m_work.set_info( FeRomInfo::Players, m_current_data );
else if ( strcmp( element, "Publisher" ) == 0 )
m_work.set_info( FeRomInfo::Manufacturer, m_current_data );
else if ( strcmp( element, "Platform" ) == 0 )
m_work_platform = m_current_data;
else if ( m_art )
{
if ( strcmp( element, "baseImgUrl" ) == 0 )
m_art->base = m_current_data;
else if ( strcmp( element, "banner" ) == 0 )
m_work_art.marquee = m_current_data;
else if ( strcmp( element, "clearlogo" ) == 0 )
m_work_art.wheel = m_current_data;
else if ( strcmp( element, "original" ) == 0 )
{
if ( m_fanart )
{
m_work_art.fanart.push_back( m_current_data );
m_fanart = false;
}
else
{
if ( m_work_art.snap.empty() )
m_work_art.snap = m_current_data;
m_screenshot=false;
}
}
else if ( strcmp( element, "boxart" ) == 0 )
m_work_art.flyer = m_current_data;
}
m_element_open=false;
m_current_data.clear();
}
if ( strcmp( element, "Game" ) == 0 )
{
std::string title_str = get_fuzzy( m_rom.get_info( FeRomInfo::Title ) );
std::string work_title = get_fuzzy( m_work.get_info(FeRomInfo::Title ) );
bool match_plat=false;
for ( std::vector<std::string>::iterator itr=m_system_list.begin(); itr!=m_system_list.end(); ++itr )
{
if ( m_work_platform.compare( *itr ) == 0 )
{
match_plat=true;
break;
}
}
if ( work_title.compare( title_str ) == 0 )
{
if ( match_plat )
{
m_work.set_info( FeRomInfo::Emulator, m_rom.get_info( FeRomInfo::Emulator ) );
m_work.set_info( FeRomInfo::Romname, m_rom.get_info( FeRomInfo::Romname ) );
m_rom = m_work;
if ( m_art )
{
m_art->snap = m_work_art.snap;
m_art->marquee = m_work_art.marquee;
m_art->flyer = m_work_art.flyer;
m_art->fanart = m_work_art.fanart;
if ( !m_work_art.wheel.empty() )
m_art->wheel = m_work_art.wheel;
}
}
else if ( m_art && !m_work_art.wheel.empty() && m_art->wheel.empty() )
m_art->wheel = m_work_art.wheel;
}
else
{
std::string work_alt = get_fuzzy( m_work.get_info(FeRomInfo::AltTitle ) );
bool match_alt = ( work_alt.compare( title_str ) == 0 );
if ( match_alt && !m_work_art.wheel.empty() && m_art->wheel.empty() )
m_art->wheel = m_work_art.wheel;
if ( match_plat && match_alt )
{
if ( !m_work_art.flyer.empty() && m_art->flyer.empty() )
m_art->flyer = m_work_art.flyer;
if ( !m_work_art.marquee.empty() && m_art->marquee.empty() )
m_art->marquee = m_work_art.marquee;
if ( !m_work_art.snap.empty() && m_art->snap.empty() )
m_art->snap = m_work_art.snap;
}
}
}
}
bool FeMessXMLParser::parse( const std::string &prog,
const std::vector < std::string > &system_names )
{
// First get our machine -listxml settings
//
std::string system_name;
FeRomInfoListType::iterator itr;
for ( std::vector<std::string>::const_iterator its=system_names.begin(); its!=system_names.end(); ++its )
{
FeRomInfoListType temp_list;
temp_list.push_back( FeRomInfo( *its ) );
FeEmulatorInfo ignored;
FeImporterContext temp( ignored, temp_list );
FeMameXMLParser listxml( temp );
if (( listxml.parse( prog ) )
&& ( !temp_list.empty() ))
{
const FeRomInfo &ri = temp_list.front();
for ( itr=m_ctx.romlist.begin(); itr!=m_ctx.romlist.end(); ++itr )
{
(*itr).set_info( FeRomInfo::Players, ri.get_info( FeRomInfo::Players ));
(*itr).set_info( FeRomInfo::Rotation, ri.get_info( FeRomInfo::Rotation ));
(*itr).set_info( FeRomInfo::Control, ri.get_info( FeRomInfo::Control ));
(*itr).set_info( FeRomInfo::Status, ri.get_info( FeRomInfo::Status ));
(*itr).set_info( FeRomInfo::DisplayCount, ri.get_info( FeRomInfo::DisplayCount ));
(*itr).set_info( FeRomInfo::DisplayType, ri.get_info( FeRomInfo::DisplayType ));
// A bit of a hack here: the Category field gets repurposed for this stage of a MESS
// import...We temporarily store a "fuzzy" match romname
//
(*itr).set_info( FeRomInfo::BuildScratchPad,
get_fuzzy( (*itr).get_info( FeRomInfo::Romname ) ) );
}
system_name=(*its);
break;
}
}
if ( system_name.empty() )
{
std::cerr << " * Error: No system identifier found that is recognized by MESS -listxml" << std::endl;
return false;
}
std::cout << " - Obtaining -listsoftware info [" << system_name << "]" << std::endl;
// Now get the individual game -listsoftware settings
//
int retval=parse_internal( prog, system_name + " -listsoftware" );
if ( !retval )
{
std::cout << " * Error: No XML output found, command: " << prog << " "
<< system_name + " -listsoftware" << std::endl;
}
// We're done with our "fuzzy" matching, so clear where we were storing them
//
for ( itr=m_ctx.romlist.begin(); itr!=m_ctx.romlist.end(); ++itr )
(*itr).set_info( FeRomInfo::BuildScratchPad, "" );
return retval;
}
bool FeSettings::thegamesdb_scraper( FeImporterContext &c )
{
#ifndef NO_NET
const char *HOSTNAME = "http://thegamesdb.net";
const char *PLATFORM_LIST_REQ = "api/GetPlatformsList.php";
const char *PLAT_REQ = "api/GetPlatform.php?id=$1";
const char *GAME_REQ = "api/GetGame.php?name=$1";
const char *FLYER = "flyer/";
const char *WHEEL = "wheel/";
const char *MARQUEE = "marquee/";
const char *SNAP = "snap/";
const char *FANART = "fanart/";
//
// Get a list of valid platforms
//
FeNetQueue q;
q.add_buffer_task( HOSTNAME, PLATFORM_LIST_REQ, 0 );
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
get_resource( "Error getting platform list from thegamesdb.net. Code: $1",
as_str( status ), c.user_message );
std::cerr << " ! " << c.user_message << " (" << err_req << ")" << std::endl;
return true;
}
int temp;
std::string body;
q.pop_completed_task( temp, body );
FeGameDBPlatformListParser gdbplp;
gdbplp.parse( body );
std::vector<std::string> system_list;
std::vector<int> system_ids;
const std::vector<std::string> &sl_temp = c.emulator.get_systems();
for ( std::vector<std::string>::const_iterator itr = sl_temp.begin();
itr != sl_temp.end(); ++itr )
{
std::string comp_fuzz = get_fuzzy( *itr );
for ( size_t i=0; i<gdbplp.m_names.size(); i++ )
{
ASSERT( gdbplp.m_names.size() == gdbplp.m_ids.size() );
std::string &n = gdbplp.m_names[i];
int id = ( i < gdbplp.m_ids.size() ) ? i : 0;
if ( comp_fuzz.compare( get_fuzzy( n ) ) == 0 )
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
else
{
size_t pos = n.find_first_of( "(" );
if (( pos != std::string::npos ) &&
(( comp_fuzz.compare( get_fuzzy( n.substr(0,pos-1))) == 0 )
|| ( comp_fuzz.compare(get_fuzzy( n.substr(pos+1,n.size()-pos-1 ))) == 0 )))
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
}
}
}
if ( system_list.size() < 1 )
{
// Correct if we can based on the configured info source,
// otherwise we error out
switch( c.emulator.get_info_source() )
{
case FeEmulatorInfo::Listxml:
system_list.push_back( "Arcade" ); break;
case FeEmulatorInfo::Steam:
system_list.push_back( "PC" ); break;
default:
get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.",
c.user_message );
std::cerr << " ! " << c.user_message << std::endl;
return true;
}
}
std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name );
std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR;
base_path += emu_name + "/";
if ( c.scrape_art )
{
//
// Get emulator-specific images
//
for ( std::vector<int>::iterator iti=system_ids.begin();
iti != system_ids.end(); ++iti )
{
std::string plat_string = PLAT_REQ;
perform_substitution( plat_string, "$1", as_str( *iti ) );
q.add_buffer_task( HOSTNAME, plat_string, 0 );
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Unable to get platform information. Status code: "
<< status << " (" << err_req << ")" << std::endl;
continue;
}
body.clear();
q.pop_completed_task( temp, body );
FeGameDBArt my_art;
FeGameDBPlatformParser gdbpp( my_art );
gdbpp.parse( body );
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
std::string path = base_path + FLYER;
if ( m_scrape_flyers && ( !my_art.flyer.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer,
path + emu_name );
}
path = base_path + WHEEL;
if ( m_scrape_wheels && ( !my_art.wheel.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel,
path + emu_name );
}
path = base_path + MARQUEE;
if ( m_scrape_marquees && ( !my_art.marquee.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee,
path + emu_name );
}
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string path_base = base_path + FANART + emu_name + "/";
confirm_directory( base_path, "" );
confirm_directory( base_path + FANART, emu_name );
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( path_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
path_base + (*itr).substr( start_pos+1,
end_pos-start_pos-1 ) );
}
}
}
}
}
}
bool prefer_alt_filename = c.emulator.is_mess();
//
// Build a map for looking up parents
//
ParentMapType parent_map;
build_parent_map( parent_map, c.romlist, prefer_alt_filename );
//
// Build a worklist of the roms where we need to lookup
//
std::vector<FeRomInfo *> worklist;
worklist.reserve( c.romlist.size() );
for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr )
{
(*itr).set_info( FeRomInfo::Emulator, emu_name );
// Don't scrape for a clone if its parent has the same name
//
if ( has_same_name_as_parent( *itr, parent_map ) )
continue;
if ( !c.scrape_art || m_scrape_fanart
|| ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) )
|| ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) )
|| ( m_scrape_snaps && (!has_image_artwork( *itr, "snap" ) ) )
|| ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) )
worklist.push_back( &(*itr) );
}
const int NUM_ARTS=5; // the number of scrape-able artwork types
int done_count( 0 );
//
// Set up our initial queue of network tasks
//
for ( unsigned int i=0; i<worklist.size(); i++ )
{
std::string req_string = GAME_REQ;
std::string game = url_escape(
name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) );
perform_substitution( req_string, "$1", game );
//
// If we don't need to scrape a wheel artwork, then add the specific platform to our request
// If we are scraping a wheel, we want to be able to grab them where the game name (but
// not the system) matches, so we don't limit ourselves by system...
//
if (( system_list.size() == 1 )
&& ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) ))
{
req_string += "&platform=";
req_string += url_escape( system_list.front() );
}
q.add_buffer_task( HOSTNAME, req_string, i );
}
//
// Create worker threads to process the queue, adding new tasks to download
// artwork files where appropriate
//
FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q );
std::string aux;
//
// Process the output queue from our worker threads
//
while ( !( q.input_done() && q.output_done() ) )
{
int id;
std::string result;
if ( q.pop_completed_task( id, result ) )
{
if ( id < 0 )
{
if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask ))
{
std::cout << " + Downloaded: " << result << std::endl;
c.download_count++;
// find second last forward slash in filename
// we assume that there will always be at least two
size_t pos = result.find_last_of( "\\/" );
if ( pos != std::string::npos )
{
pos = result.find_last_of( "\\/", pos-1 );
if ( pos != std::string::npos )
aux = result.substr( pos );
}
}
if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask
done_count++;
}
else
{
FeGameDBArt my_art;
FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) );
gdbp.parse( result );
if ( c.scrape_art && !my_art.base.empty() )
{
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
const FeRomInfo &rom = *(worklist[id]);
if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) )
{
std::string fname = base_path + FLYER;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer, fname );
}
else
done_count++;
if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) )
{
std::string fname = base_path + WHEEL;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel, fname );
}
else
done_count++;
if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) )
{
std::string fname = base_path + MARQUEE;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee, fname );
}
else
done_count++;
if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_image_artwork( rom, "snap" )) )
{
std::string fname = base_path + SNAP;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, SNAP );
q.add_file_task( hostn, base_req + my_art.snap, fname );
}
else
done_count++;
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string fname_base = base_path + FANART;
confirm_directory( base_path, "" );
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
{
fname_base += rom.get_info( FeRomInfo::AltRomname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::AltRomname ) );
}
else
{
fname_base += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::Romname ) );
}
fname_base += "/";
bool done_first=false; // we only count the first fanart against our percentage completed...
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ),
done_first );
done_first=true;
}
}
}
}
else
done_count++;
}
else
{
aux = (worklist[id])->get_info( FeRomInfo::Title );
done_count+=NUM_ARTS;
}
}
if (( c.uiupdate ) && !worklist.empty() )
{
int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() );
if ( c.uiupdate( c.uiupdatedata, p, aux ) == false )
return false;
}
}
else if ( q.output_done() )
{
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Error processing request. Status code: "
<< status << " (" << err_req << ")" << std::endl;
}
}
else
sf::sleep( sf::milliseconds( 10 ) );
}
#endif
return true;
}
