bool FeSettings::mamedb_scraper( FeImporterContext &c ) { #ifndef NO_NET if ( !c.emulator.is_mame() || ( !m_scrape_snaps && !m_scrape_marquees )) return true; // // Build a map for looking up parents // ParentMapType parent_map; build_parent_map( parent_map, c.romlist, false ); const char *MAMEDB = "http://mamedb.com"; std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name ); std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR; base_path += emu_name + "/"; FeNetQueue q; int taskc( 0 ); for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr ) { // ugh, this must be set for has_artwork() to correctly function (*itr).set_info( FeRomInfo::Emulator, emu_name ); // Don't scrape for a clone if its parent has the same name // if ( has_same_name_as_parent( *itr, parent_map ) ) continue; if ( m_scrape_marquees && !has_artwork( *itr, "marquee" ) ) { const char *MARQUEE = "marquee/"; std::string fname = base_path + MARQUEE + (*itr).get_info( FeRomInfo::Romname ); confirm_directory( base_path, MARQUEE ); q.add_file_task( MAMEDB, "marquees/" + (*itr).get_info( FeRomInfo::Romname ) +".png", fname ); taskc++; } if ( m_scrape_snaps && !has_image_artwork( *itr, "snap" ) ) { const char *SNAP = "snap/"; std::string fname = base_path + SNAP + (*itr).get_info( FeRomInfo::Romname ); confirm_directory( base_path, SNAP ); q.add_file_task( MAMEDB, SNAP + (*itr).get_info( FeRomInfo::Romname ) +".png", fname ); taskc++; } } return process_q_simple( q, c, taskc ); #else return true; #endif }
bool FeSettings::build_romlist( const std::string &emu_name, UiUpdate uiu, void *uid, int &size ) { FeEmulatorInfo *emu = m_rl.get_emulator( emu_name ); if ( emu == NULL ) return false; // // Put up the "building romlist" message at 0 percent while we get going... // if ( uiu ) uiu( uid, 0 ); std::cout << "*** Generating Collection/Rom List" << std::endl; FeRomInfoListType romlist; FeImporterContext ctx( *emu, romlist ); ctx.uiupdate = uiu; ctx.uiupdatedata = uid; build_basic_romlist( ctx ); apply_xml_import( ctx ); apply_import_extras( ctx ); apply_emulator_name( emu_name, romlist ); romlist.sort( FeRomListSorter() ); // strip duplicate entries std::cout << " - Removing any duplicate entries..." << std::endl; romlist.unique(); size = romlist.size(); std::string filename = get_config_dir(); confirm_directory( filename, FE_ROMLIST_SUBDIR ); if ( uiu ) uiu( uid, 100 ); filename += FE_ROMLIST_SUBDIR; filename += emu_name; filename += FE_ROMLIST_FILE_EXTENSION; write_romlist( filename, romlist ); return true; }
bool FeEmulatorEditMenu::save( FeConfigContext &ctx ) { if ( !m_emulator ) return m_parent_save; for ( int i=0; i < FeEmulatorInfo::LAST_INDEX; i++ ) m_emulator->set_info( (FeEmulatorInfo::Index)i, ctx.opt_list[i].get_value() ); std::string filename = ctx.fe_settings.get_config_dir(); confirm_directory( filename, FE_EMULATOR_SUBDIR ); filename += FE_EMULATOR_SUBDIR; filename += m_emulator->get_info( FeEmulatorInfo::Name ); filename += FE_EMULATOR_FILE_EXTENSION; m_emulator->save( filename ); return m_parent_save; }
int main(int argc, char* argv[]) { musicd_start_time = time(NULL); config_init(); config_set_hook("log-level", log_level_changed); config_set_hook("log-time-format", log_time_format_changed); config_set("log-level", "debug"); config_set_hook("directory", directory_changed); config_set("config", "~/.musicd.conf"); config_set("directory", "~/.musicd"); config_set("bind", "any"); config_set("port", "6800"); config_set_hook("image-prefix", scan_image_prefix_changed); config_set("image-prefix", "front,cover,jacket"); config_set("server-name", "musicd server"); if (config_load_args(argc, argv)) { musicd_log(LOG_FATAL, "main", "invalid command line arguments"); print_usage(argv[0]); return -1; } if (config_get_value("help")) { print_usage(argv[0]); return 0; } if (config_get_value("version")) { print_version(); return 0; } if (!config_to_bool("no-config") && config_load_file(config_to_path("config"))) { musicd_log(LOG_FATAL, "main", "could not read config file"); return -1; } /* Reload command line arguments - this is because the config file might have * overwritten them, and the command line has the highest priority. */ config_load_args(argc, argv); confirm_directory(); musicd_log(LOG_INFO, "main", "musicd version %s", MUSICD_VERSION_STRING); srand(time(NULL)); av_register_all(); avcodec_register_all(); av_lockmgr_register(&musicd_av_lockmgr); av_log_set_level(AV_LOG_QUIET); if (db_open()) { musicd_log(LOG_FATAL, "library", "can't open database"); return -1; } if (library_open()) { musicd_log(LOG_FATAL, "main", "could not open library"); return -1; } if (cache_open()) { musicd_log(LOG_FATAL, "main", "could not open cache"); return -1; } if (server_start()) { musicd_log(LOG_FATAL, "main", "could not start server"); return -1; } signal(SIGUSR1, start_scan_signal); scan_start(); while (1) { sleep(1); } return 0; }
bool FeSettings::build_romlist( const std::vector< FeImportTask > &task_list, const std::string &output_name, FeFilter &filter, bool full ) { FeRomInfoListType total_romlist; std::string best_name, list_name, path; for ( std::vector<FeImportTask>::const_iterator itr=task_list.begin(); itr < task_list.end(); ++itr ) { if ( (*itr).task_type == FeImportTask::BuildRomlist ) { // Build romlist task std::cout << "*** Generating Collection/Rom List" << std::endl; FeEmulatorInfo *emu = m_rl.get_emulator( (*itr).emulator_name ); if ( emu == NULL ) { std::cout << "Error: Invalid -build-rom-list target: " << (*itr).emulator_name << std::endl; } else { FeRomInfoListType romlist; best_name = emu->get_info( FeEmulatorInfo::Name ); FeImporterContext ctx( *emu, romlist ); build_basic_romlist( ctx ); apply_xml_import( ctx ); apply_import_extras( ctx ); apply_emulator_name( best_name, romlist ); total_romlist.splice( total_romlist.end(), romlist ); } } else if ( (*itr).task_type == FeImportTask::ImportRomlist ) { // import romlist from file task std::cout << "*** Importing Collection/Rom List" << std::endl; FeRomInfoListType romlist; std::string emu_name; if ( (*itr).emulator_name.empty() ) { // deduce the emulator name from the filename provided size_t my_start = (*itr).file_name.find_last_of( "\\/" ); if ( my_start == std::string::npos ) // if there is no / we start at the beginning my_start = 0; else my_start += 1; size_t my_end = (*itr).file_name.find_last_of( "." ); if ( my_end != std::string::npos ) emu_name = (*itr).file_name.substr( my_start, my_end - my_start ); } else emu_name = (*itr).emulator_name; best_name = emu_name; if ( tail_compare( (*itr).file_name, ".txt" ) ) { // Attract-Mode format list // FeRomList temp_list( m_config_path ); temp_list.load_from_file( (*itr).file_name, ";" ); FeRomInfoListType &entries = temp_list.get_list(); for ( FeRomInfoListType::iterator itr = entries.begin(); itr != entries.end(); ++itr ) romlist.push_back( *itr ); } else if ( tail_compare( (*itr).file_name, ".lst" ) ) { // Mamewah/Wahcade! format list // import_mamewah( (*itr).file_name, emu_name, romlist ); } else if ( tail_compare( (*itr).file_name, ".xml" ) ) { // HyperSpin format list // FeHyperSpinXMLParser my_parser( romlist ); if ( my_parser.parse( (*itr).file_name ) ) apply_emulator_name( emu_name, romlist ); } else { std::cerr << "Error: Unsupported --import-rom-list file: " << (*itr).file_name << std::endl; } std::cout << "[Import " << (*itr).file_name << "] - Imported " << romlist.size() << " entries." << std::endl; FeEmulatorInfo *emu = m_rl.get_emulator( emu_name ); if ( emu == NULL ) { std::cout << "Warning: The emulator specified with --import-rom-list was not found: " << emu_name << std::endl; } else { FeImporterContext ctx( *emu, romlist ); apply_import_extras( ctx ); } total_romlist.splice( total_romlist.end(), romlist ); } else // scrape artwork { FeEmulatorInfo *emu = m_rl.get_emulator( (*itr).emulator_name ); if ( emu == NULL ) return false; std::cout << "*** Scraping artwork for: " << (*itr).emulator_name << std::endl; FeRomInfoListType romlist; std::string fn = get_config_dir() + FE_ROMLIST_SUBDIR + (*itr).emulator_name + FE_ROMLIST_FILE_EXTENSION; FeImporterContext ctx( *emu, romlist ); if ( file_exists( fn ) ) { FeRomList loader( get_config_dir() ); loader.load_from_file( fn, ";" ); ctx.romlist.swap( loader.get_list() ); } else { build_basic_romlist( ctx ); apply_xml_import( ctx ); } ctx.scrape_art = true; confirm_directory( get_config_dir(), FE_SCRAPER_SUBDIR ); // do the mamedb scraper first (which only does anything for mame) followed // by the more general thegamesdb scraper. mamedb_scraper( ctx ); thegamesdb_scraper( ctx ); std::cout << "*** Scraping done." << std::endl; } } // return now if all we did was scrape artwork if ( total_romlist.empty() ) return true; total_romlist.sort( FeRomListSorter() ); // strip duplicate entries std::cout << " - Removing any duplicate entries..." << std::endl; total_romlist.unique(); // Apply the specified filter if ( filter.get_rule_count() > 0 ) { std::cout << " - Applying filter..." << std::endl; filter.init(); FeRomInfoListType::iterator last_it=total_romlist.begin(); for ( FeRomInfoListType::iterator it=total_romlist.begin(); it!=total_romlist.end(); ) { if ( filter.apply_filter( *it ) ) { if ( last_it != it ) it = total_romlist.erase( last_it, it ); else ++it; last_it = it; } else ++it; } if ( last_it != total_romlist.end() ) total_romlist.erase( last_it, total_romlist.end() ); } if ( task_list.size() > 1 ) best_name = "multi"; path = get_config_dir(); confirm_directory( path, FE_ROMLIST_SUBDIR ); path += FE_ROMLIST_SUBDIR; // if we weren't given a specific output name, then we come up with a name // that doesn't exist already // if ( output_name.empty() ) get_available_filename( path, best_name, FE_ROMLIST_FILE_EXTENSION, list_name ); else list_name = path + output_name + FE_ROMLIST_FILE_EXTENSION; write_romlist( list_name, total_romlist ); return true; }
bool FeSettings::thegamesdb_scraper( FeImporterContext &c ) { #ifndef NO_NET const char *HOSTNAME = "http://thegamesdb.net"; const char *PLATFORM_REQ = "api/GetPlatformsList.php"; const char *GAME_REQ = "api/GetGame.php?name=$1"; // // Get a list of valid platforms // FeNetQueue q; q.add_buffer_task( HOSTNAME, PLATFORM_REQ, 0 ); sf::Http::Response::Status status; q.do_next_task( status ); if ( status != sf::Http::Response::Ok ) { get_resource( "Error getting platform list from thegamesdb.net. Code: $1", as_str( status ), c.user_message ); std::cout << " * " << c.user_message << std::endl; return true; } int temp; std::string body; q.pop_completed_task( temp, body ); FeGameDBPlatformParser gdbpp; gdbpp.parse( body ); const std::vector<std::string> &sl_temp = c.emulator.get_systems(); std::vector<std::string> system_list; for ( std::vector<std::string>::const_iterator itr = sl_temp.begin(); itr!=sl_temp.end(); ++itr ) { if ( gdbpp.m_set.find( *itr ) != gdbpp.m_set.end() ) system_list.push_back( *itr ); else std::cout << " * System identifier '" << (*itr) << "' not recognized by " << HOSTNAME << std::endl; } if ( system_list.size() < 1 ) { // Correct if we can based on the configured info source, // otherwise we error out const std::string source = c.emulator.get_info( FeEmulatorInfo::Info_source ); if ( source.compare( "mame" ) == 0 ) system_list.push_back( "Arcade" ); else if ( source.compare( "steam" ) == 0 ) system_list.push_back( "PC" ); else { get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.", c.user_message ); std::cout << " * " << c.user_message << std::endl; return true; } } std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name ); // // Build a map for looking up parents // std::map < std::string, FeRomInfo * > parent_map; build_parent_map( parent_map, c.romlist ); // // Build a worklist of the roms where we need to lookup // std::vector<FeRomInfo *> worklist; worklist.reserve( c.romlist.size() ); for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr ) { (*itr).set_info( FeRomInfo::Emulator, emu_name ); // Don't scrape for a clone if its parent has the same name // if ( has_same_name_as_parent( *itr, parent_map ) ) continue; if ( !c.scrape_art || m_scrape_fanart || ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) ) || ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) ) || ( m_scrape_snaps && (!has_artwork( *itr, "snap" ) ) ) || ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) ) worklist.push_back( &(*itr) ); } const int NUM_ARTS=5; // the number of scrape-able artwork types int done_count( 0 ); // // Set up our initial queue of network tasks // for ( unsigned int i=0; i<worklist.size(); i++ ) { std::string req_string = GAME_REQ; std::string game = url_escape( name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) ); perform_substitution( req_string, "$1", game ); // // If we don't need to scrape a wheel artwork, then add the specific platform to our request // If we are scraping a wheel, we want to be able to grab them where the game name (but // not the system) matches, so we don't limit ourselves by system... // if (( system_list.size() == 1 ) && ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) )) { req_string += "&platform="; req_string += url_escape( system_list.front() ); } q.add_buffer_task( HOSTNAME, req_string, i ); } std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR; base_path += emu_name + "/"; // // Create worker threads to process the queue, adding new tasks to download // artwork files where appropriate // FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q ); // // Process the output queue from our worker threads // while ( !q.input_done() || !q.output_done() ) { int id; std::string result; if ( q.pop_completed_task( id, result ) ) { if ( id < 0 ) { if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask )) { std::cout << " + Downloaded: " << result << std::endl; c.download_count++; } if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask done_count++; } else { FeGameDBArt my_art; FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) ); gdbp.parse( result ); if ( c.scrape_art && !my_art.base.empty() ) { std::string hostn = HOSTNAME; std::string base_req = "banners/"; size_t pos=0; for ( int i=0; i<3 && ( pos != std::string::npos ); i++ ) pos = my_art.base.find_first_of( '/', pos+1 ); if (( pos != std::string::npos ) && ( pos < my_art.base.size()-1 ) ) { hostn = my_art.base.substr( 0, pos+1 ); base_req = my_art.base.substr( pos+1 ); } FeRomInfo &rom = *(worklist[id]); if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) ) { const char *FLYER = "flyer/"; std::string fname = base_path + FLYER + rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, FLYER ); q.add_file_task( hostn, base_req + my_art.flyer, fname ); } else done_count++; if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) ) { const char *WHEEL = "wheel/"; std::string fname = base_path + WHEEL + rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, WHEEL ); q.add_file_task( hostn, base_req + my_art.wheel, fname ); } else done_count++; if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) ) { const char *MARQUEE = "marquee/"; std::string fname = base_path + MARQUEE + rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, MARQUEE ); q.add_file_task( hostn, base_req + my_art.marquee, fname ); } else done_count++; if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_artwork( rom, "snap" )) ) { const char *SNAP = "snap/"; std::string fname = base_path + SNAP + rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, SNAP ); q.add_file_task( hostn, base_req + my_art.snap, fname ); } else done_count++; if ( m_scrape_fanart && !my_art.fanart.empty() ) { const char *FANART = "fanart/"; std::string fname_base = base_path + FANART + rom.get_info( FeRomInfo::Romname ) + "/"; confirm_directory( base_path, "" ); confirm_directory( base_path + FANART, rom.get_info( FeRomInfo::Romname ) ); bool done_first=false; // we only count the first fanart against our percentage completed... for ( std::vector<std::string>::iterator itr = my_art.fanart.begin(); itr != my_art.fanart.end(); ++itr ) { size_t start_pos = (*itr).find_last_of( "/\\" ); size_t end_pos = (*itr).find_last_of( '.' ); if (( start_pos != std::string::npos ) && ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) )) { if (( end_pos != std::string::npos ) && ( end_pos > start_pos )) { q.add_file_task( hostn, base_req + (*itr), fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ), done_first ); done_first=true; } } } } else done_count++; } else done_count+=NUM_ARTS; } if ( c.uiupdate ) { int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() ); if ( c.uiupdate( c.uiupdatedata, p ) == false ) return false; } } else if ( q.output_done() ) { sf::Http::Response::Status status; q.do_next_task( status ); } else sf::sleep( sf::milliseconds( 10 ) ); } #endif return true; }
bool FeSettings::mamedb_scraper( FeImporterContext &c ) { #ifndef NO_NET if (( c.emulator.get_info( FeEmulatorInfo::Info_source ).compare( "mame" ) != 0 ) || ( !m_scrape_snaps && !m_scrape_marquees )) return true; // // Build a map for looking up parents // std::map < std::string, FeRomInfo * > parent_map; build_parent_map( parent_map, c.romlist ); const char *MAMEDB = "http://mamedb.com"; std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name ); std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR; base_path += emu_name + "/"; FeNetQueue q; int taskc( 0 ); int done( 0 ); for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr ) { // ugh, this must be set for has_artwork() to correctly function (*itr).set_info( FeRomInfo::Emulator, emu_name ); // Don't scrape for a clone if its parent has the same name // if ( has_same_name_as_parent( *itr, parent_map ) ) continue; if ( m_scrape_marquees && !has_artwork( *itr, "marquee" ) ) { const char *MARQUEE = "marquee/"; std::string fname = base_path + MARQUEE + (*itr).get_info( FeRomInfo::Romname ); confirm_directory( base_path, MARQUEE ); q.add_file_task( MAMEDB, "marquees/" + (*itr).get_info( FeRomInfo::Romname ) +".png", fname ); taskc++; } if ( m_scrape_snaps && !has_artwork( *itr, "snap" ) ) { const char *SNAP = "snap/"; std::string fname = base_path + SNAP + (*itr).get_info( FeRomInfo::Romname ); confirm_directory( base_path, SNAP ); q.add_file_task( MAMEDB, SNAP + (*itr).get_info( FeRomInfo::Romname ) +".png", fname ); taskc++; } } // // Create worker threads to process the queue. // FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q ); // // Process the output queue from our worker threads // while ( !( q.input_done() && q.output_done() ) ) { int id; std::string result; if ( q.pop_completed_task( id, result ) ) { if ( id < 0 ) { if ( id == -1 ) { std::cout << " + Downloaded: " << result << std::endl; c.download_count++; } done++; } } if ( c.uiupdate ) { int p = c.progress_past + done * c.progress_range / taskc; if ( c.uiupdate( c.uiupdatedata, p ) == false ) return false; } } #endif return true; }
bool FeSettings::scrape_artwork( const std::string &emu_name, UiUpdate uiu, void *uid, std::string &msg ) { FeEmulatorInfo *emu = m_rl.get_emulator( emu_name ); if ( emu == NULL ) return false; // // Put up the "scraping artwork" message at 0 percent while we get going... // if ( uiu ) uiu( uid, 0 ); std::cout << "*** Scraping artwork for: " << emu_name << std::endl; FeRomInfoListType romlist; std::string fn = get_config_dir() + FE_ROMLIST_SUBDIR + emu_name + FE_ROMLIST_FILE_EXTENSION; FeImporterContext ctx( *emu, romlist ); ctx.uiupdate = uiu; ctx.uiupdatedata = uid; if ( file_exists( fn ) ) { FeRomList loader( get_config_dir() ); loader.load_from_file( fn, ";" ); ctx.romlist.swap( loader.get_list() ); } else { ctx.progress_range=33; build_basic_romlist( ctx ); apply_xml_import( ctx ); ctx.progress_past=33; } ctx.progress_range = ( 100-ctx.progress_past ) / 2; ctx.scrape_art = true; confirm_directory( get_config_dir(), FE_SCRAPER_SUBDIR ); // do the mamedb scraper first (which only does anything for mame) followed // by the more general thegamesdb scraper. these return false if the user // cancels... if ( mamedb_scraper( ctx ) ) { ctx.progress_past = ctx.progress_past + ctx.progress_range; thegamesdb_scraper( ctx ); } if ( uiu ) uiu( uid, 100 ); std::cout << "*** Scraping done." << std::endl; if ( ctx.user_message.empty() ) get_resource( "Scraped $1 artwork file(s)", as_str( ctx.download_count ), msg ); else msg = ctx.user_message; return true; }
bool FeSettings::thegamesdb_scraper( FeImporterContext &c ) { #ifndef NO_NET const char *HOSTNAME = "http://thegamesdb.net"; const char *PLATFORM_LIST_REQ = "api/GetPlatformsList.php"; const char *PLAT_REQ = "api/GetPlatform.php?id=$1"; const char *GAME_REQ = "api/GetGame.php?name=$1"; const char *FLYER = "flyer/"; const char *WHEEL = "wheel/"; const char *MARQUEE = "marquee/"; const char *SNAP = "snap/"; const char *FANART = "fanart/"; // // Get a list of valid platforms // FeNetQueue q; q.add_buffer_task( HOSTNAME, PLATFORM_LIST_REQ, 0 ); sf::Http::Response::Status status; std::string err_req; q.do_next_task( status, err_req ); if ( status != sf::Http::Response::Ok ) { get_resource( "Error getting platform list from thegamesdb.net. Code: $1", as_str( status ), c.user_message ); std::cerr << " ! " << c.user_message << " (" << err_req << ")" << std::endl; return true; } int temp; std::string body; q.pop_completed_task( temp, body ); FeGameDBPlatformListParser gdbplp; gdbplp.parse( body ); std::vector<std::string> system_list; std::vector<int> system_ids; const std::vector<std::string> &sl_temp = c.emulator.get_systems(); for ( std::vector<std::string>::const_iterator itr = sl_temp.begin(); itr != sl_temp.end(); ++itr ) { std::string comp_fuzz = get_fuzzy( *itr ); for ( size_t i=0; i<gdbplp.m_names.size(); i++ ) { ASSERT( gdbplp.m_names.size() == gdbplp.m_ids.size() ); std::string &n = gdbplp.m_names[i]; int id = ( i < gdbplp.m_ids.size() ) ? i : 0; if ( comp_fuzz.compare( get_fuzzy( n ) ) == 0 ) { system_list.push_back( n ); system_ids.push_back( id ); break; } else { size_t pos = n.find_first_of( "(" ); if (( pos != std::string::npos ) && (( comp_fuzz.compare( get_fuzzy( n.substr(0,pos-1))) == 0 ) || ( comp_fuzz.compare(get_fuzzy( n.substr(pos+1,n.size()-pos-1 ))) == 0 ))) { system_list.push_back( n ); system_ids.push_back( id ); break; } } } } if ( system_list.size() < 1 ) { // Correct if we can based on the configured info source, // otherwise we error out switch( c.emulator.get_info_source() ) { case FeEmulatorInfo::Listxml: system_list.push_back( "Arcade" ); break; case FeEmulatorInfo::Steam: system_list.push_back( "PC" ); break; default: get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.", c.user_message ); std::cerr << " ! " << c.user_message << std::endl; return true; } } std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name ); std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR; base_path += emu_name + "/"; if ( c.scrape_art ) { // // Get emulator-specific images // for ( std::vector<int>::iterator iti=system_ids.begin(); iti != system_ids.end(); ++iti ) { std::string plat_string = PLAT_REQ; perform_substitution( plat_string, "$1", as_str( *iti ) ); q.add_buffer_task( HOSTNAME, plat_string, 0 ); q.do_next_task( status, err_req ); if ( status != sf::Http::Response::Ok ) { std::cout << " * Unable to get platform information. Status code: " << status << " (" << err_req << ")" << std::endl; continue; } body.clear(); q.pop_completed_task( temp, body ); FeGameDBArt my_art; FeGameDBPlatformParser gdbpp( my_art ); gdbpp.parse( body ); std::string hostn = HOSTNAME; std::string base_req = "banners/"; get_url_components( my_art.base, hostn, base_req ); std::string path = base_path + FLYER; if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && ( !art_exists( path, emu_name ) )) { confirm_directory( base_path, FLYER ); q.add_file_task( hostn, base_req + my_art.flyer, path + emu_name ); } path = base_path + WHEEL; if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && ( !art_exists( path, emu_name ) )) { confirm_directory( base_path, WHEEL ); q.add_file_task( hostn, base_req + my_art.wheel, path + emu_name ); } path = base_path + MARQUEE; if ( m_scrape_marquees && ( !my_art.marquee.empty() ) && ( !art_exists( path, emu_name ) )) { confirm_directory( base_path, MARQUEE ); q.add_file_task( hostn, base_req + my_art.marquee, path + emu_name ); } if ( m_scrape_fanart && !my_art.fanart.empty() ) { std::string path_base = base_path + FANART + emu_name + "/"; confirm_directory( base_path, "" ); confirm_directory( base_path + FANART, emu_name ); for ( std::vector<std::string>::iterator itr = my_art.fanart.begin(); itr != my_art.fanart.end(); ++itr ) { size_t start_pos = (*itr).find_last_of( "/\\" ); size_t end_pos = (*itr).find_last_of( '.' ); if (( start_pos != std::string::npos ) && ( !file_exists( path_base + (*itr).substr( start_pos+1 ) ) )) { if (( end_pos != std::string::npos ) && ( end_pos > start_pos )) { q.add_file_task( hostn, base_req + (*itr), path_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ) ); } } } } } } bool prefer_alt_filename = c.emulator.is_mess(); // // Build a map for looking up parents // ParentMapType parent_map; build_parent_map( parent_map, c.romlist, prefer_alt_filename ); // // Build a worklist of the roms where we need to lookup // std::vector<FeRomInfo *> worklist; worklist.reserve( c.romlist.size() ); for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr ) { (*itr).set_info( FeRomInfo::Emulator, emu_name ); // Don't scrape for a clone if its parent has the same name // if ( has_same_name_as_parent( *itr, parent_map ) ) continue; if ( !c.scrape_art || m_scrape_fanart || ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) ) || ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) ) || ( m_scrape_snaps && (!has_image_artwork( *itr, "snap" ) ) ) || ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) ) worklist.push_back( &(*itr) ); } const int NUM_ARTS=5; // the number of scrape-able artwork types int done_count( 0 ); // // Set up our initial queue of network tasks // for ( unsigned int i=0; i<worklist.size(); i++ ) { std::string req_string = GAME_REQ; std::string game = url_escape( name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) ); perform_substitution( req_string, "$1", game ); // // If we don't need to scrape a wheel artwork, then add the specific platform to our request // If we are scraping a wheel, we want to be able to grab them where the game name (but // not the system) matches, so we don't limit ourselves by system... // if (( system_list.size() == 1 ) && ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) )) { req_string += "&platform="; req_string += url_escape( system_list.front() ); } q.add_buffer_task( HOSTNAME, req_string, i ); } // // Create worker threads to process the queue, adding new tasks to download // artwork files where appropriate // FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q ); std::string aux; // // Process the output queue from our worker threads // while ( !( q.input_done() && q.output_done() ) ) { int id; std::string result; if ( q.pop_completed_task( id, result ) ) { if ( id < 0 ) { if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask )) { std::cout << " + Downloaded: " << result << std::endl; c.download_count++; // find second last forward slash in filename // we assume that there will always be at least two size_t pos = result.find_last_of( "\\/" ); if ( pos != std::string::npos ) { pos = result.find_last_of( "\\/", pos-1 ); if ( pos != std::string::npos ) aux = result.substr( pos ); } } if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask done_count++; } else { FeGameDBArt my_art; FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) ); gdbp.parse( result ); if ( c.scrape_art && !my_art.base.empty() ) { std::string hostn = HOSTNAME; std::string base_req = "banners/"; get_url_components( my_art.base, hostn, base_req ); const FeRomInfo &rom = *(worklist[id]); if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) ) { std::string fname = base_path + FLYER; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, FLYER ); q.add_file_task( hostn, base_req + my_art.flyer, fname ); } else done_count++; if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) ) { std::string fname = base_path + WHEEL; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, WHEEL ); q.add_file_task( hostn, base_req + my_art.wheel, fname ); } else done_count++; if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) ) { std::string fname = base_path + MARQUEE; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, MARQUEE ); q.add_file_task( hostn, base_req + my_art.marquee, fname ); } else done_count++; if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_image_artwork( rom, "snap" )) ) { std::string fname = base_path + SNAP; const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) fname += rom.get_info( FeRomInfo::AltRomname ); else fname += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path, SNAP ); q.add_file_task( hostn, base_req + my_art.snap, fname ); } else done_count++; if ( m_scrape_fanart && !my_art.fanart.empty() ) { std::string fname_base = base_path + FANART; confirm_directory( base_path, "" ); const std::string &altname = rom.get_info( FeRomInfo::AltRomname ); if ( prefer_alt_filename && !altname.empty() ) { fname_base += rom.get_info( FeRomInfo::AltRomname ); confirm_directory( base_path + FANART, rom.get_info( FeRomInfo::AltRomname ) ); } else { fname_base += rom.get_info( FeRomInfo::Romname ); confirm_directory( base_path + FANART, rom.get_info( FeRomInfo::Romname ) ); } fname_base += "/"; bool done_first=false; // we only count the first fanart against our percentage completed... for ( std::vector<std::string>::iterator itr = my_art.fanart.begin(); itr != my_art.fanart.end(); ++itr ) { size_t start_pos = (*itr).find_last_of( "/\\" ); size_t end_pos = (*itr).find_last_of( '.' ); if (( start_pos != std::string::npos ) && ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) )) { if (( end_pos != std::string::npos ) && ( end_pos > start_pos )) { q.add_file_task( hostn, base_req + (*itr), fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ), done_first ); done_first=true; } } } } else done_count++; } else { aux = (worklist[id])->get_info( FeRomInfo::Title ); done_count+=NUM_ARTS; } } if (( c.uiupdate ) && !worklist.empty() ) { int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() ); if ( c.uiupdate( c.uiupdatedata, p, aux ) == false ) return false; } } else if ( q.output_done() ) { sf::Http::Response::Status status; std::string err_req; q.do_next_task( status, err_req ); if ( status != sf::Http::Response::Ok ) { std::cout << " * Error processing request. Status code: " << status << " (" << err_req << ")" << std::endl; } } else sf::sleep( sf::milliseconds( 10 ) ); } #endif return true; }
void FeRomList::save_tags() { if (( !m_tags_changed ) || ( m_user_path.empty() ) || ( m_romlist_name.empty() )) return; // First construct a mapping of tags to rom entries // std::multimap< std::string, const char * > tag_to_rom_map; for ( std::multimap< std::string, const char * >::const_iterator ite = m_extra_tags.begin(); ite != m_extra_tags.end(); ++ite ) { tag_to_rom_map.insert( std::pair<std::string,const char *>( (*ite).second, (*ite).first.c_str() ) ); } for ( FeRomInfoListType::const_iterator itr = m_list.begin(); itr != m_list.end(); ++itr ) { const std::string &my_tags = (*itr).get_info( FeRomInfo::Tags ); size_t pos=0; do { std::string one_tag; const char sep[] = { FE_TAGS_SEP, 0 }; token_helper( my_tags, pos, one_tag, sep ); if ( !one_tag.empty() ) { tag_to_rom_map.insert( std::pair<std::string,const char *>( one_tag, ((*itr).get_info( FeRomInfo::Romname )).c_str() ) ); } } while ( pos < my_tags.size() ); } confirm_directory( m_user_path, m_romlist_name ); std::string my_path = m_user_path + m_romlist_name + "/"; // // Now save the tags // std::map<std::string,bool>::const_iterator itt; for ( itt = m_tags.begin(); itt != m_tags.end(); ++itt ) { if ( (*itt).second == false ) continue; std::string file_name = my_path + (*itt).first + FE_FAVOURITE_FILE_EXTENSION; std::pair<std::multimap<std::string,const char *>::const_iterator,std::multimap<std::string,const char *>::const_iterator> ret; ret = tag_to_rom_map.equal_range( (*itt).first ); if ( ret.first == ret.second ) { delete_file( file_name ); } else { std::ofstream outfile( file_name.c_str() ); if ( !outfile.is_open() ) continue; for ( std::multimap<std::string,const char *>::const_iterator ito = ret.first; ito != ret.second; ++ito ) outfile << (*ito).second << std::endl; outfile.close(); } } }