void AdminPortalProtocol::admin_net_del_link(const std::string& name) throw (P4PProtocolError)
{
check_txn();
make_request("DELETE", "admin/" + admin_token_ + "/net/link/" + url_escape(name));
}
/* test unquoting and canonizing */
int test_quote(void)
{
su_home_t home[1] = { SU_HOME_INIT(home) };
url_t *u;
char s[] = "%73ip:q%74est%01:%01%02%00@host%2enokia.com;%70aram=%01%02";
char c[] = "sip:qtest%01:%01%02%[email protected];param=%01%02";
char *d;
#define RESERVED ";/?:@&=+$,"
#define DELIMS "<>#%\""
#define UNWISE "{}|\\^[]`"
#define EXCLUDED RESERVED DELIMS UNWISE
char escaped[1 + 3 * 23 + 1];
#define UNRESERVED "ABCDEFGHIJKLMNOPQRSTUVWXYZ" \
"abcdefghijklmnopqrstuvwxyz" \
"0123456789" \
"-_.!~*'()"
char unreserved[26 + 26 + 10 + 9 + 1];
BEGIN();
d = url_as_string(home, (url_t *)"sip:[email protected]");
TEST_S(d, "sip:[email protected]");
TEST(strlen(EXCLUDED), 23);
TEST(strlen(UNRESERVED), 71);
TEST_1(!url_reserved_p("foo"));
TEST_1(!url_reserved_p(""));
TEST_1(url_reserved_p("foobar:bar"));
TEST_SIZE(url_esclen("a" EXCLUDED, ""),
1 + strlen(RESERVED) + 3 * strlen(DELIMS UNWISE));
TEST_SIZE(url_esclen("a" EXCLUDED, DELIMS UNWISE),
1 + strlen(RESERVED) + 3 * strlen(DELIMS UNWISE));
TEST_SIZE(url_esclen("a" EXCLUDED, EXCLUDED), 1 + 3 * strlen(EXCLUDED));
TEST_SIZE(url_esclen("a" EXCLUDED, NULL), 1 + 3 * strlen(EXCLUDED));
TEST_S(url_escape(escaped, "a" EXCLUDED, NULL),
"a%3B%2F%3F%3A%40%26%3D%2B%24%2C"
"%3C%3E%23%25%22"
"%7B%7D%7C%5C%5E%5B%5D%60");
TEST_S(url_unescape(escaped, escaped), "a" EXCLUDED);
TEST_SIZE(url_esclen(UNRESERVED, NULL), strlen(UNRESERVED));
TEST_S(url_escape(unreserved, UNRESERVED, NULL), UNRESERVED);
TEST_S(url_unescape(unreserved, UNRESERVED), UNRESERVED);
d = "%53ip:%75@%48"; /* Sip:u@H */
u = url_hdup(home, (url_t *)d); TEST_1(u);
url_digest(hash1, sizeof(hash1), u, NULL);
url_digest(hash2, sizeof(hash2), (url_t const *)d, NULL);
TEST(memcmp(hash1, hash2, sizeof(hash1)), 0);
d = "sip:u@h";
u = url_hdup(home, (url_t *)d); TEST_1(u);
url_digest(hash1, sizeof(hash1), u, NULL);
TEST(memcmp(hash1, hash2, sizeof(hash1)), 0);
url_digest(hash2, sizeof(hash2), (url_t const *)d, NULL);
TEST(memcmp(hash1, hash2, sizeof(hash1)), 0);
u = url_hdup(home, (url_t *)s); TEST_1(u);
d = url_as_string(home, u); TEST_1(d);
TEST_S(d, c);
d = "sip:&=+$,;?/:&=+$,@[::1]:56001;param=+$,/:@&;another=@%40%2F"
"?header=" RESERVED "&%3b%2f%3f%3a%40%26%3d%2b%24%2c";
u = url_hdup(home, (url_t *)d); TEST_1(u);
TEST_S(u->url_user, "&=+$,;?/");
TEST_S(u->url_host, "[::1]");
TEST_S(u->url_params, "param=+$,/:@&;another=@%40/");
TEST_S(u->url_headers, "header=" RESERVED "&%3B%2F%3F%3A%40%26%3D%2B%24%2C");
url_digest(hash1, sizeof(hash1), u, NULL);
url_digest(hash2, sizeof(hash2), (url_t const *)d, NULL);
TEST(memcmp(hash1, hash2, sizeof(hash1)), 0);
u = url_hdup(home, (url_t *)s); TEST_1(u);
d = url_as_string(home, u); TEST_1(d);
TEST_S(d, c);
d = "http://&=+$,;:&=+$,;@host:8080/foo%2F%3B%3D"
";param=+$,%2f%3b%3d/bar;param=:@&;another=@"
"?query=" RESERVED;
u = url_hdup(home, (url_t *)d); TEST_1(u);
TEST_S(u->url_user, "&=+$,;"); TEST_S(u->url_password, "&=+$,;");
TEST_S(u->url_path, "foo%2F%3B%3D;param=+$,%2F%3B%3D/bar;param=:@&;another=@");
url_digest(hash1, sizeof(hash1), u, NULL);
url_digest(hash2, sizeof(hash2), (url_t const *)d, NULL);
TEST(memcmp(hash1, hash2, sizeof(hash1)), 0);
u = url_hdup(home, (url_t *)s); TEST_1(u);
d = url_as_string(home, u); TEST_1(d);
TEST_S(d, c);
url_digest(hash1, sizeof(hash1), u, NULL);
url_digest(hash2, sizeof(hash2), (url_t const *)s, NULL);
TEST(memcmp(hash1, hash2, sizeof(hash1)), 0);
url_digest(hash2, sizeof(hash2), (url_t const *)c, NULL);
TEST(memcmp(hash1, hash2, sizeof(hash1)), 0);
END();
}
void AdminPortalProtocol::admin_net_add_node(const std::string& name, bool external) throw (P4PProtocolError)
{
check_txn();
make_request("PUT", "admin/" + admin_token_ + "/net/node/" + url_escape(name) + (external ? "?ext=1" : ""));
}
void AdminPortalProtocol::admin_net_add_link(const NamedNetLink& link) throw (P4PProtocolError)
{
check_txn();
make_request("PUT", "admin/" + admin_token_ + "/net/link/" + url_escape(link.name) + "?src=" + url_escape(link.link.src) + "&dst=" + url_escape(link.link.dst));
}
void AdminPortalProtocol::admin_view_add(const std::string& name) throw (P4PProtocolError)
{
check_txn();
make_request("PUT", "admin/" + admin_token_ + '/' + url_escape(name));
}
void AdminPortalProtocol::admin_view_update_pdistance(const std::string& view) throw (P4PProtocolError)
{
check_txn();
make_request("POST", "admin/" + admin_token_ + '/' + url_escape(view) + "/pdistance");
}
void AdminPortalProtocol::admin_view_set_pidlink_weight(const std::string& view, const std::string& link, double value) throw (P4PProtocolError)
{
check_txn();
make_request("PUT", "admin/" + admin_token_ + '/' + url_escape(view) + "/link/" + url_escape(link) + "/weight?v=" + p4p::detail::p4p_token_cast<std::string>(value));
}
bool FeSettings::thegamesdb_scraper( FeImporterContext &c )
{
#ifndef NO_NET
const char *HOSTNAME = "http://thegamesdb.net";
const char *PLATFORM_LIST_REQ = "api/GetPlatformsList.php";
const char *PLAT_REQ = "api/GetPlatform.php?id=$1";
const char *GAME_REQ = "api/GetGame.php?name=$1";
const char *FLYER = "flyer/";
const char *WHEEL = "wheel/";
const char *MARQUEE = "marquee/";
const char *SNAP = "snap/";
const char *FANART = "fanart/";
//
// Get a list of valid platforms
//
FeNetQueue q;
q.add_buffer_task( HOSTNAME, PLATFORM_LIST_REQ, 0 );
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
get_resource( "Error getting platform list from thegamesdb.net. Code: $1",
as_str( status ), c.user_message );
std::cerr << " ! " << c.user_message << " (" << err_req << ")" << std::endl;
return true;
}
int temp;
std::string body;
q.pop_completed_task( temp, body );
FeGameDBPlatformListParser gdbplp;
gdbplp.parse( body );
std::vector<std::string> system_list;
std::vector<int> system_ids;
const std::vector<std::string> &sl_temp = c.emulator.get_systems();
for ( std::vector<std::string>::const_iterator itr = sl_temp.begin();
itr != sl_temp.end(); ++itr )
{
std::string comp_fuzz = get_fuzzy( *itr );
for ( size_t i=0; i<gdbplp.m_names.size(); i++ )
{
ASSERT( gdbplp.m_names.size() == gdbplp.m_ids.size() );
std::string &n = gdbplp.m_names[i];
int id = ( i < gdbplp.m_ids.size() ) ? i : 0;
if ( comp_fuzz.compare( get_fuzzy( n ) ) == 0 )
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
else
{
size_t pos = n.find_first_of( "(" );
if (( pos != std::string::npos ) &&
(( comp_fuzz.compare( get_fuzzy( n.substr(0,pos-1))) == 0 )
|| ( comp_fuzz.compare(get_fuzzy( n.substr(pos+1,n.size()-pos-1 ))) == 0 )))
{
system_list.push_back( n );
system_ids.push_back( id );
break;
}
}
}
}
if ( system_list.size() < 1 )
{
// Correct if we can based on the configured info source,
// otherwise we error out
switch( c.emulator.get_info_source() )
{
case FeEmulatorInfo::Listxml:
system_list.push_back( "Arcade" ); break;
case FeEmulatorInfo::Steam:
system_list.push_back( "PC" ); break;
default:
get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.",
c.user_message );
std::cerr << " ! " << c.user_message << std::endl;
return true;
}
}
std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name );
std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR;
base_path += emu_name + "/";
if ( c.scrape_art )
{
//
// Get emulator-specific images
//
for ( std::vector<int>::iterator iti=system_ids.begin();
iti != system_ids.end(); ++iti )
{
std::string plat_string = PLAT_REQ;
perform_substitution( plat_string, "$1", as_str( *iti ) );
q.add_buffer_task( HOSTNAME, plat_string, 0 );
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Unable to get platform information. Status code: "
<< status << " (" << err_req << ")" << std::endl;
continue;
}
body.clear();
q.pop_completed_task( temp, body );
FeGameDBArt my_art;
FeGameDBPlatformParser gdbpp( my_art );
gdbpp.parse( body );
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
std::string path = base_path + FLYER;
if ( m_scrape_flyers && ( !my_art.flyer.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer,
path + emu_name );
}
path = base_path + WHEEL;
if ( m_scrape_wheels && ( !my_art.wheel.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel,
path + emu_name );
}
path = base_path + MARQUEE;
if ( m_scrape_marquees && ( !my_art.marquee.empty() )
&& ( !art_exists( path, emu_name ) ))
{
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee,
path + emu_name );
}
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string path_base = base_path + FANART + emu_name + "/";
confirm_directory( base_path, "" );
confirm_directory( base_path + FANART, emu_name );
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( path_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
path_base + (*itr).substr( start_pos+1,
end_pos-start_pos-1 ) );
}
}
}
}
}
}
bool prefer_alt_filename = c.emulator.is_mess();
//
// Build a map for looking up parents
//
ParentMapType parent_map;
build_parent_map( parent_map, c.romlist, prefer_alt_filename );
//
// Build a worklist of the roms where we need to lookup
//
std::vector<FeRomInfo *> worklist;
worklist.reserve( c.romlist.size() );
for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr )
{
(*itr).set_info( FeRomInfo::Emulator, emu_name );
// Don't scrape for a clone if its parent has the same name
//
if ( has_same_name_as_parent( *itr, parent_map ) )
continue;
if ( !c.scrape_art || m_scrape_fanart
|| ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) )
|| ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) )
|| ( m_scrape_snaps && (!has_image_artwork( *itr, "snap" ) ) )
|| ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) )
worklist.push_back( &(*itr) );
}
const int NUM_ARTS=5; // the number of scrape-able artwork types
int done_count( 0 );
//
// Set up our initial queue of network tasks
//
for ( unsigned int i=0; i<worklist.size(); i++ )
{
std::string req_string = GAME_REQ;
std::string game = url_escape(
name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) );
perform_substitution( req_string, "$1", game );
//
// If we don't need to scrape a wheel artwork, then add the specific platform to our request
// If we are scraping a wheel, we want to be able to grab them where the game name (but
// not the system) matches, so we don't limit ourselves by system...
//
if (( system_list.size() == 1 )
&& ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) ))
{
req_string += "&platform=";
req_string += url_escape( system_list.front() );
}
q.add_buffer_task( HOSTNAME, req_string, i );
}
//
// Create worker threads to process the queue, adding new tasks to download
// artwork files where appropriate
//
FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q );
std::string aux;
//
// Process the output queue from our worker threads
//
while ( !( q.input_done() && q.output_done() ) )
{
int id;
std::string result;
if ( q.pop_completed_task( id, result ) )
{
if ( id < 0 )
{
if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask ))
{
std::cout << " + Downloaded: " << result << std::endl;
c.download_count++;
// find second last forward slash in filename
// we assume that there will always be at least two
size_t pos = result.find_last_of( "\\/" );
if ( pos != std::string::npos )
{
pos = result.find_last_of( "\\/", pos-1 );
if ( pos != std::string::npos )
aux = result.substr( pos );
}
}
if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask
done_count++;
}
else
{
FeGameDBArt my_art;
FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) );
gdbp.parse( result );
if ( c.scrape_art && !my_art.base.empty() )
{
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
get_url_components( my_art.base,
hostn, base_req );
const FeRomInfo &rom = *(worklist[id]);
if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) )
{
std::string fname = base_path + FLYER;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer, fname );
}
else
done_count++;
if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) )
{
std::string fname = base_path + WHEEL;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel, fname );
}
else
done_count++;
if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) )
{
std::string fname = base_path + MARQUEE;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee, fname );
}
else
done_count++;
if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_image_artwork( rom, "snap" )) )
{
std::string fname = base_path + SNAP;
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
fname += rom.get_info( FeRomInfo::AltRomname );
else
fname += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, SNAP );
q.add_file_task( hostn, base_req + my_art.snap, fname );
}
else
done_count++;
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
std::string fname_base = base_path + FANART;
confirm_directory( base_path, "" );
const std::string &altname = rom.get_info( FeRomInfo::AltRomname );
if ( prefer_alt_filename && !altname.empty() )
{
fname_base += rom.get_info( FeRomInfo::AltRomname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::AltRomname ) );
}
else
{
fname_base += rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path + FANART,
rom.get_info( FeRomInfo::Romname ) );
}
fname_base += "/";
bool done_first=false; // we only count the first fanart against our percentage completed...
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ),
done_first );
done_first=true;
}
}
}
}
else
done_count++;
}
else
{
aux = (worklist[id])->get_info( FeRomInfo::Title );
done_count+=NUM_ARTS;
}
}
if (( c.uiupdate ) && !worklist.empty() )
{
int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() );
if ( c.uiupdate( c.uiupdatedata, p, aux ) == false )
return false;
}
}
else if ( q.output_done() )
{
sf::Http::Response::Status status;
std::string err_req;
q.do_next_task( status, err_req );
if ( status != sf::Http::Response::Ok )
{
std::cout << " * Error processing request. Status code: "
<< status << " (" << err_req << ")" << std::endl;
}
}
else
sf::sleep( sf::milliseconds( 10 ) );
}
#endif
return true;
}
void AdminPortalProtocol::admin_view_add_pidlink(const std::string& view, const NamedPIDLink& link) throw (P4PProtocolError)
{
check_txn();
make_request("PUT", "admin/" + admin_token_ + '/' + url_escape(view) + "/link/" + url_escape(link.name) + "?src=" + url_escape(link.link.src) + "&dst=" + url_escape(link.link.dst));
}
void AdminPortalProtocol::admin_view_del_pidlink(const std::string& view, const std::string& name) throw (P4PProtocolError)
{
check_txn();
make_request("DELETE", "admin/" + admin_token_ + '/' + url_escape(view) + "/link/" + url_escape(name));
}
void AdminPortalProtocol::admin_view_del_pid_node(const std::string& view, const std::string& pid, const std::string& node) throw (P4PProtocolError)
{
check_txn();
make_request("DELETE", "admin/" + admin_token_ + '/' + url_escape(view) + "/pid/" + url_escape(pid) + "/nodes/" + url_escape(node));
}
void AdminPortalProtocol::admin_view_add_pid(const std::string& view, const NamedPID& pid) throw (P4PProtocolError)
{
check_txn();
make_request("PUT", "admin/" + admin_token_ + '/' + url_escape(view) + "/pid/" + url_escape(pid.name) + "?v=" + p4p::detail::p4p_token_cast<std::string>(pid.pid));
}
void AdminPortalProtocol::admin_view_set_prop(const std::string& view, const std::string& name, const std::string& value) throw (P4PProtocolError)
{
check_txn();
make_request("PUT", "admin/" + admin_token_ + '/' + url_escape(view) + "/prop/" + url_escape(name) + "?v=" + url_escape(value));
}
static char *create_post_data(
/* PARAMETERS */
const char *formdata,
const char *user,
const char *password,
const char *realm
/* END PARAMETERS */
)
{
/* VARIABLES */
const char *ptr, *line_ptr;
char *esc_user = NULL, *esc_password = NULL, *esc_realm = NULL;
char *buf = NULL, *buf_ptr;
int filtersize;
int ulen, plen, rlen;
int numpercents=0;
int biggest;
size_t i;
/* END VARIABLES */
user = esc_user = url_escape(user);
password = esc_password = url_escape(password);
realm = esc_realm = url_escape(realm);
if (!user || !password || !realm) {
logger(LOG_ERR, "auth_httpform:create_post_data", "failed to allocate memory");
goto CLEANUP;
}
/* calculate memory needed for creating the complete query string. */
ulen = strlen(user);
plen = strlen(password);
rlen = strlen(realm);
/* what if we have multiple %foo occurrences in the input query? */
for (i = 0; i < strlen(formdata); i++) {
if (formdata[i] == '%') {
numpercents++;
}
}
/* find the biggest of ulen, plen */
biggest = MAX(MAX(ulen, plen), rlen);
/* don't forget the trailing 0x0 */
filtersize = strlen(formdata) + 1 + (numpercents*biggest)+1;
/* ok, now try to allocate a chunk of that size */
buf = (char *) malloc(filtersize);
if (!buf) {
logger(LOG_ERR, "auth_httpform:create_post_data", "failed to allocate memory");
goto CLEANUP;
}
buf_ptr = buf;
line_ptr = formdata;
/* replace the strings */
while ( (ptr = strchr(line_ptr, '%')) ) {
/* copy up to but not including the next % */
memcpy(buf_ptr, line_ptr, ptr - line_ptr);
buf_ptr += ptr - line_ptr;
ptr++;
switch (ptr[0]) {
case '%':
buf_ptr[0] = '%';
buf_ptr++;
break;
case 'u':
memcpy(buf_ptr, user, ulen);
buf_ptr += ulen;
break;
case 'p':
memcpy(buf_ptr, password, plen);
buf_ptr += plen;
break;
case 'r':
memcpy(buf_ptr, realm, rlen);
buf_ptr += rlen;
break;
default:
buf_ptr[0] = '%';
buf_ptr[1] = ptr[0];
buf_ptr += 2;
break;
}
ptr++;
line_ptr = ptr;
}
/* don't forget the rest */
memcpy(buf_ptr, line_ptr, strlen(line_ptr)+1);
CLEANUP:
if (esc_user) {
memset(esc_user, 0, strlen(esc_user));
free(esc_user);
}
if (esc_password) {
memset(esc_password, 0, strlen(esc_password));
free(esc_password);
}
if (esc_realm) {
memset(esc_realm, 0, strlen(realm));
free(esc_realm);
}
return buf;
}
void AdminPortalProtocol::admin_view_del_pid_prefix(const std::string& view, const std::string& name, const std::string& address, unsigned short len) throw (P4PProtocolError)
{
check_txn();
make_request("DELETE", "admin/" + admin_token_ + '/' + url_escape(view) + "/pid/" + url_escape(name) + "/prefixes/" + url_escape(address) + "?len=" + p4p::detail::p4p_token_cast<std::string>(len));
}
/**
* Writes the browse host data of the context ``ctx'' to the buffer
* ``dest''. This must be called multiple times to retrieve the complete
* data until zero is returned i.e., the end of file is reached.
*
* This routine deals with HTML data generation.
*
* @param ctx an initialized browse host context.
* @param dest the destination buffer.
* @param size the amount of bytes ``dest'' can hold.
*
* @return -1 on failure, zero at the end-of-file condition or if size
* was zero. On success, the amount of bytes copied to ``dest''
* is returned.
*/
static ssize_t
browse_host_read_html(struct special_upload *ctx,
void *const dest, size_t size)
{
static const char header[] =
"<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\r\n"
"<html>\r\n"
"<head>\r\n"
"<title>Browse Host</title>\r\n"
"</head>\r\n"
"<body>\r\n";
static const char trailer[] = "</ul>\r\n</body>\r\n</html>\r\n";
struct browse_host_upload *bh = cast_to_browse_host_upload(ctx);
char *p = dest;
g_assert(NULL != bh);
g_assert(NULL != dest);
g_assert(size <= INT_MAX);
g_assert(UNSIGNED(bh->state) < NUM_BH_STATES);
g_assert(bh->b_size <= INT_MAX);
g_assert(bh->b_offset <= bh->b_size);
do {
switch (bh->state) {
case BH_STATE_HEADER:
if (!bh->b_data) {
bh->b_data = header;
bh->b_size = CONST_STRLEN(header);
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_LIBRARY_INFO);
break;
case BH_STATE_LIBRARY_INFO:
if (!bh->b_data) {
bh->w_buf_size = w_concat_strings(&bh->w_buf,
"<h1>", product_get_name(), "</h1>\r\n"
"<h3>", version_get_string(),
" sharing ",
uint64_to_string(shared_files_scanned()),
" file",
plural(shared_files_scanned()),
" ",
short_kb_size(shared_kbytes_scanned(),
GNET_PROPERTY(display_metric_units)),
" total</h3>\r\n"
"<ul>\r\n", (void *) 0);
bh->b_data = bh->w_buf;
bh->b_size = bh->w_buf_size - 1; /* minus trailing NUL */
bh->b_offset = 0;
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_FILES);
break;
case BH_STATE_TRAILER:
if (!bh->b_data) {
bh->b_data = trailer;
bh->b_size = CONST_STRLEN(trailer);
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_EOF);
break;
case BH_STATE_FILES:
if (bh->b_data && bh->b_size == bh->b_offset) {
g_assert(bh->w_buf == bh->b_data);
wfree(bh->w_buf, bh->w_buf_size);
bh->w_buf = NULL;
bh->w_buf_size = 0;
bh->b_data = NULL;
}
if (!bh->b_data) {
shared_file_t *sf;
bh->file_index++;
sf = shared_file_sorted(bh->file_index);
if (!sf) {
if (bh->file_index > shared_files_scanned())
browse_host_next_state(bh, BH_STATE_TRAILER);
/* Skip holes in the file_index table */
} else if (SHARE_REBUILDING == sf) {
browse_host_next_state(bh, BH_STATE_REBUILDING);
} else {
const char * const name_nfc = shared_file_name_nfc(sf);
const filesize_t file_size = shared_file_size(sf);
size_t html_size;
char *html_name;
{
const char *dir;
char *name;
dir = shared_file_relative_path(sf);
if (dir) {
name = h_strconcat(dir, "/", name_nfc, (void *) 0);
} else {
name = deconstify_char(name_nfc);
}
html_size = 1 + html_escape(name, NULL, 0);
html_name = walloc(html_size);
html_escape(name, html_name, html_size);
if (name != name_nfc) {
HFREE_NULL(name);
}
}
if (sha1_hash_available(sf)) {
const struct sha1 *sha1 = shared_file_sha1(sf);
bh->w_buf_size = w_concat_strings(&bh->w_buf,
"<li><a href=\"/uri-res/N2R?urn:sha1:",
sha1_base32(sha1),
"\">", html_name, "</a> [",
short_html_size(file_size,
GNET_PROPERTY(display_metric_units)),
"]</li>\r\n",
(void *) 0);
} else {
char *escaped;
escaped = url_escape(name_nfc);
bh->w_buf_size = w_concat_strings(&bh->w_buf,
"<li><a href=\"/get/",
uint32_to_string(shared_file_index(sf)),
"/", escaped, "\">", html_name, "</a>"
" [",
short_html_size(file_size,
GNET_PROPERTY(display_metric_units)),
"]</li>\r\n", (void *) 0);
if (escaped != name_nfc) {
HFREE_NULL(escaped);
}
}
wfree(html_name, html_size);
bh->b_data = bh->w_buf;
bh->b_size = bh->w_buf_size - 1; /* minus trailing NUL */
bh->b_offset = 0;
}
shared_file_unref(&sf);
}
if (bh->b_data)
p += browse_host_read_data(bh, p, &size);
break;
case BH_STATE_REBUILDING:
if (!bh->b_data) {
static const char msg[] =
"<li>"
"<b>"
"The library is currently being rebuild. Please, "
"try again in a moment."
"</b>"
"</li>";
bh->b_data = msg;
bh->b_size = CONST_STRLEN(msg);
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_TRAILER);
break;
case BH_STATE_EOF:
return p - cast_to_char_ptr(dest);
case NUM_BH_STATES:
g_assert_not_reached();
}
} while (size > 0);
return p - cast_to_char_ptr(dest);
}
void AdminPortalProtocol::admin_view_clear_pid_prefixes(const std::string& view, const std::string& name) throw (P4PProtocolError)
{
check_txn();
make_request("DELETE", "admin/" + admin_token_ + '/' + url_escape(view) + "/pid/" + url_escape(name) + "/prefixes");
}
bool FeSettings::thegamesdb_scraper( FeImporterContext &c )
{
#ifndef NO_NET
const char *HOSTNAME = "http://thegamesdb.net";
const char *PLATFORM_REQ = "api/GetPlatformsList.php";
const char *GAME_REQ = "api/GetGame.php?name=$1";
//
// Get a list of valid platforms
//
FeNetQueue q;
q.add_buffer_task( HOSTNAME, PLATFORM_REQ, 0 );
sf::Http::Response::Status status;
q.do_next_task( status );
if ( status != sf::Http::Response::Ok )
{
get_resource( "Error getting platform list from thegamesdb.net. Code: $1",
as_str( status ), c.user_message );
std::cout << " * " << c.user_message << std::endl;
return true;
}
int temp;
std::string body;
q.pop_completed_task( temp, body );
FeGameDBPlatformParser gdbpp;
gdbpp.parse( body );
const std::vector<std::string> &sl_temp = c.emulator.get_systems();
std::vector<std::string> system_list;
for ( std::vector<std::string>::const_iterator itr = sl_temp.begin(); itr!=sl_temp.end(); ++itr )
{
if ( gdbpp.m_set.find( *itr ) != gdbpp.m_set.end() )
system_list.push_back( *itr );
else
std::cout << " * System identifier '" << (*itr) << "' not recognized by "
<< HOSTNAME << std::endl;
}
if ( system_list.size() < 1 )
{
// Correct if we can based on the configured info source,
// otherwise we error out
const std::string source = c.emulator.get_info( FeEmulatorInfo::Info_source );
if ( source.compare( "mame" ) == 0 )
system_list.push_back( "Arcade" );
else if ( source.compare( "steam" ) == 0 )
system_list.push_back( "PC" );
else
{
get_resource( "Error: None of the configured system identifier(s) are recognized by thegamesdb.net.",
c.user_message );
std::cout << " * " << c.user_message << std::endl;
return true;
}
}
std::string emu_name = c.emulator.get_info( FeEmulatorInfo::Name );
//
// Build a map for looking up parents
//
std::map < std::string, FeRomInfo * > parent_map;
build_parent_map( parent_map, c.romlist );
//
// Build a worklist of the roms where we need to lookup
//
std::vector<FeRomInfo *> worklist;
worklist.reserve( c.romlist.size() );
for ( FeRomInfoListType::iterator itr=c.romlist.begin(); itr!=c.romlist.end(); ++itr )
{
(*itr).set_info( FeRomInfo::Emulator, emu_name );
// Don't scrape for a clone if its parent has the same name
//
if ( has_same_name_as_parent( *itr, parent_map ) )
continue;
if ( !c.scrape_art || m_scrape_fanart
|| ( m_scrape_flyers && (!has_artwork( *itr, "flyer" ) ) )
|| ( m_scrape_wheels && (!has_artwork( *itr, "wheel" ) ) )
|| ( m_scrape_snaps && (!has_artwork( *itr, "snap" ) ) )
|| ( m_scrape_marquees && (!has_artwork( *itr, "marquee" ) ) ) )
worklist.push_back( &(*itr) );
}
const int NUM_ARTS=5; // the number of scrape-able artwork types
int done_count( 0 );
//
// Set up our initial queue of network tasks
//
for ( unsigned int i=0; i<worklist.size(); i++ )
{
std::string req_string = GAME_REQ;
std::string game = url_escape(
name_with_brackets_stripped( worklist[i]->get_info( FeRomInfo::Title ) ) );
perform_substitution( req_string, "$1", game );
//
// If we don't need to scrape a wheel artwork, then add the specific platform to our request
// If we are scraping a wheel, we want to be able to grab them where the game name (but
// not the system) matches, so we don't limit ourselves by system...
//
if (( system_list.size() == 1 )
&& ( !c.scrape_art || !m_scrape_wheels || has_artwork( *(worklist[i]), "wheel" ) ))
{
req_string += "&platform=";
req_string += url_escape( system_list.front() );
}
q.add_buffer_task( HOSTNAME, req_string, i );
}
std::string base_path = get_config_dir() + FE_SCRAPER_SUBDIR;
base_path += emu_name + "/";
//
// Create worker threads to process the queue, adding new tasks to download
// artwork files where appropriate
//
FeNetWorker worker1( q ), worker2( q ), worker3( q ), worker4( q );
//
// Process the output queue from our worker threads
//
while ( !q.input_done() || !q.output_done() )
{
int id;
std::string result;
if ( q.pop_completed_task( id, result ) )
{
if ( id < 0 )
{
if (( id == FeNetTask::FileTask ) || ( id == FeNetTask::SpecialFileTask ))
{
std::cout << " + Downloaded: " << result << std::endl;
c.download_count++;
}
if ( id == FeNetTask::FileTask ) // we don't increment if id = FeNetTask::SpecialFileTask
done_count++;
}
else
{
FeGameDBArt my_art;
FeGameDBParser gdbp( system_list, *(worklist[id]), ( c.scrape_art ? &my_art : NULL ) );
gdbp.parse( result );
if ( c.scrape_art && !my_art.base.empty() )
{
std::string hostn = HOSTNAME;
std::string base_req = "banners/";
size_t pos=0;
for ( int i=0; i<3 && ( pos != std::string::npos ); i++ )
pos = my_art.base.find_first_of( '/', pos+1 );
if (( pos != std::string::npos ) && ( pos < my_art.base.size()-1 ) )
{
hostn = my_art.base.substr( 0, pos+1 );
base_req = my_art.base.substr( pos+1 );
}
FeRomInfo &rom = *(worklist[id]);
if ( m_scrape_flyers && ( !my_art.flyer.empty() ) && (!has_artwork( rom, "flyer" )) )
{
const char *FLYER = "flyer/";
std::string fname = base_path + FLYER + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, FLYER );
q.add_file_task( hostn, base_req + my_art.flyer, fname );
}
else
done_count++;
if ( m_scrape_wheels && ( !my_art.wheel.empty() ) && (!has_artwork( rom, "wheel" )) )
{
const char *WHEEL = "wheel/";
std::string fname = base_path + WHEEL + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, WHEEL );
q.add_file_task( hostn, base_req + my_art.wheel, fname );
}
else
done_count++;
if ( m_scrape_marquees && (!my_art.marquee.empty() ) && (!has_artwork( rom, "marquee" )) )
{
const char *MARQUEE = "marquee/";
std::string fname = base_path + MARQUEE + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, MARQUEE );
q.add_file_task( hostn, base_req + my_art.marquee, fname );
}
else
done_count++;
if ( m_scrape_snaps && (!my_art.snap.empty() ) && (!has_artwork( rom, "snap" )) )
{
const char *SNAP = "snap/";
std::string fname = base_path + SNAP + rom.get_info( FeRomInfo::Romname );
confirm_directory( base_path, SNAP );
q.add_file_task( hostn, base_req + my_art.snap, fname );
}
else
done_count++;
if ( m_scrape_fanart && !my_art.fanart.empty() )
{
const char *FANART = "fanart/";
std::string fname_base = base_path + FANART + rom.get_info( FeRomInfo::Romname ) + "/";
confirm_directory( base_path, "" );
confirm_directory( base_path + FANART, rom.get_info( FeRomInfo::Romname ) );
bool done_first=false; // we only count the first fanart against our percentage completed...
for ( std::vector<std::string>::iterator itr = my_art.fanart.begin();
itr != my_art.fanart.end(); ++itr )
{
size_t start_pos = (*itr).find_last_of( "/\\" );
size_t end_pos = (*itr).find_last_of( '.' );
if (( start_pos != std::string::npos )
&& ( !file_exists( fname_base + (*itr).substr( start_pos+1 ) ) ))
{
if (( end_pos != std::string::npos ) && ( end_pos > start_pos ))
{
q.add_file_task( hostn,
base_req + (*itr),
fname_base + (*itr).substr( start_pos+1, end_pos-start_pos-1 ),
done_first );
done_first=true;
}
}
}
}
else
done_count++;
}
else
done_count+=NUM_ARTS;
}
if ( c.uiupdate )
{
int p = c.progress_past + done_count * c.progress_range / ( NUM_ARTS * worklist.size() );
if ( c.uiupdate( c.uiupdatedata, p ) == false )
return false;
}
}
else if ( q.output_done() )
{
sf::Http::Response::Status status;
q.do_next_task( status );
}
else
sf::sleep( sf::milliseconds( 10 ) );
}
#endif
return true;
}
/* RESTAuth request dispatcher */
static int pam_restauth_check(
const char *base_url,
const char *service_user,
const char *service_password,
const char *group,
int validate_certificate,
const char *user,
const char *password) {
/* allocate structures */
CURL *session = curl_easy_init();
char *escaped_user = url_escape(user);
char *escaped_password = url_escape(password);
char *url = malloc(strlen(base_url)+strlen("/users/")+strlen(user)*3+1+1);
char *post_data = malloc(strlen("password="******"%s%susers/%s/", base_url,
*(base_url+strlen(base_url)-1) == '/' ? "":"/",
escaped_user);
/* create POST data: password=<password> */
sprintf(post_data, "password=%s", escaped_password);
/* set up CURL request */
pam_restauth_curl_session_init(session, url,
service_user, service_password,
validate_certificate, post_data);
/* perform request */
long curl_http_code = 0;
int curl_status = curl_easy_perform(session);
curl_status += curl_easy_getinfo(session, CURLINFO_RESPONSE_CODE,
&curl_http_code);
if (curl_status == CURLE_OK && curl_http_code >= 200 && curl_http_code < 300) {
/* success. perform group check if a group is specified */
if (group) {
/* create group URL */
char *escaped_group = url_escape(group);
char *group_url = malloc(strlen(base_url)+strlen("/groups/")+strlen(escaped_group)+strlen("/users/")+strlen(escaped_user)+1);
sprintf(group_url, "%s%sgroups/%s/users/%s/", base_url,
*(base_url+strlen(base_url)-1) == '/' ? "":"/",
escaped_group, escaped_user);
/* set up CURL request again and perform call */
pam_restauth_curl_session_init(session, group_url,
service_user, service_password,
validate_certificate, NULL);
curl_status = curl_easy_perform(session);
curl_status += curl_easy_getinfo(session, CURLINFO_RESPONSE_CODE,
&curl_http_code);
if (curl_status == CURLE_OK && curl_http_code >= 200 && curl_http_code < 300) {
ret = 0; /* success */
}
else {
ret = -1; /* failure */
syslog(LOG_AUTHPRIV|LOG_NOTICE, __FILE__": user `%s' failed RESTAuth check for group `%s'", escaped_user, escaped_group);
}
if (escaped_group) free(escaped_group);
if (group_url) free(group_url);
}
else {
ret = 0; /* success */
}
}
else {
ret = -1; /* failure */
}
cleanup:
if (session) curl_easy_cleanup(session);
if (escaped_user) free(escaped_user);
if (escaped_password) free(escaped_password);
if (url) free(url);
if (post_data) free(post_data);
return ret;
}