static bool purge_buckets( bool force )
{
bool ok=true;
bool required=force;
for( int i=0; !required && i<NBR_BUCKETS; i++ )
{
std::vector<std::pair<int,int>> *bucket = &buckets[i];
int count = bucket->size();
if( count > 0 )
required = true;
}
if( required )
{
ProgressBar pb( create_mode ? "Creating database, step 3 of 4" : "Adding games to database, step 2 of 2", "Writing to database" , window_parent );
for( int i=0; ok && i<NBR_BUCKETS; i++ )
{
ok = purge_bucket(i);
if( ok )
{
if( i%10 == 0 )
cprintf( "Purging bucket %d\n", i );
if( pb.Permill( (i*1000) / NBR_BUCKETS ) )
{
error_msg = "cancel";
ok = false;
}
}
}
}
return ok;
}
static void purge_buckets()
{
for( int i=0; i<NBR_BUCKETS; i++ )
{
purge_bucket(i);
}
}
// returns bool inserted
bool db_primitive_insert_game( bool &signal_error, const char *white, const char *black, const char *event, const char *site, const char *round, const char *result,
const char *date, const char *white_elo, const char *black_elo, const char *eco,
int nbr_moves, thc::Move *moves, uint64_t *hashes )
{
signal_error = false; // use this mechanism because returning false doesn't necessarily mean error
char *errmsg;
char insert_buf[2000];
char blob_txt_buf[4000]; // about 500 moves each
char blob_buf[2000];
char white_buf[200];
char black_buf[200];
char event_buf[200];
char site_buf[200];
char round_buf[200];
char eco_buf[200];
if( nbr_moves < 3 ) // skip 'games' with zero, one or two moves
return false; // not inserted, not error
if( white_elo && black_elo )
{
int elo_w = atoi(white_elo);
int elo_b = atoi(black_elo);
if( 0<elo_w && elo_w<2000 && 0<elo_b && elo_b<2000 )
// if any elo information, need at least one good player
return false; // not inserted, not error
}
bool alphaWhite = sanitise( white, white_buf, sizeof(white_buf) );
bool alphaBlack = sanitise( black, black_buf, sizeof(black_buf) );
if( !alphaWhite || !alphaBlack ) // skip games with no names
return false; // not inserted, not error
sanitise( event, event_buf, sizeof(event_buf) );
sanitise( site, site_buf, sizeof(site_buf) );
sanitise( round, round_buf, sizeof(round_buf) );
sanitise( eco, eco_buf, sizeof(eco_buf) );
CompressMoves press;
uint64_t hash = press.cr.Hash64Calculate();
uint64_t counter=0, game_hash=hash;
char *put_txt = blob_txt_buf;
char *put = blob_buf;
for( int i=0; i<nbr_moves && put_txt<blob_txt_buf+sizeof(blob_txt_buf)-10; i++ )
{
thc::Move mv = moves[i];
char c = press.CompressMove(mv);
*put++ = c;
char hi = (c>>4)&0x0f;
*put_txt++ = (hi>=10 ? hi-10+'A' : hi+'0');
char lo = c&0x0f;
*put_txt++ = (lo>=10 ? lo-10+'A' : lo+'0');
hash = press.cr.Hash64Update( hash, mv );
game_hash = game_hash ^ hash ^ counter;
counter++;
}
*put_txt = '\0';
*put = '\0';
// Try to insert the game
sprintf( insert_buf, "INSERT INTO games VALUES(%d,%lld,'%s','%s','%s','%s','%s','%s','%s','%s','%s','%s',X'%s')",
game_id, game_hash, white_buf, black_buf, event_buf, site_buf, round_buf, result,
date, white_elo, black_elo, eco_buf, blob_txt_buf );
int retval = sqlite3_exec( handle, insert_buf,0,0,&errmsg);
// This is one spot where a database failure is (kind of) expected - the game_hash has "unique" property
if( retval && errmsg )
{
char buf[200];
strncpy( buf, errmsg, sizeof(buf)-1 );
buf[ sizeof(buf)-1 ] = '\0';
char *p = buf;
while( *p )
{
if( isalpha(*p) && isupper(*p) )
*p = tolower(*p);
p++;
}
// If it wasn't a "unique" issue, we bail out
if( !strstr(buf,"unique") )
{
char buf2[1000];
sprintf( buf2, "Database error: db_primitive_insert_game() 1 %s (%s)\n", errmsg, insert_buf );
error_msg = buf2;
signal_error = true;
return false; // not inserted, error
}
//cprintf("Non unique game hash %s (%s)\n", errmsg, insert_buf );
// Otherwise see if the game is a duplicate of existing game (if it's not, then it will be the same
// game played on a different occasion)
bool is_duplicate = db_primitive_check_for_duplicate( signal_error, game_hash, white_buf, black_buf, result, blob_buf );
if( signal_error )
return false; // not inserted, error
// A duplicate game is simply discarded
if( is_duplicate )
{
//cprintf( "*** DUPLICATE GAME DISCARDED\n" );
// ZOMBIE bug fix - return here, don't add bogus moves and increment game count
//db_temporary_hack( white, black, event, site, round, result,
// date, white_elo, black_elo, eco,
// nbr_moves, moves );
return false; // not inserted, not error
}
// If it's the same game played by different players, save it with game_hash NULL, which is doesn't have to
// be unique fortunately
else
{
//cprintf( "*** DUPLICATE GAME: %s-%s %s %s %s\n", white_buf, black_buf, event_buf, site_buf, result, date );
sprintf( insert_buf, "INSERT INTO games VALUES(%d,NULL,'%s','%s','%s','%s','%s','%s','%s','%s','%s','%s',X'%s')",
game_id, white_buf, black_buf, event_buf, site_buf, round_buf, result,
date, white_elo, black_elo, eco_buf, blob_txt_buf );
int retval = sqlite3_exec( handle, insert_buf,0,0,&errmsg);
if( retval && errmsg )
{
char buf2[1000];
sprintf( buf2, "Database error: db_primitive_insert_game() 2 %s (%s)", errmsg, insert_buf );
signal_error = true;
return false; // not inserted, error
}
else if( retval )
{
char buf2[1000];
sprintf( buf2, "Database error: db_primitive_insert_game() 2 (%s)", insert_buf );
signal_error = true;
return false; // not inserted, error
}
sprintf( insert_buf, "INSERT INTO games_duplicates VALUES(%d,%lld,'%s','%s','%s','%s','%s','%s','%s','%s','%s','%s',X'%s')",
game_id, game_hash, white_buf, black_buf, event_buf, site_buf, round_buf, result,
date, white_elo, black_elo, eco_buf, blob_txt_buf );
retval = sqlite3_exec( handle, insert_buf,0,0,&errmsg);
if( retval && errmsg )
{
char buf2[1000];
sprintf( buf2, "Database error: db_primitive_insert_game() 3 %s (%s)", errmsg, insert_buf );
signal_error = true;
return false; // not inserted, error
}
else if( retval )
{
char buf2[1000];
sprintf( buf2, "Database error: db_primitive_insert_game() 3 (%s)", insert_buf );
signal_error = true;
return false; // not inserted, error
}
}
}
// Save the position hashes
for( int i=0; i<nbr_moves; i++ )
{
uint64_t hash64 = *hashes++;
int hash32 = (int)(hash64);
int table_nbr = ((int)(hash64>>32))&(NBR_BUCKETS-1);
std::vector<std::pair<int,int>> *bucket = &buckets[table_nbr];
std::pair<int,int> duo(hash32,game_id);
bucket->push_back(duo);
int count = bucket->size();
if( count >= PURGE_QUOTA )
{
bool ok = purge_bucket(table_nbr);
if( !ok )
{
signal_error = true;
return false; // not inserted, error
}
}
}
game_id++;
return true; // inserted, no error
}
void db_primitive_insert_game_multi( const char *white, const char *black, const char *event, const char *site, const char *result, int nbr_moves, thc::Move *moves, uint64_t *hashes )
{
//printf( "db_primitive_gameover(%s,%s)\n", white, black );
//uint32_t *move_ptr = (uint32_t *)moves;
char *errmsg;
char insert_buf[2000];
char blob_buf[1000]; // about 500 moves each
char white_buf[200];
char black_buf[200];
strcpy( white_buf, white );
char *s=white_buf;
while( *s )
{
if( !isascii(*s) )
*s = '_';
else if( *s!=' ' && *s!='.' && *s!=',' && !isalnum(*s) )
*s = '_';
s++;
}
strcpy( black_buf, black );
s=black_buf;
while( *s )
{
if( !isascii(*s) )
*s = '_';
else if( *s!=' ' && *s!='.' && *s!=',' && !isalnum(*s) )
*s = '_';
s++;
}
CompressMoves press;
char *put = blob_buf;
for( int i=0; i<nbr_moves && put<blob_buf+sizeof(blob_buf)-10; i++ )
{
char buf[2];
thc::Move mv = moves[i];
int nbr = press.compress_move( mv, buf );
/* std::string s = mv.TerseOut();
if( nbr == 1 )
printf( "compress temp> %s -> %02x\n", s.c_str(), buf[0]&0xff );
else if( nbr == 2 )
printf( "compress temp> %s -> %02x,%02x\n", s.c_str(), buf[0]&0xff, buf[1]&0xff );
else
printf( "nbr == 0 ?\n"); */
if( nbr == 0 )
break;
for( int j=0; j<nbr && j<2; j++ )
{
char c = buf[j];
char hi = (c>>4)&0x0f;
*put++ = (hi>=10 ? hi-10+'A' : hi+'0');
char lo = c&0x0f;
*put++ = (lo>=10 ? lo-10+'A' : lo+'0');
}
//printf( "%s %s\n", (i<nbr_moves?"true":"false"), (put<blob_buf+sizeof(blob_buf-10) ? "true" ? "false") );
}
*put = '\0';
//printf( "%d %s\n", nbr_moves, blob_buf );
sprintf( insert_buf, "INSERT INTO games VALUES(%d,'%s','%s','%s',X'%s')", game_id, white_buf, black_buf, result, blob_buf );
//printf( "%s\n", insert_buf );
int retval = sqlite3_exec( handle, insert_buf,0,0,&errmsg);
if( retval )
{
printf("sqlite3_exec(INSERT 1) FAILED %s\n", errmsg );
}
for( int i=0; i<nbr_moves; i++ )
{
uint64_t hash64 = *hashes++;
int hash32 = (int)(hash64);
int table_nbr = ((int)(hash64>>32))&(NBR_BUCKETS-1);
std::vector<std::pair<int,int>> *bucket = &buckets[table_nbr];
std::pair<int,int> duo(hash32,game_id);
bucket->push_back(duo);
int count = bucket->size();
if( count >= PURGE_QUOTA )
purge_bucket(table_nbr);
}
game_id++;
}
