static int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, char *column)
{
unsigned char blob_value[r->source_record_size*2+1];
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("populating with sql rows at offset %d",r->buffer_length);
if (r->source_index>=r->source_count)
{
/* All done */
return 0;
}
int record_count=(r->buffer_size-r->buffer_length)/r->source_record_size;
if (record_count<1) {
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("r->buffer_size=%d, r->buffer_length=%d, r->source_record_size=%d",
r->buffer_size, r->buffer_length, r->source_record_size);
return WHY("Not enough space to fit any records");
}
sqlite3_stmt *statement = sqlite_prepare("%s LIMIT %lld,%d", r->source, r->source_index, record_count);
if (!statement)
return -1;
if (debug & DEBUG_RHIZOME_TX)
DEBUG(sqlite3_sql(statement));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
while( r->buffer_length + r->source_record_size < r->buffer_size
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW
) {
r->source_index++;
if (sqlite3_column_count(statement)!=2) {
sqlite3_finalize(statement);
return WHY("sqlite3 returned multiple columns for a single column query");
}
sqlite3_blob *blob;
const unsigned char *value;
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_TEXT: value=sqlite3_column_text(statement, 0); break;
case SQLITE_BLOB:
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("table='%s',col='%s',rowid=%lld", table, column, sqlite3_column_int64(statement,1));
int ret;
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", table, column, rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
continue;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
if (sqlite3_blob_read(blob,&blob_value[0],
/* copy number of bytes based on whether we need to
de-hex the string or not */
r->source_record_size*(1+(r->source_flags&1)),0)
!=SQLITE_OK) {
WHYF("sqlite3_blob_read() failed, %s", sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob);
continue;
}
value = blob_value;
sqlite3_blob_close(blob);
break;
default:
/* improper column type, so don't include in report */
WHYF("Bad column type %d", column_type);
continue;
}
if (r->source_flags&1) {
/* hex string to be converted */
int i;
for(i=0;i<r->source_record_size;i++)
/* convert the two nybls and make a byte */
r->buffer[r->buffer_length+i]
=(hexvalue(value[i<<1])<<4)|hexvalue(value[(i<<1)+1]);
} else
/* direct binary value */
bcopy(value,&r->buffer[r->buffer_length],r->source_record_size);
r->buffer_length+=r->source_record_size;
}
sqlite3_finalize(statement);
return 0;
}
/* Read upto the <bars_requested> next BARs from the Rhizome database,
beginning from the first BAR that corresponds to a manifest with
BID>=<bid_low>.
Sets <bid_high> to the highest BID for which a BAR was returned.
Return value is the number of BARs written into <bars_out>.
Only returns BARs for bundles within the specified size range.
This is used by the cursor wrapper function that passes over all of the
BARs in prioritised order.
XXX Once the rhizome database gets big, we will need to make sure
that we have suitable indexes. It is tempting to just pack BARs
by row_id, but the far end needs them in an orderly manner so that
it is possible to make provably complete comparison of the contents
of the respective rhizome databases.
*/
int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES],
unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES],
long long size_low,long long size_high,
const unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES],
unsigned char *bars_out,
int bars_requested)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
char query[1024];
snprintf(query,1024,
"SELECT BAR,ROWID,ID,FILESIZE FROM MANIFESTS"
" WHERE"
" FILESIZE BETWEEN %lld AND %lld"
" AND ID>='%s' AND ID<='%s'"
// The following formulation doesn't remove the weird returning of
// bundles with out of range filesize values
// " WHERE ID>='%s' AND ID<='%s' AND FILESIZE > %lld AND FILESIZE < %lld"
" ORDER BY BAR LIMIT %d;",
size_low, size_high,
alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES),
alloca_tohex(bid_max,RHIZOME_MANIFEST_ID_BYTES),
bars_requested);
sqlite3_stmt *statement=sqlite_prepare(&retry, query);
sqlite3_blob *blob=NULL;
int bars_written=0;
while(bars_written<bars_requested
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW)
{
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_BLOB:
if (blob)
sqlite3_blob_close(blob);
blob = NULL;
int ret;
int64_t filesize = sqlite3_column_int64(statement, 3);
if (filesize<size_low||filesize>size_high) {
DEBUGF("WEIRDNESS ALERT: filesize=%lld, but query was: %s",
filesize,query);
break;
}
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", "manifests", "bar",
rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
continue;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
int blob_bytes=sqlite3_blob_bytes(blob);
if (blob_bytes!=RHIZOME_BAR_BYTES) {
if (config.debug.rhizome)
DEBUG("Found a BAR that is the wrong size - ignoring");
sqlite3_blob_close(blob);
blob=NULL;
continue;
}
sqlite3_blob_read(blob,&bars_out[bars_written*RHIZOME_BAR_BYTES],
RHIZOME_BAR_BYTES,0);
sqlite3_blob_close(blob);
blob=NULL;
/* Remember the BID so that we cant write it into bid_high so that the
caller knows how far we got. */
fromhex(bid_high,
(const char *)sqlite3_column_text(statement, 2),
RHIZOME_MANIFEST_ID_BYTES);
bars_written++;
break;
default:
/* non-BLOB field. This is an error, but we will persevere with subsequent
rows, because they might be fine. */
break;
}
}
if (statement)
sqlite3_finalize(statement);
statement = NULL;
return bars_written;
}
int main( int argc, char** argv ) {
google::ParseCommandLineFlags( &argc, &argv, true);
google::InitGoogleLogging( argv[0] );
FLAGS_logtostderr = 1;
FLAGS_v = 1;
LOG(INFO) << "Starting!";
OTF_FileManager * manager = OTF_FileManager_open( 100 ); // limit to 100 filehandles
CHECK_NOTNULL( manager );
CHECK_NE( FLAGS_otf, "" ) << "Please specify input OTF file";
reader = OTF_Reader_open( FLAGS_otf.c_str(), manager );
CHECK_NOTNULL( reader );
OTF_Reader_setBufferSizes( reader, FLAGS_buffer_size );
if( FLAGS_db != "" ) {
auto retval = sqlite3_open( FLAGS_db.c_str(), &db );
if( retval ) {
LOG(FATAL) << "Can't open db " << FLAGS_db << ": " << sqlite3_errmsg(db);
}
std::stringstream ss;
ss << "CREATE TABLE IF NOT EXISTS " << FLAGS_table << "("
<< " TIME INT8 PRIMARY KEY NOT NULL,"
<< " COUNTER_NAME TEXT,"
<< " PROCESS INT,"
<< " INT_VALUE INT8,"
<< " DOUBLE_VALUE REAL);";
sqlite_exec( db, ss.str().c_str(), NULL, NULL );
sqlite_exec(db, "BEGIN TRANSACTION", NULL, NULL);
std::stringstream ss2;
ss2 << "INSERT OR IGNORE INTO " << FLAGS_table
<< " VALUES (?, ?, ?, ?, ?);";
sqlite_prepare( db, ss2.str().c_str(), -1, &stmt );
}
if( -1 != FLAGS_max_time ) {
OTF_Reader_setTimeInterval( reader, FLAGS_min_time, FLAGS_max_time );
}
OTF_HandlerArray * handlers = OTF_HandlerArray_open();
CHECK_NOTNULL( handlers );
// register handlers to do whatever you want.
// many handlers can be registered; full list here:
// http://wwwpub.zih.tu-dresden.de/~jurenz/otf/api/current/OTF__Definitions_8h.html
register_otf_handler( handlers, OTF_DEFCOUNTER_RECORD, defcounter_handler );
register_otf_handler( handlers, OTF_COUNTER_RECORD, counter_handler );
register_otf_handler( handlers, OTF_DEFTIMERRESOLUTION_RECORD, handle_time_resolution );
uint64_t read;
read = OTF_Reader_readDefinitions( reader, handlers );
LOG(INFO) << "Read " << read << " definitions";
read = OTF_Reader_readEvents( reader, handlers );
LOG(INFO) << "Read " << read << " events";
read = OTF_Reader_readStatistics( reader, handlers );
LOG(INFO) << "Read " << read << " statistics";
read = OTF_Reader_readSnapshots( reader, handlers );
LOG(INFO) << "Read " << read << " snapshots";
read = OTF_Reader_readMarkers( reader, handlers );
LOG(INFO) << "Read " << read << " markers";
sqlite_exec(db, "COMMIT TRANSACTION", NULL, NULL);
LOG(INFO) << "Done.";
if( stmt ) sqlite3_finalize( stmt );
if( db ) sqlite3_close( db );
OTF_Reader_close( reader );
OTF_HandlerArray_close( handlers );
OTF_FileManager_close( manager );
return 0;
}
rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix,int prefix_length)
{
/* Give a BID prefix, e.g., from a BAR, find the matching manifest and return it.
Of course, it is possible that more than one manifest matches. This should
occur only very rarely (with the possible exception of intentional attack, and
even then a 64-bit prefix creates a reasonable barrier. If we move to a new
BAR format with 120 or 128 bits of BID prefix, then we should be safe for some
time, thus this function taking the BID prefix as an input in preparation for
that change).
Of course, we need to be able to find the manifest.
Easiest way is to select with a BID range. We could instead have an extra
database column with the prefix.
*/
assert(prefix_length>=0);
assert(prefix_length<=RHIZOME_MANIFEST_ID_BYTES);
unsigned char low[RHIZOME_MANIFEST_ID_BYTES];
unsigned char high[RHIZOME_MANIFEST_ID_BYTES];
memset(low,0x00,RHIZOME_MANIFEST_ID_BYTES);
memset(high,0xff,RHIZOME_MANIFEST_ID_BYTES);
bcopy(bid_prefix,low,prefix_length);
bcopy(bid_prefix,high,prefix_length);
char query[1024];
snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS WHERE ID>='%s' AND ID<='%s'",
alloca_tohex(low,RHIZOME_MANIFEST_ID_BYTES),
alloca_tohex(high,RHIZOME_MANIFEST_ID_BYTES));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, query);
sqlite3_blob *blob=NULL;
if (sqlite_step_retry(&retry, statement) == SQLITE_ROW)
{
int ret;
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", "manifests", "bar",
rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
return NULL;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
/* Read manifest data from blob */
size_t manifestblobsize = sqlite3_column_bytes(statement, 0);
if (manifestblobsize<1||manifestblobsize>1024) goto error;
const char *manifestblob = (char *) sqlite3_column_blob(statement, 0);
if (!manifestblob) goto error;
rhizome_manifest *m=rhizome_new_manifest();
if (rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1)
{
rhizome_manifest_free(m);
goto error;
}
DEBUGF("Read manifest");
sqlite3_blob_close(blob);
sqlite3_finalize(statement);
return m;
error:
sqlite3_blob_close(blob);
sqlite3_finalize(statement);
return NULL;
}
else
{
DEBUGF("no matching manifests");
sqlite3_finalize(statement);
return NULL;
}
}
