void lsamp_add_file(lsamp_data *ls, const char *filename, const char *keyword) {
sqlite3_stmt *pStmt;
int rc;
int rowid;
int count;
int bufsize = LSBUFSIZE;
int blob_offset = 0;
if(!lsamp_open_sfile(ls, filename)) {
return;
}
rc = sqlite3_prepare_v2(ls->db,
"INSERT INTO lsamp(keyword, data) VALUES(?,ZEROBLOB(?));",
-1,
&pStmt,
NULL);
if(rc) {
fprintf(stderr, "SQL Error: %s\n", sqlite3_errmsg(ls->db));
return;
}
sqlite3_bind_text(pStmt, 1, keyword, -1, SQLITE_STATIC);
sqlite3_bind_int(pStmt, 2, ls->sfinfo.frames * sizeof(LSFLOAT));
rc = sqlite3_step(pStmt);
rc = sqlite3_finalize(pStmt);
sqlite3_blob *pBlob;
rc = sqlite3_blob_open(ls->db, "main", "lsamp", "data",
sqlite3_last_insert_rowid(ls->db), 1, &pBlob);
if(rc) {
fprintf(stderr, "Error opening blob: %s\n", sqlite3_errmsg(ls->db));
return;
}
count = ls->sfinfo.frames;
while(count != 0) {
#ifdef LS_DEBUG
fprintf(stderr, "Reading buffersize %d.\n", bufsize);
fflush(stderr);
#endif
bufsize = sf_read_float(ls->sfile, ls->buf, bufsize);
rc = sqlite3_blob_write(pBlob, ls->buf,
bufsize * sizeof(LSFLOAT), blob_offset * sizeof(LSFLOAT));
#ifdef LS_DEBUG
fprintf(stderr, "%d bytes Written. Writing %d bytes.\n",
blob_offset, bufsize);
fflush(stderr);
#endif
if(rc) {
fprintf(stderr, "SQL Error: %s\n", sqlite3_errmsg(ls->db));
return;
}
blob_offset += bufsize;
count -= bufsize;
}
lsamp_close_sfile(ls);
sqlite3_blob_close(pBlob);
}
void SQLiteBlob::OpenBlob(SQLiteDatabase *db, std::string symbolicDatabaseName, std::string tableName, std::string columnName, int64 rowId, BLOB_ACCESS_MODE accessMode)
{
if(mBlobHandle != 0)
CloseBlob();
mDatabase = db;
if(sqlite3_blob_open(mDatabase->GetDatabaseHandle(), symbolicDatabaseName.c_str(), tableName.c_str(), columnName.c_str(), rowId, accessMode, &mBlobHandle) != SQLITE_OK)
KOMPEX_EXCEPT(sqlite3_errmsg(mDatabase->GetDatabaseHandle()), sqlite3_errcode(mDatabase->GetDatabaseHandle()));
}
sqlite3_blob* BlobLocator::openBlob(const char* dbName, const char* tblName, const char* colName, sqlite3_int64 rowID, bool readonly, bool throwEx)
{
sqlite3* db = _database->getPeer();
sqlite3_blob* blob = NULL;
int err = sqlite3_blob_open(db, dbName, tblName, colName, rowID, readonly ? 0 : 1, &blob);
if (err && throwEx)
{
String msg = sqlite3_errmsg(db);
NIT_THROW_FMT(EX_DATABASE, "can't open blob '%s.%s.%s[%lld]': %s", dbName, tblName, colName, rowID, msg.c_str());
}
return blob;
}
QSharedPointer<SQLiteIO> SQLiteIO::openBlob(sqlite3 *pDb, const QString &db, const QString &table, const QString &column, int rowid, int &rc)
{
sqlite3_blob *pBlob;
rc = sqlite3_blob_open(pDb,
db.toStdString().c_str(),
table.toStdString().c_str(),
column.toStdString().c_str(),
rowid,
1,
&pBlob);
if (rc == SQLITE_OK) {
return QSharedPointer<SQLiteIO>(new SQLiteIO(pBlob));
}
return QSharedPointer<SQLiteIO>(0);
}
error blob::select(session& s,
char const* ztable,
char const* zcolumn,
rowid id,
bool readandwrite) noexcept
{
close();
m_session = &s;
int result = sqlite3_blob_open(s.get_connection(),
"main",
ztable,
zcolumn,
id,
readandwrite ? 1 : 0,
&m_blob);
return detail::sqliteerror(__file__, __line__, result);
}
Error blob::select (session& s,
char const* zTable,
char const* zColumn,
rowid id,
bool readAndWrite) noexcept
{
close();
m_session = &s;
int result = sqlite3_blob_open (s.get_connection(),
"main",
zTable,
zColumn,
id,
readAndWrite ? 1 : 0,
&m_blob );
return detail::sqliteError (__FILE__, __LINE__, result);
}
int lsamp_extract_info(lsamp_data *ls, lsamp_handle *lh, uint32_t rowid) {
/* is this function needed, or are we okay with memset?
lsamp_zero_handle(lp);
*/
lh->rowid = rowid;
lh->total_frames = 0;
lh->frmpos = 0;
int rc = sqlite3_blob_open(ls->db, "main", "lsamp", "data",
rowid, 0, &lh->blob);
if(rc) {
fprintf(stderr, "lsamp_open_handle() Error: %s\n", sqlite3_errmsg(ls->db));
return 0;
}
lh->total_frames = sqlite3_blob_bytes(lh->blob) / sizeof(LSFLOAT);
#ifdef LS_DEBUG
fprintf(stderr, "lsamp_open_handle: lh->total_frames is %d.\n", lh->total_frames);
fflush(stderr);
#endif
return 1;
}
/** Create a trigram expression for matching against a single trigram */
int exprTrigram(expr **ppExpr, trilite_vtab *pTrgVtab, trilite_trigram trigram){
int rc = SQLITE_OK;
sqlite3_blob *pBlob;
char *zTable = sqlite3_mprintf("%s_index", pTrgVtab->zName);
/* Open the blob */
rc = sqlite3_blob_open(pTrgVtab->db, pTrgVtab->zDb, zTable, "doclist", trigram, 0, &pBlob);
sqlite3_free(zTable);
/* If we didn't get a blob */
if(rc != SQLITE_OK){
*ppExpr = NULL;
return SQLITE_OK;
}
/* Get size of blob */
int nSize = sqlite3_blob_bytes(pBlob);
/* Allocate space for expr and doclist at the same time */
*ppExpr = (expr*)sqlite3_malloc(sizeof(expr) + nSize);
/* Set the expr */
(*ppExpr)->eType = EXPR_TRIGRAM;
(*ppExpr)->expr.trigram.docList = ((unsigned char*)(*ppExpr)) + sizeof(expr);
(*ppExpr)->expr.trigram.nSize = nSize;
/* Read doclist into memory */
sqlite3_blob_read(pBlob, (*ppExpr)->expr.trigram.docList, nSize, 0);
/* Release blob */
sqlite3_blob_close(pBlob);
/* Read first id */
int read = readVarInt((*ppExpr)->expr.trigram.docList, &(*ppExpr)->expr.trigram.curId);
(*ppExpr)->expr.trigram.curId += DELTA_LIST_OFFSET;
(*ppExpr)->expr.trigram.nSize -= read;
(*ppExpr)->expr.trigram.docList += read;
return rc;
}
static int rhizome_server_parse_http_request(rhizome_http_request *r)
{
/* Switching to writing, so update the call-back */
r->alarm.poll.events=POLLOUT;
watch(&r->alarm);
// Start building up a response.
r->request_type = 0;
// Parse the HTTP "GET" line.
char *path = NULL;
size_t pathlen = 0;
if (str_startswith(r->request, "GET ", &path)) {
char *p;
// This loop is guaranteed to terminate before the end of the buffer, because we know that the
// buffer contains at least "\n\n" and maybe "\r\n\r\n" at the end of the header block.
for (p = path; !isspace(*p); ++p)
;
pathlen = p - path;
if ( str_startswith(p, " HTTP/1.", &p)
&& (str_startswith(p, "0", &p) || str_startswith(p, "1", &p))
&& (str_startswith(p, "\r\n", &p) || str_startswith(p, "\n", &p))
)
path[pathlen] = '\0';
else
path = NULL;
}
if (path) {
char *id = NULL;
INFOF("RHIZOME HTTP SERVER, GET %s", alloca_toprint(1024, path, pathlen));
if (strcmp(path, "/favicon.ico") == 0) {
r->request_type = RHIZOME_HTTP_REQUEST_FAVICON;
rhizome_server_http_response_header(r, 200, "image/vnd.microsoft.icon", favicon_len);
} else if (strcmp(path, "/rhizome/groups") == 0) {
/* Return the list of known groups */
rhizome_server_sql_query_http_response(r, "id", "groups", "from groups", 32, 1);
} else if (strcmp(path, "/rhizome/files") == 0) {
/* Return the list of known files */
rhizome_server_sql_query_http_response(r, "id", "files", "from files", 32, 1);
} else if (strcmp(path, "/rhizome/bars") == 0) {
/* Return the list of known BARs */
rhizome_server_sql_query_http_response(r, "bar", "manifests", "from manifests", 32, 0);
} else if (str_startswith(path, "/rhizome/file/", &id)) {
/* Stream the specified payload */
if (!rhizome_str_is_file_hash(id)) {
rhizome_server_simple_http_response(r, 400, "<html><h1>Invalid payload ID</h1></html>\r\n");
} else {
// TODO: Check for Range: header and return 206 if returning partial content
str_toupper_inplace(id);
long long rowid = -1;
sqlite_exec_int64(&rowid, "select rowid from files where id='%s';", id);
if (rowid >= 0 && sqlite3_blob_open(rhizome_db, "main", "files", "data", rowid, 0, &r->blob) != SQLITE_OK)
rowid = -1;
if (rowid == -1) {
rhizome_server_simple_http_response(r, 404, "<html><h1>Payload not found</h1></html>\r\n");
} else {
r->source_index = 0;
r->blob_end = sqlite3_blob_bytes(r->blob);
rhizome_server_http_response_header(r, 200, "application/binary", r->blob_end - r->source_index);
r->request_type |= RHIZOME_HTTP_REQUEST_BLOB;
}
}
} else if (str_startswith(path, "/rhizome/manifest/", &id)) {
// TODO: Stream the specified manifest
rhizome_server_simple_http_response(r, 500, "<html><h1>Not implemented</h1></html>\r\n");
} else {
rhizome_server_simple_http_response(r, 404, "<html><h1>Not found</h1></html>\r\n");
}
} else {
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (const char *)r->request, r->request_length));
rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
}
/* Try sending data immediately. */
rhizome_server_http_send_bytes(r);
return 0;
}
static int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, char *column)
{
unsigned char blob_value[r->source_record_size*2+1];
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("populating with sql rows at offset %d",r->buffer_length);
if (r->source_index>=r->source_count)
{
/* All done */
return 0;
}
int record_count=(r->buffer_size-r->buffer_length)/r->source_record_size;
if (record_count<1) {
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("r->buffer_size=%d, r->buffer_length=%d, r->source_record_size=%d",
r->buffer_size, r->buffer_length, r->source_record_size);
return WHY("Not enough space to fit any records");
}
sqlite3_stmt *statement = sqlite_prepare("%s LIMIT %lld,%d", r->source, r->source_index, record_count);
if (!statement)
return -1;
if (debug & DEBUG_RHIZOME_TX)
DEBUG(sqlite3_sql(statement));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
while( r->buffer_length + r->source_record_size < r->buffer_size
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW
) {
r->source_index++;
if (sqlite3_column_count(statement)!=2) {
sqlite3_finalize(statement);
return WHY("sqlite3 returned multiple columns for a single column query");
}
sqlite3_blob *blob;
const unsigned char *value;
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_TEXT: value=sqlite3_column_text(statement, 0); break;
case SQLITE_BLOB:
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("table='%s',col='%s',rowid=%lld", table, column, sqlite3_column_int64(statement,1));
int ret;
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", table, column, rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
continue;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
if (sqlite3_blob_read(blob,&blob_value[0],
/* copy number of bytes based on whether we need to
de-hex the string or not */
r->source_record_size*(1+(r->source_flags&1)),0)
!=SQLITE_OK) {
WHYF("sqlite3_blob_read() failed, %s", sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob);
continue;
}
value = blob_value;
sqlite3_blob_close(blob);
break;
default:
/* improper column type, so don't include in report */
WHYF("Bad column type %d", column_type);
continue;
}
if (r->source_flags&1) {
/* hex string to be converted */
int i;
for(i=0;i<r->source_record_size;i++)
/* convert the two nybls and make a byte */
r->buffer[r->buffer_length+i]
=(hexvalue(value[i<<1])<<4)|hexvalue(value[(i<<1)+1]);
} else
/* direct binary value */
bcopy(value,&r->buffer[r->buffer_length],r->source_record_size);
r->buffer_length+=r->source_record_size;
}
sqlite3_finalize(statement);
return 0;
}
int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
{
int pass;
int bytes=e->sizeLimit-e->length;
int overhead=1+8+1+3+1+1+1; /* maximum overhead */
int slots=(bytes-overhead)/RHIZOME_BAR_BYTES;
if (slots>30) slots=30;
int slots_used=0;
int bytes_used=0;
int bytes_available=bytes-overhead-1 /* one byte held for expanding RFS */;
int bundles_advertised=0;
if (slots<1) return WHY("No room for node advertisements");
if (!rhizome_db) return WHY("Rhizome not enabled");
if (ob_append_byte(e,OF_TYPE_RHIZOME_ADVERT))
return WHY("could not add rhizome bundle advertisement header");
ob_append_byte(e,1); /* TTL */
int rfs_offset=e->length; /* remember where the RFS byte gets stored
so that we can patch it later */
ob_append_byte(e,1+8+1+1+1+RHIZOME_BAR_BYTES*slots_used/* RFS */);
/* Stuff in dummy address fields */
ob_append_byte(e,OA_CODE_BROADCAST);
{ int i; for(i=0;i<8;i++) ob_append_byte(e,random()&0xff); } /* BPI for broadcast */
ob_append_byte(e,OA_CODE_PREVIOUS);
ob_append_byte(e,OA_CODE_SELF);
/* Randomly choose whether to advertise manifests or BARs first. */
int skipmanifests=random()&1;
/* Version of rhizome advert block:
1 = manifests then BARs,
2 = BARs only */
ob_append_byte(e,1+skipmanifests);
/* XXX Should add priority bundles here.
XXX Should prioritise bundles for subscribed groups, Serval-authorised files
etc over common bundles.
XXX Should wait a while after going through bundle list so that we don't waste
CPU on db queries if there are not many bundles. Actually, we probably just
shouldn't be sending bundles blindly on every tick.
XXX How do we indicate group membership with BARs? Or do groups actively poll?
*/
if (debug&DEBUG_RHIZOME)
WHY("Group handling not completely thought out here yet.");
/* Get number of bundles available if required */
bundles_available=sqlite_exec_int64("SELECT COUNT(BAR) FROM MANIFESTS;");
if (bundles_available==-1||(bundle_offset[0]>=bundles_available))
bundle_offset[0]=0;
if (bundles_available==-1||(bundle_offset[1]>=bundles_available))
bundle_offset[1]=0;
for(pass=skipmanifests;pass<2;pass++)
{
sqlite3_stmt *statement;
char query[1024];
switch(pass) {
case 0: /* Full manifests */
snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS LIMIT %d,%d",
bundle_offset[pass],slots);
break;
case 1: /* BARs */
snprintf(query,1024,"SELECT BAR,ROWID FROM MANIFESTS LIMIT %d,%d",
bundle_offset[pass],slots);
break;
}
switch (sqlite3_prepare_v2(rhizome_db,query,-1,&statement,NULL))
{
case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW:
break;
default:
sqlite3_finalize(statement);
sqlite3_close(rhizome_db);
rhizome_db=NULL;
WHY(query);
WHY(sqlite3_errmsg(rhizome_db));
return WHY("Could not prepare sql statement for fetching BARs for advertisement.");
}
while((bytes_used<bytes_available)&&(sqlite3_step(statement)==SQLITE_ROW)&&
(e->length+RHIZOME_BAR_BYTES<=e->sizeLimit))
{
sqlite3_blob *blob;
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_BLOB:
if (sqlite3_blob_open(rhizome_db,"main","manifests",
pass?"bar":"manifest",
sqlite3_column_int64(statement,1) /* rowid */,
0 /* read only */,&blob)!=SQLITE_OK)
{
WHY("Couldn't open blob");
continue;
}
int blob_bytes=sqlite3_blob_bytes(blob);
if (pass&&(blob_bytes!=RHIZOME_BAR_BYTES)) {
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"Found a BAR that is the wrong size - ignoring\n");
continue;
}
/* Only include manifests that are <=1KB inline.
Longer ones are only advertised by BAR */
if (blob_bytes>1024) continue;
int overhead=0;
if (!pass) overhead=2;
if (ob_makespace(e,overhead+blob_bytes)) {
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"Stopped cramming %s into Rhizome advertisement frame.\n",
pass?"BARs":"manifests");
break;
}
if (!pass) {
/* put manifest length field and manifest ID */
e->bytes[e->length]=(blob_bytes>>8)&0xff;
e->bytes[e->length+1]=(blob_bytes>>0)&0xff;
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"length bytes written at offset 0x%x\n",e->length);
}
if (sqlite3_blob_read(blob,&e->bytes[e->length+overhead],blob_bytes,0)
!=SQLITE_OK) {
if (debug&DEBUG_RHIZOME) WHY("Couldn't read from blob");
sqlite3_blob_close(blob);
continue;
}
e->length+=overhead+blob_bytes;
bytes_used+=overhead+blob_bytes;
bundles_advertised++;
sqlite3_blob_close(blob);
}
}
sqlite3_finalize(statement);
if (!pass)
{
/* Mark end of whole manifests by writing 0xff, which is more than the MSB
of a manifest's length is allowed to be. */
ob_append_byte(e,0xff);
}
}
/**
@SYMTestCaseID PDS-SQLITE3-UT-4039
@SYMTestCaseDesc SQLITE3 - blob API tests.
List of called SQLITE3 functions:
- sqlite3_bind_zeroblob;
- sqlite3_blob_bytes;
- sqlite3_blob_close;
- sqlite3_blob_open;
- sqlite3_blob_read;
- sqlite3_blob_write;
- sqlite3_sql;
@SYMTestPriority High
@SYMTestActions SQLITE3 - blob API tests.
@SYMTestExpectedResults Test must not fail
@SYMREQ REQ10424
*/
static void TestSqliteBlobApi()
{
int err;
const char* tail = 0;
sqlite3_blob* blob = 0;
int bytes = 0;
const char KBlobData[] = "ABCDEFGH";
char sql[100];
const int KBlobLen = strlen(KBlobData);
const char* sql2 = 0;
const char KSqlFmt[] = "UPDATE BlobTbl SET B=:Prm WHERE ROWID=1";
TEST(TheDb != 0);
TEST(!TheStmt);
/* Create the table, insert one record with a blob */
err = sqlite3_exec(TheDb, "CREATE TABLE BlobTbl(I INTEGER PRIMARY KEY, B BLOB)", 0, 0, 0);
TEST2(err, SQLITE_OK);
sprintf(sql, "INSERT INTO BlobTbl VALUES(1, zeroblob(%d))", KBlobLen);
err = sqlite3_exec(TheDb, sql, 0, 0, 0);
TEST2(err, SQLITE_OK);
err = sqlite3_prepare_v2(TheDb, KSqlFmt, -1, &TheStmt, &tail);
TEST2(err, SQLITE_OK);
TEST(TheStmt != 0);
sql2 = sqlite3_sql(TheStmt);
TEST(sql2 != NULL);
err = strcmp(sql2, KSqlFmt);
TEST2(err, 0);
err = sqlite3_bind_zeroblob(TheStmt, 1, KBlobLen);
TEST2(err, SQLITE_OK);
err = sqlite3_step(TheStmt);
TEST2(err, SQLITE_DONE);
err = sqlite3_finalize(TheStmt);
TEST2(err, SQLITE_OK);
TheStmt = 0;
/* Open the blob and write to it */
err = sqlite3_blob_open(TheDb, "main", "BlobTbl", "B", 1, 1, &blob);
TEST2(err, SQLITE_OK);
TEST(blob != 0);
bytes = sqlite3_blob_bytes(blob);
TEST2(bytes, KBlobLen);
err = sqlite3_blob_write(blob, KBlobData, KBlobLen, 0);
TEST2(err, SQLITE_OK);
err = sqlite3_blob_close(blob);
TEST2(err, SQLITE_OK);
blob = 0;
/* Open the blob and read from it */
err = sqlite3_blob_open(TheDb, "main", "BlobTbl", "B", 1, 1, &blob);
TEST2(err, SQLITE_OK);
TEST(blob != 0);
bytes = sqlite3_blob_bytes(blob);
TEST2(bytes, KBlobLen);
err = sqlite3_blob_read(blob, sql, KBlobLen, 0);
TEST2(err, SQLITE_OK);
sql[bytes] = 0;
err = sqlite3_blob_close(blob);
TEST2(err, SQLITE_OK);
blob = 0;
err = strcmp(sql, KBlobData);
TEST2(err, 0);
err = sqlite3_exec(TheDb, "DROP TABLE BlobTbl", 0, 0, 0);
TEST2(err, SQLITE_OK);
}
/* Read upto the <bars_requested> next BARs from the Rhizome database,
beginning from the first BAR that corresponds to a manifest with
BID>=<bid_low>.
Sets <bid_high> to the highest BID for which a BAR was returned.
Return value is the number of BARs written into <bars_out>.
Only returns BARs for bundles within the specified size range.
This is used by the cursor wrapper function that passes over all of the
BARs in prioritised order.
XXX Once the rhizome database gets big, we will need to make sure
that we have suitable indexes. It is tempting to just pack BARs
by row_id, but the far end needs them in an orderly manner so that
it is possible to make provably complete comparison of the contents
of the respective rhizome databases.
*/
int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES],
unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES],
long long size_low,long long size_high,
const unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES],
unsigned char *bars_out,
int bars_requested)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
char query[1024];
snprintf(query,1024,
"SELECT BAR,ROWID,ID,FILESIZE FROM MANIFESTS"
" WHERE"
" FILESIZE BETWEEN %lld AND %lld"
" AND ID>='%s' AND ID<='%s'"
// The following formulation doesn't remove the weird returning of
// bundles with out of range filesize values
// " WHERE ID>='%s' AND ID<='%s' AND FILESIZE > %lld AND FILESIZE < %lld"
" ORDER BY BAR LIMIT %d;",
size_low, size_high,
alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES),
alloca_tohex(bid_max,RHIZOME_MANIFEST_ID_BYTES),
bars_requested);
sqlite3_stmt *statement=sqlite_prepare(&retry, query);
sqlite3_blob *blob=NULL;
int bars_written=0;
while(bars_written<bars_requested
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW)
{
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_BLOB:
if (blob)
sqlite3_blob_close(blob);
blob = NULL;
int ret;
int64_t filesize = sqlite3_column_int64(statement, 3);
if (filesize<size_low||filesize>size_high) {
DEBUGF("WEIRDNESS ALERT: filesize=%lld, but query was: %s",
filesize,query);
break;
}
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", "manifests", "bar",
rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
continue;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
int blob_bytes=sqlite3_blob_bytes(blob);
if (blob_bytes!=RHIZOME_BAR_BYTES) {
if (config.debug.rhizome)
DEBUG("Found a BAR that is the wrong size - ignoring");
sqlite3_blob_close(blob);
blob=NULL;
continue;
}
sqlite3_blob_read(blob,&bars_out[bars_written*RHIZOME_BAR_BYTES],
RHIZOME_BAR_BYTES,0);
sqlite3_blob_close(blob);
blob=NULL;
/* Remember the BID so that we cant write it into bid_high so that the
caller knows how far we got. */
fromhex(bid_high,
(const char *)sqlite3_column_text(statement, 2),
RHIZOME_MANIFEST_ID_BYTES);
bars_written++;
break;
default:
/* non-BLOB field. This is an error, but we will persevere with subsequent
rows, because they might be fine. */
break;
}
}
if (statement)
sqlite3_finalize(statement);
statement = NULL;
return bars_written;
}
rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix,int prefix_length)
{
/* Give a BID prefix, e.g., from a BAR, find the matching manifest and return it.
Of course, it is possible that more than one manifest matches. This should
occur only very rarely (with the possible exception of intentional attack, and
even then a 64-bit prefix creates a reasonable barrier. If we move to a new
BAR format with 120 or 128 bits of BID prefix, then we should be safe for some
time, thus this function taking the BID prefix as an input in preparation for
that change).
Of course, we need to be able to find the manifest.
Easiest way is to select with a BID range. We could instead have an extra
database column with the prefix.
*/
assert(prefix_length>=0);
assert(prefix_length<=RHIZOME_MANIFEST_ID_BYTES);
unsigned char low[RHIZOME_MANIFEST_ID_BYTES];
unsigned char high[RHIZOME_MANIFEST_ID_BYTES];
memset(low,0x00,RHIZOME_MANIFEST_ID_BYTES);
memset(high,0xff,RHIZOME_MANIFEST_ID_BYTES);
bcopy(bid_prefix,low,prefix_length);
bcopy(bid_prefix,high,prefix_length);
char query[1024];
snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS WHERE ID>='%s' AND ID<='%s'",
alloca_tohex(low,RHIZOME_MANIFEST_ID_BYTES),
alloca_tohex(high,RHIZOME_MANIFEST_ID_BYTES));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, query);
sqlite3_blob *blob=NULL;
if (sqlite_step_retry(&retry, statement) == SQLITE_ROW)
{
int ret;
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", "manifests", "bar",
rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
return NULL;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
/* Read manifest data from blob */
size_t manifestblobsize = sqlite3_column_bytes(statement, 0);
if (manifestblobsize<1||manifestblobsize>1024) goto error;
const char *manifestblob = (char *) sqlite3_column_blob(statement, 0);
if (!manifestblob) goto error;
rhizome_manifest *m=rhizome_new_manifest();
if (rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1)
{
rhizome_manifest_free(m);
goto error;
}
DEBUGF("Read manifest");
sqlite3_blob_close(blob);
sqlite3_finalize(statement);
return m;
error:
sqlite3_blob_close(blob);
sqlite3_finalize(statement);
return NULL;
}
else
{
DEBUGF("no matching manifests");
sqlite3_finalize(statement);
return NULL;
}
}
/* The following function just stores the file (or silently returns if it already
exists).
The relationships of manifests to this file are the responsibility of the
caller. */
int rhizome_store_file(char *file,char *hash,int priority) {
int fd=open(file,O_RDONLY);
if (fd<0) return WHY("Could not open associated file");
struct stat stat;
if (fstat(fd,&stat)) {
close(fd);
return WHY("Could not stat() associated file");
}
unsigned char *addr =
mmap(NULL, stat.st_size, PROT_READ, MAP_FILE|MAP_SHARED, fd, 0);
if (addr==MAP_FAILED) {
close(fd);
return WHY("mmap() of associated file failed.");
}
/* Get hash of file if not supplied */
char hexhash[SHA512_DIGEST_STRING_LENGTH];
if (!hash)
{
/* Hash the file */
SHA512_CTX c;
SHA512_Init(&c);
SHA512_Update(&c,addr,stat.st_size);
SHA512_End(&c,hexhash);
hash=hexhash;
}
/* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer).
BUT, we have to do this incrementally so that we can handle blobs larger than available memory.
This is possible using:
int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
That binds an all zeroes blob to a field. We can then populate the data by
opening a handle to the blob using:
int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
*/
char sqlcmd[1024];
const char *cmdtail;
/* See if the file is already stored, and if so, don't bother storing it again */
int count=sqlite_exec_int64("SELECT COUNT(*) FROM FILES WHERE id='%s' AND datavalid<>0;",hash);
if (count==1) {
/* File is already stored, so just update the highestPriority field if required. */
long long storedPriority = sqlite_exec_int64("SELECT highestPriority FROM FILES WHERE id='%s' AND datavalid!=0",hash);
if (storedPriority<priority)
{
snprintf(sqlcmd,1024,"UPDATE FILES SET highestPriority=%d WHERE id='%s';",
priority,hash);
if (sqlite3_exec(rhizome_db,sqlcmd,NULL,NULL,NULL)!=SQLITE_OK) {
close(fd);
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite failed to update highestPriority field for stored file.");
}
}
close(fd);
return 0;
} else if (count>1) {
/* This should never happen! */
return WHY("Duplicate records for a file in the rhizome database. Database probably corrupt.");
}
/* Okay, so there are no records that match, but we should delete any half-baked record (with datavalid=0) so that the insert below doesn't fail.
Don't worry about the return result, since it might not delete any records. */
sqlite3_exec(rhizome_db,"DELETE FROM FILES WHERE datavalid=0;",NULL,NULL,NULL);
snprintf(sqlcmd,1024,"INSERT INTO FILES(id,data,length,highestpriority,datavalid) VALUES('%s',?,%lld,%d,0);",
hash,(long long)stat.st_size,priority);
sqlite3_stmt *statement;
if (sqlite3_prepare_v2(rhizome_db,sqlcmd,strlen(sqlcmd)+1,&statement,&cmdtail)
!= SQLITE_OK)
{
close(fd);
sqlite3_finalize(statement);
return WHY(sqlite3_errmsg(rhizome_db));
}
/* Bind appropriate sized zero-filled blob to data field */
int dud=0;
int r;
if ((r=sqlite3_bind_zeroblob(statement,1,stat.st_size))!=SQLITE_OK)
{
dud++;
WHY("sqlite3_bind_zeroblob() failed");
WHY(sqlite3_errmsg(rhizome_db));
}
/* Do actual insert, and abort if it fails */
if (!dud)
switch(sqlite3_step(statement)) {
case SQLITE_OK: case SQLITE_ROW: case SQLITE_DONE:
break;
default:
dud++;
WHY("sqlite3_step() failed");
WHY(sqlite3_errmsg(rhizome_db));
}
if (sqlite3_finalize(statement)) dud++;
if (dud) {
close(fd);
if (sqlite3_finalize(statement)!=SQLITE_OK)
{
WHY("sqlite3_finalize() failed");
WHY(sqlite3_errmsg(rhizome_db));
}
return WHY("SQLite3 failed to insert row for file");
}
/* Get rowid for inserted row, so that we can modify the blob */
int rowid=sqlite3_last_insert_rowid(rhizome_db);
if (rowid<1) {
close(fd);
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite3 failed return rowid of inserted row");
}
sqlite3_blob *blob;
if (sqlite3_blob_open(rhizome_db,"main","FILES","data",rowid,
1 /* read/write */,
&blob) != SQLITE_OK)
{
WHY(sqlite3_errmsg(rhizome_db));
close(fd);
sqlite3_blob_close(blob);
return WHY("SQLite3 failed to open file blob for writing");
}
{
long long i;
for(i=0;i<stat.st_size;i+=65536)
{
int n=65536;
if (i+n>stat.st_size) n=stat.st_size-i;
if (sqlite3_blob_write(blob,&addr[i],n,i) !=SQLITE_OK) dud++;
}
}
close(fd);
sqlite3_blob_close(blob);
/* Mark file as up-to-date */
sqlite_exec_int64("UPDATE FILES SET datavalid=1 WHERE id='%s';",
hash);
if (dud) {
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite3 failed write all blob data");
}
printf("stored file\n");
return 0;
}
