size_t BlobReader::readRaw(void* buf, size_t size)
{
if (_pos + size >= _size)
size = _size - _pos;
if (size == 0) return 0;
int err = sqlite3_blob_read(_blob, buf, size, _pos);
if (err) return 0;
_pos += size;
return size;
}
int SQLiteBlobInputStream::read(char *buffer, int length, U2OpStatus &os) {
SAFE_POINT_EXT(NULL != handle, os.setError("blob handle is not opened"), 0);
int targetLength = (offset + length < size) ? length : (size - offset);
if (0 == targetLength) {
return -1;
}
int status = sqlite3_blob_read(handle, (void*)buffer, targetLength, offset);
if (SQLITE_OK != status) {
os.setError(QObject::tr("Can not read data. The database is closed or the data were changed."));
return 0;
}
offset += targetLength;
return targetLength;
}
void SQLiteBlob::ReadBlob(void *buffer, int numberOfBytes, int offset)
{
if(mBlobHandle == 0)
KOMPEX_EXCEPT("ReadBlob() no open BLOB handle", -1);
if((offset + numberOfBytes) > GetBlobSize())
KOMPEX_EXCEPT("ReadBlob() offset and numberOfBytes exceed the BLOB size", -1);
switch(sqlite3_blob_read(mBlobHandle, buffer, numberOfBytes, offset))
{
case SQLITE_OK:
break;
case SQLITE_ABORT:
KOMPEX_EXCEPT("ReadBlob() BLOB handle expired - can not read BLOB", -1);
default:
KOMPEX_EXCEPT(sqlite3_errmsg(mDatabase->GetDatabaseHandle()), sqlite3_errcode(mDatabase->GetDatabaseHandle()));
}
}
/** Create a trigram expression for matching against a single trigram */
int exprTrigram(expr **ppExpr, trilite_vtab *pTrgVtab, trilite_trigram trigram){
int rc = SQLITE_OK;
sqlite3_blob *pBlob;
char *zTable = sqlite3_mprintf("%s_index", pTrgVtab->zName);
/* Open the blob */
rc = sqlite3_blob_open(pTrgVtab->db, pTrgVtab->zDb, zTable, "doclist", trigram, 0, &pBlob);
sqlite3_free(zTable);
/* If we didn't get a blob */
if(rc != SQLITE_OK){
*ppExpr = NULL;
return SQLITE_OK;
}
/* Get size of blob */
int nSize = sqlite3_blob_bytes(pBlob);
/* Allocate space for expr and doclist at the same time */
*ppExpr = (expr*)sqlite3_malloc(sizeof(expr) + nSize);
/* Set the expr */
(*ppExpr)->eType = EXPR_TRIGRAM;
(*ppExpr)->expr.trigram.docList = ((unsigned char*)(*ppExpr)) + sizeof(expr);
(*ppExpr)->expr.trigram.nSize = nSize;
/* Read doclist into memory */
sqlite3_blob_read(pBlob, (*ppExpr)->expr.trigram.docList, nSize, 0);
/* Release blob */
sqlite3_blob_close(pBlob);
/* Read first id */
int read = readVarInt((*ppExpr)->expr.trigram.docList, &(*ppExpr)->expr.trigram.curId);
(*ppExpr)->expr.trigram.curId += DELTA_LIST_OFFSET;
(*ppExpr)->expr.trigram.nSize -= read;
(*ppExpr)->expr.trigram.docList += read;
return rc;
}
int lsamp_load_to_buf(lsamp_handle *lh, uint32_t offset){
int bufsize = 0;
int rc;
memset(lh->buffer, 0, lh->bufsize);
if(lh->total_frames - lh->frmpos >= lh->bufsize) {
bufsize = lh->bufsize;
}else{
#ifdef LS_DEBUG
printf("lsamp_load_to_buf: using smaller buffer size.\n");
//fflush(stderr);
#endif
bufsize = lh->total_frames - lh->frmpos;
}
rc = sqlite3_blob_read(lh->blob, lh->buffer, bufsize * sizeof(LSFLOAT), offset * sizeof(LSFLOAT));
if(rc) {
//fprintf(stderr, "lsamp_load_to_buf() Error\n");
printf("lsamp_load_to_buf() Error\n");
return 0;
}
lh->frmpos += bufsize;
return 1;
}
bool SQLiteIO::read(char *buffer, int size, int offset, int &rc)
{
rc = sqlite3_blob_read(m_pBlob, buffer, size, offset);
return (rc == SQLITE_OK);
}
error blob::read(int offset, void* buf, int toread)
{
return detail::sqliteerror(__file__, __line__,
sqlite3_blob_read(m_blob, static_cast<char*>(buf), toread, offset));
}
/*
return codes:
1: connection still open.
0: connection finished.
<0: an error occurred.
*/
static int rhizome_server_http_send_bytes(rhizome_http_request *r)
{
// keep writing until the write would block or we run out of data
while(r->request_type){
/* Flush anything out of the buffer if present, before doing any further
processing */
if (r->request_type&RHIZOME_HTTP_REQUEST_FROMBUFFER)
{
int bytes=r->buffer_length-r->buffer_offset;
bytes=write(r->alarm.poll.fd,&r->buffer[r->buffer_offset],bytes);
if (bytes<=0){
// stop writing when the tcp buffer is full
// TODO errors?
return 1;
}
if (0)
dump("bytes written",&r->buffer[r->buffer_offset],bytes);
r->buffer_offset+=bytes;
// reset inactivity timer
r->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT;
r->alarm.deadline = r->alarm.alarm+RHIZOME_IDLE_TIMEOUT;
unschedule(&r->alarm);
schedule(&r->alarm);
if (r->buffer_offset>=r->buffer_length) {
/* Buffer's cleared */
r->request_type&=~RHIZOME_HTTP_REQUEST_FROMBUFFER;
r->buffer_offset=0; r->buffer_length=0;
}
// go around the loop again to work out what we should do next
continue;
}
switch(r->request_type&(~RHIZOME_HTTP_REQUEST_FROMBUFFER))
{
case RHIZOME_HTTP_REQUEST_FAVICON:
if (r->buffer_size<favicon_len) {
free(r->buffer);
r->buffer_size=0;
r->buffer=malloc(favicon_len);
if (!r->buffer) r->request_type=0;
}
if (r->buffer)
{
int i;
for(i=0;i<favicon_len;i++)
r->buffer[i]=favicon_bytes[i];
r->buffer_length=i;
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("favicon buffer_length=%d\n", r->buffer_length);
r->request_type=RHIZOME_HTTP_REQUEST_FROMBUFFER;
}
break;
case RHIZOME_HTTP_REQUEST_BLOB:
{
/* Get more data from the file and put it in the buffer */
int read_size = 65536;
if (r->blob_end-r->source_index < read_size)
read_size = r->blob_end-r->source_index;
r->request_type=0;
if (read_size>0){
if (r->buffer_size < read_size) {
if (r->buffer)
free(r->buffer);
r->buffer=malloc(read_size);
if (!r->buffer) {
WHY_perror("malloc");
r->request_type=0; break;
}
r->buffer_size=read_size;
}
if(sqlite3_blob_read(r->blob,&r->buffer[0],read_size,r->source_index)
==SQLITE_OK)
{
r->buffer_length = read_size;
r->source_index+=read_size;
r->request_type|=RHIZOME_HTTP_REQUEST_FROMBUFFER;
}
}
if (r->source_index >= r->blob_end){
sqlite3_blob_close(r->blob);
r->blob=0;
}else
r->request_type|=RHIZOME_HTTP_REQUEST_BLOB;
}
break;
default:
WHY("sending data from this type of HTTP request not implemented");
r->request_type=0;
break;
}
}
if (!r->request_type) return rhizome_server_free_http_request(r);
return 1;
}
static int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, char *column)
{
unsigned char blob_value[r->source_record_size*2+1];
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("populating with sql rows at offset %d",r->buffer_length);
if (r->source_index>=r->source_count)
{
/* All done */
return 0;
}
int record_count=(r->buffer_size-r->buffer_length)/r->source_record_size;
if (record_count<1) {
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("r->buffer_size=%d, r->buffer_length=%d, r->source_record_size=%d",
r->buffer_size, r->buffer_length, r->source_record_size);
return WHY("Not enough space to fit any records");
}
sqlite3_stmt *statement = sqlite_prepare("%s LIMIT %lld,%d", r->source, r->source_index, record_count);
if (!statement)
return -1;
if (debug & DEBUG_RHIZOME_TX)
DEBUG(sqlite3_sql(statement));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
while( r->buffer_length + r->source_record_size < r->buffer_size
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW
) {
r->source_index++;
if (sqlite3_column_count(statement)!=2) {
sqlite3_finalize(statement);
return WHY("sqlite3 returned multiple columns for a single column query");
}
sqlite3_blob *blob;
const unsigned char *value;
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_TEXT: value=sqlite3_column_text(statement, 0); break;
case SQLITE_BLOB:
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("table='%s',col='%s',rowid=%lld", table, column, sqlite3_column_int64(statement,1));
int ret;
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", table, column, rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
continue;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
if (sqlite3_blob_read(blob,&blob_value[0],
/* copy number of bytes based on whether we need to
de-hex the string or not */
r->source_record_size*(1+(r->source_flags&1)),0)
!=SQLITE_OK) {
WHYF("sqlite3_blob_read() failed, %s", sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob);
continue;
}
value = blob_value;
sqlite3_blob_close(blob);
break;
default:
/* improper column type, so don't include in report */
WHYF("Bad column type %d", column_type);
continue;
}
if (r->source_flags&1) {
/* hex string to be converted */
int i;
for(i=0;i<r->source_record_size;i++)
/* convert the two nybls and make a byte */
r->buffer[r->buffer_length+i]
=(hexvalue(value[i<<1])<<4)|hexvalue(value[(i<<1)+1]);
} else
/* direct binary value */
bcopy(value,&r->buffer[r->buffer_length],r->source_record_size);
r->buffer_length+=r->source_record_size;
}
sqlite3_finalize(statement);
return 0;
}
Error blob::read (std::size_t offset, void* buf, std::size_t toRead)
{
return detail::sqliteError (__FILE__, __LINE__,
sqlite3_blob_read(m_blob, static_cast<char*>(buf), toRead, offset));
}
int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
{
int pass;
int bytes=e->sizeLimit-e->length;
int overhead=1+8+1+3+1+1+1; /* maximum overhead */
int slots=(bytes-overhead)/RHIZOME_BAR_BYTES;
if (slots>30) slots=30;
int slots_used=0;
int bytes_used=0;
int bytes_available=bytes-overhead-1 /* one byte held for expanding RFS */;
int bundles_advertised=0;
if (slots<1) return WHY("No room for node advertisements");
if (!rhizome_db) return WHY("Rhizome not enabled");
if (ob_append_byte(e,OF_TYPE_RHIZOME_ADVERT))
return WHY("could not add rhizome bundle advertisement header");
ob_append_byte(e,1); /* TTL */
int rfs_offset=e->length; /* remember where the RFS byte gets stored
so that we can patch it later */
ob_append_byte(e,1+8+1+1+1+RHIZOME_BAR_BYTES*slots_used/* RFS */);
/* Stuff in dummy address fields */
ob_append_byte(e,OA_CODE_BROADCAST);
{ int i; for(i=0;i<8;i++) ob_append_byte(e,random()&0xff); } /* BPI for broadcast */
ob_append_byte(e,OA_CODE_PREVIOUS);
ob_append_byte(e,OA_CODE_SELF);
/* Randomly choose whether to advertise manifests or BARs first. */
int skipmanifests=random()&1;
/* Version of rhizome advert block:
1 = manifests then BARs,
2 = BARs only */
ob_append_byte(e,1+skipmanifests);
/* XXX Should add priority bundles here.
XXX Should prioritise bundles for subscribed groups, Serval-authorised files
etc over common bundles.
XXX Should wait a while after going through bundle list so that we don't waste
CPU on db queries if there are not many bundles. Actually, we probably just
shouldn't be sending bundles blindly on every tick.
XXX How do we indicate group membership with BARs? Or do groups actively poll?
*/
if (debug&DEBUG_RHIZOME)
WHY("Group handling not completely thought out here yet.");
/* Get number of bundles available if required */
bundles_available=sqlite_exec_int64("SELECT COUNT(BAR) FROM MANIFESTS;");
if (bundles_available==-1||(bundle_offset[0]>=bundles_available))
bundle_offset[0]=0;
if (bundles_available==-1||(bundle_offset[1]>=bundles_available))
bundle_offset[1]=0;
for(pass=skipmanifests;pass<2;pass++)
{
sqlite3_stmt *statement;
char query[1024];
switch(pass) {
case 0: /* Full manifests */
snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS LIMIT %d,%d",
bundle_offset[pass],slots);
break;
case 1: /* BARs */
snprintf(query,1024,"SELECT BAR,ROWID FROM MANIFESTS LIMIT %d,%d",
bundle_offset[pass],slots);
break;
}
switch (sqlite3_prepare_v2(rhizome_db,query,-1,&statement,NULL))
{
case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW:
break;
default:
sqlite3_finalize(statement);
sqlite3_close(rhizome_db);
rhizome_db=NULL;
WHY(query);
WHY(sqlite3_errmsg(rhizome_db));
return WHY("Could not prepare sql statement for fetching BARs for advertisement.");
}
while((bytes_used<bytes_available)&&(sqlite3_step(statement)==SQLITE_ROW)&&
(e->length+RHIZOME_BAR_BYTES<=e->sizeLimit))
{
sqlite3_blob *blob;
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_BLOB:
if (sqlite3_blob_open(rhizome_db,"main","manifests",
pass?"bar":"manifest",
sqlite3_column_int64(statement,1) /* rowid */,
0 /* read only */,&blob)!=SQLITE_OK)
{
WHY("Couldn't open blob");
continue;
}
int blob_bytes=sqlite3_blob_bytes(blob);
if (pass&&(blob_bytes!=RHIZOME_BAR_BYTES)) {
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"Found a BAR that is the wrong size - ignoring\n");
continue;
}
/* Only include manifests that are <=1KB inline.
Longer ones are only advertised by BAR */
if (blob_bytes>1024) continue;
int overhead=0;
if (!pass) overhead=2;
if (ob_makespace(e,overhead+blob_bytes)) {
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"Stopped cramming %s into Rhizome advertisement frame.\n",
pass?"BARs":"manifests");
break;
}
if (!pass) {
/* put manifest length field and manifest ID */
e->bytes[e->length]=(blob_bytes>>8)&0xff;
e->bytes[e->length+1]=(blob_bytes>>0)&0xff;
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"length bytes written at offset 0x%x\n",e->length);
}
if (sqlite3_blob_read(blob,&e->bytes[e->length+overhead],blob_bytes,0)
!=SQLITE_OK) {
if (debug&DEBUG_RHIZOME) WHY("Couldn't read from blob");
sqlite3_blob_close(blob);
continue;
}
e->length+=overhead+blob_bytes;
bytes_used+=overhead+blob_bytes;
bundles_advertised++;
sqlite3_blob_close(blob);
}
}
sqlite3_finalize(statement);
if (!pass)
{
/* Mark end of whole manifests by writing 0xff, which is more than the MSB
of a manifest's length is allowed to be. */
ob_append_byte(e,0xff);
}
}
/**
@SYMTestCaseID PDS-SQLITE3-UT-4039
@SYMTestCaseDesc SQLITE3 - blob API tests.
List of called SQLITE3 functions:
- sqlite3_bind_zeroblob;
- sqlite3_blob_bytes;
- sqlite3_blob_close;
- sqlite3_blob_open;
- sqlite3_blob_read;
- sqlite3_blob_write;
- sqlite3_sql;
@SYMTestPriority High
@SYMTestActions SQLITE3 - blob API tests.
@SYMTestExpectedResults Test must not fail
@SYMREQ REQ10424
*/
static void TestSqliteBlobApi()
{
int err;
const char* tail = 0;
sqlite3_blob* blob = 0;
int bytes = 0;
const char KBlobData[] = "ABCDEFGH";
char sql[100];
const int KBlobLen = strlen(KBlobData);
const char* sql2 = 0;
const char KSqlFmt[] = "UPDATE BlobTbl SET B=:Prm WHERE ROWID=1";
TEST(TheDb != 0);
TEST(!TheStmt);
/* Create the table, insert one record with a blob */
err = sqlite3_exec(TheDb, "CREATE TABLE BlobTbl(I INTEGER PRIMARY KEY, B BLOB)", 0, 0, 0);
TEST2(err, SQLITE_OK);
sprintf(sql, "INSERT INTO BlobTbl VALUES(1, zeroblob(%d))", KBlobLen);
err = sqlite3_exec(TheDb, sql, 0, 0, 0);
TEST2(err, SQLITE_OK);
err = sqlite3_prepare_v2(TheDb, KSqlFmt, -1, &TheStmt, &tail);
TEST2(err, SQLITE_OK);
TEST(TheStmt != 0);
sql2 = sqlite3_sql(TheStmt);
TEST(sql2 != NULL);
err = strcmp(sql2, KSqlFmt);
TEST2(err, 0);
err = sqlite3_bind_zeroblob(TheStmt, 1, KBlobLen);
TEST2(err, SQLITE_OK);
err = sqlite3_step(TheStmt);
TEST2(err, SQLITE_DONE);
err = sqlite3_finalize(TheStmt);
TEST2(err, SQLITE_OK);
TheStmt = 0;
/* Open the blob and write to it */
err = sqlite3_blob_open(TheDb, "main", "BlobTbl", "B", 1, 1, &blob);
TEST2(err, SQLITE_OK);
TEST(blob != 0);
bytes = sqlite3_blob_bytes(blob);
TEST2(bytes, KBlobLen);
err = sqlite3_blob_write(blob, KBlobData, KBlobLen, 0);
TEST2(err, SQLITE_OK);
err = sqlite3_blob_close(blob);
TEST2(err, SQLITE_OK);
blob = 0;
/* Open the blob and read from it */
err = sqlite3_blob_open(TheDb, "main", "BlobTbl", "B", 1, 1, &blob);
TEST2(err, SQLITE_OK);
TEST(blob != 0);
bytes = sqlite3_blob_bytes(blob);
TEST2(bytes, KBlobLen);
err = sqlite3_blob_read(blob, sql, KBlobLen, 0);
TEST2(err, SQLITE_OK);
sql[bytes] = 0;
err = sqlite3_blob_close(blob);
TEST2(err, SQLITE_OK);
blob = 0;
err = strcmp(sql, KBlobData);
TEST2(err, 0);
err = sqlite3_exec(TheDb, "DROP TABLE BlobTbl", 0, 0, 0);
TEST2(err, SQLITE_OK);
}
/* Read upto the <bars_requested> next BARs from the Rhizome database,
beginning from the first BAR that corresponds to a manifest with
BID>=<bid_low>.
Sets <bid_high> to the highest BID for which a BAR was returned.
Return value is the number of BARs written into <bars_out>.
Only returns BARs for bundles within the specified size range.
This is used by the cursor wrapper function that passes over all of the
BARs in prioritised order.
XXX Once the rhizome database gets big, we will need to make sure
that we have suitable indexes. It is tempting to just pack BARs
by row_id, but the far end needs them in an orderly manner so that
it is possible to make provably complete comparison of the contents
of the respective rhizome databases.
*/
int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES],
unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES],
long long size_low,long long size_high,
const unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES],
unsigned char *bars_out,
int bars_requested)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
char query[1024];
snprintf(query,1024,
"SELECT BAR,ROWID,ID,FILESIZE FROM MANIFESTS"
" WHERE"
" FILESIZE BETWEEN %lld AND %lld"
" AND ID>='%s' AND ID<='%s'"
// The following formulation doesn't remove the weird returning of
// bundles with out of range filesize values
// " WHERE ID>='%s' AND ID<='%s' AND FILESIZE > %lld AND FILESIZE < %lld"
" ORDER BY BAR LIMIT %d;",
size_low, size_high,
alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES),
alloca_tohex(bid_max,RHIZOME_MANIFEST_ID_BYTES),
bars_requested);
sqlite3_stmt *statement=sqlite_prepare(&retry, query);
sqlite3_blob *blob=NULL;
int bars_written=0;
while(bars_written<bars_requested
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW)
{
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_BLOB:
if (blob)
sqlite3_blob_close(blob);
blob = NULL;
int ret;
int64_t filesize = sqlite3_column_int64(statement, 3);
if (filesize<size_low||filesize>size_high) {
DEBUGF("WEIRDNESS ALERT: filesize=%lld, but query was: %s",
filesize,query);
break;
}
int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", "manifests", "bar",
rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
continue;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
int blob_bytes=sqlite3_blob_bytes(blob);
if (blob_bytes!=RHIZOME_BAR_BYTES) {
if (config.debug.rhizome)
DEBUG("Found a BAR that is the wrong size - ignoring");
sqlite3_blob_close(blob);
blob=NULL;
continue;
}
sqlite3_blob_read(blob,&bars_out[bars_written*RHIZOME_BAR_BYTES],
RHIZOME_BAR_BYTES,0);
sqlite3_blob_close(blob);
blob=NULL;
/* Remember the BID so that we cant write it into bid_high so that the
caller knows how far we got. */
fromhex(bid_high,
(const char *)sqlite3_column_text(statement, 2),
RHIZOME_MANIFEST_ID_BYTES);
bars_written++;
break;
default:
/* non-BLOB field. This is an error, but we will persevere with subsequent
rows, because they might be fine. */
break;
}
}
if (statement)
sqlite3_finalize(statement);
statement = NULL;
return bars_written;
}
