/*
** CSV virtual table module xColumn method.
*/
static int csvColumn(sqlite3_vtab_cursor *pVtabCursor, sqlite3_context *ctx, int i){
CSV *pCSV = (CSV *)pVtabCursor->pVtab;
if( i<0 || i>=pCSV->nCol ){
sqlite3_result_null( ctx );
}else{
// TODO SQLite uses dynamic typing...
const char *col = pCSV->aCols[i];
if( !col ){
sqlite3_result_null( ctx );
}else if( pCSV->aEscapedQuotes[i] ){
char *z;
int nByte = (int)(strlen(col) - pCSV->aEscapedQuotes[i] + 1);
if( nByte>sqlite3_limit(pCSV->db, SQLITE_LIMIT_LENGTH, -1) ){
sqlite3_result_error_toobig( ctx );
z = 0;
}else{
z = sqlite3_malloc( nByte );
if( !z ){
sqlite3_result_error_nomem( ctx );
}
}
if( z ){
int j,k;
for(j=0, k=0; col[j]; j++){
z[k++] = col[j];
if( col[j]=='\"' ){
/* unescape quote */
j++;
}
}
z[k] = 0;
sqlite3_result_text( ctx, z, k, sqlite3_free ); // FIXME sqlite3_result_int64/double
}
}else{
sqlite3_result_text( ctx, col, -1, SQLITE_TRANSIENT ); // FIXME sqlite3_result_int64/double
}
}
return SQLITE_OK;
}
/**
@SYMTestCaseID PDS-SQLITE3-UT-4038
@SYMTestCaseDesc Database handle SQLITE3 tests.
List of called SQLITE3 functions:
- sqlite3_db_status;
- sqlite3_file_control;
- sqlite3_limit;
- sqlite3_next_stmt;
- sqlite3_randomness;
@SYMTestPriority High
@SYMTestActions Database handle SQLITE3 tests.
@SYMTestExpectedResults Test must not fail
@SYMREQ REQ10424
*/
static void TestSqliteApi2()
{
int used = 0;
int high = 0;
int lock = -1;
int limit = 0;
sqlite3_stmt* next = 0;
int err;
unsigned char buf[10];
TEST(TheDb != 0);
err = sqlite3_db_status(TheDb, SQLITE_DBSTATUS_LOOKASIDE_USED, &used, &high, 0);
TEST2(err, SQLITE_OK);
PrintI("Lookaside slots: %d\r\n", used);
PrintI("Max used lookaside slots: %d\r\n", high);
err = sqlite3_file_control(TheDb, "main", SQLITE_FCNTL_LOCKSTATE, &lock);
TEST2(err, SQLITE_OK);
TEST2(lock, SQLITE_LOCK_NONE);
limit = sqlite3_limit(TheDb, SQLITE_LIMIT_LENGTH, -1);
TEST(limit > 0);
next = sqlite3_next_stmt(TheDb, 0);
TEST(!next);
memset(buf, 0, sizeof(buf));
sqlite3_randomness(8, buf);
memset(buf, 0, sizeof(buf));
sqlite3_randomness(7, buf);
memset(buf, 0, sizeof(buf));
sqlite3_randomness(3, buf);
}
size_t Repo::stringLengthLimit() const {
static const size_t limit = sqlite3_limit(m_dbc, SQLITE_LIMIT_LENGTH, -1);
return limit;
}
static int maxLimit(sqlite3 *db, int eCode){
int iOrig = sqlite3_limit(db, eCode, 0x7fffffff);
return sqlite3_limit(db, eCode, iOrig);
}
int db_tags_from_files(int *files, int num_files, int **tags) {
GHashTable *table = NULL;
GHashTableIter iter;
char *file_id_str = NULL;
char *query = NULL;
char or_outline[] = " OR file_id = ";
char query_outline[] = "SELECT DISTINCT tag_id FROM all_tables WHERE file_id = ";
const int QUERY_LENGTH_CAP = 1024;
gpointer key = NULL;
gpointer value = NULL;
int file_id = 0;
int hash_table_size = 0;
int i = 0;
int initial_query_length = 0;
int num_digits_in_id = 0;
int query_outline_length = 0;
int sql_length_avail = 0;
int sql_max_length = 0;
int written = 0; /* number of characters written */
sqlite3 *conn = NULL;
DEBUG(ENTRY);
assert(files != NULL);
assert(num_files > 0);
assert(*tags == NULL);
DEBUG("Retrieving tags from %d files", num_files);
/* prepare query/table */
query_outline_length = strlen(query_outline);
table = g_hash_table_new(NULL, NULL);
assert(table != NULL);
/* connect to database */
conn = db_connect();
assert(conn != NULL);
/* calculate maximum query length */
sql_max_length = sqlite3_limit(conn, SQLITE_LIMIT_SQL_LENGTH, -1);
assert(sql_max_length > query_outline_length);
DEBUG("Database reports maximum query length of %d", sql_max_length);
sql_max_length = sql_max_length < QUERY_LENGTH_CAP ? sql_max_length : QUERY_LENGTH_CAP;
assert(sql_max_length > query_outline_length);
DEBUG("Maximum query length has been set to %d", sql_max_length);
/* for each file id */
for(i = 0; i < num_files; i++) {
num_digits_in_id = num_digits(files[i]);
file_id = files[i];
/* start new query */
if(sql_length_avail <= 0) {
DEBUG("Starting new query");
sql_length_avail = sql_max_length - query_outline_length;
assert(sql_length_avail > 0);
query = calloc(sql_max_length + 1, sizeof(*query));
assert(query != NULL);
initial_query_length = query_outline_length + num_digits_in_id;
written = snprintf(query, initial_query_length + 1, "%s%d", query_outline, file_id);
assert(written == query_outline_length + num_digits_in_id);
sql_length_avail -= num_digits_in_id;
continue;
}
/* test if query can contain next file id */
sql_length_avail -= strlen(or_outline) + num_digits(files[i]);
/* if the query is at maximum length */
if(sql_length_avail <= 0) {
DEBUG("Query has reached maximum length...");
i--; /* leave it for next iteration */
db_insert_query_results_into_hashtable(conn, query, table);
free_single_ptr((void *)&query);
continue;
}
/* append file ID to query */
file_id_str = calloc(num_digits_in_id + 1, sizeof(*file_id_str));
snprintf(file_id_str, num_digits_in_id + 1, "%d", file_id);
strncat(strncat(query, or_outline, strlen(or_outline)), file_id_str, strlen(file_id_str));
free_single_ptr((void *)&file_id_str);
}
db_insert_query_results_into_hashtable(conn, query, table);
free_single_ptr((void *)&query);
db_disconnect(conn);
/* copy results of set to array */
hash_table_size = g_hash_table_size(table);
*tags = malloc(hash_table_size * sizeof(**tags));
g_hash_table_iter_init(&iter, table);
i = 0;
while(g_hash_table_iter_next(&iter, &key, &value)) {
(*tags)[i++] = (unsigned long)value;
}
g_hash_table_destroy(table);
DEBUG("Returning array of %d elements", hash_table_size);
DEBUG(EXIT);
return hash_table_size;
} /* db_tags_from_files */
/*
** CSV virtual table module xNext method.
*/
static int csvNext( sqlite3_vtab_cursor* pVtabCursor ){
CSV *pCSV = (CSV *)pVtabCursor->pVtab;
CSVCursor *pCsr = (CSVCursor *)pVtabCursor;
int nCol = 0;
char *s;
char zDelims[3] = ",\n";
char cDelim; /* char that delimited current col */
if( pCSV->eof ){
return SQLITE_ERROR;
}
/* update the cursor */
pCsr->csvpos = csv_tell( pCSV );
/* read the next row of data */
s = csv_getline( pCSV );
if( !s ){
/* and error or eof occured */
pCSV->eof = -1;
return SQLITE_OK;
}
/* allocate initial space for the column pointers */
if( pCSV->maxCol < 1 ){
/* take a guess */
int maxCol = (int)(strlen(pCSV->zRow) / 5 + 1);
pCSV->aCols = (char **)sqlite3_malloc( sizeof(char*) * maxCol );
pCSV->aEscapedQuotes = (int *)sqlite3_malloc( sizeof(int) * maxCol );
if( pCSV->aCols ){
pCSV->maxCol = maxCol;
}
}
if( !pCSV->aCols || !pCSV->aEscapedQuotes ) return SQLITE_NOMEM;
/* add custom delim character */
zDelims[0] = pCSV->cDelim;
/* parse the zRow into individual columns */
do{
/* if it begins with a quote, assume it's a quoted col */
if( *s=='\"' ){
s++; /* skip quote */
pCSV->aCols[nCol] = s; /* save pointer for this col */
pCSV->aEscapedQuotes[nCol] = 0;
/* find closing quote */
while( 1 ){
s = strchr(s, '\"');
if( !s ){
/* no closing quote */
pCSV->eof = -1;
return SQLITE_ERROR;
}else if ( *(s+1)=='\"' ){
/* escaped quote */
pCSV->aEscapedQuotes[nCol]++;
s+=2;
}else{
break;
}
}
*s = '\0'; /* null terminate this col */
/* fall through and look for following ",\n" */
s++;
}else{
pCSV->aCols[nCol] = s; /* save pointer for this col */
pCSV->aEscapedQuotes[nCol] = 0;
}
s = strpbrk(s, zDelims);
if( !s ){
/* no col delimiter */
pCSV->eof = -1;
return SQLITE_ERROR;
}
// TODO Maybe we should treat empty string as null value (if an option is
// activated).
cDelim = *s;
/* null terminate the column by overwriting the delimiter */
*s = '\0';
nCol++;
/* if end of zRow, stop parsing cols */
if( cDelim == '\n' ) break;
/* move to start of next col */
s++; /* skip delimiter */
if(nCol >= pCSV->maxCol ){
if( nCol>=sqlite3_limit(pCSV->db, SQLITE_LIMIT_COLUMN, -1) ){
return SQLITE_ERROR;
}
/* we need to grow our col pointer array */
char **p = (char **)sqlite3_realloc( pCSV->aCols, sizeof(char*) * (nCol+5) );
if( !p ){
/* out of memory */
return SQLITE_ERROR;
}
pCSV->maxCol = nCol + 5;
pCSV->aCols = p;
int *p1 = (int *)sqlite3_realloc( pCSV->aEscapedQuotes, sizeof(int) * (nCol+5) );
if( !p1 ){
/* out of memory */
return SQLITE_ERROR;
}
pCSV->aEscapedQuotes = p1;
}
}while( *s );
pCSV->nCol = nCol;
return SQLITE_OK;
}
/*
** This routine reads a line of text from FILE in, stores
** the text in memory obtained from malloc() and returns a pointer
** to the text. NULL is returned at end of file, or if malloc()
** fails.
**
** The interface is like "readline" but no command-line editing
** is done.
**
** This code was modified from existing code in shell.c of the sqlite3 CLI.
*/
static char *csv_getline( CSV *pCSV ){
int n = 0;
int bEol = 0;
int bShrink = 0;
int bQuotedCol = 0;
/* allocate initial row buffer */
if( pCSV->maxRow < 1 ){
pCSV->zRow = sqlite3_malloc( 100 );
if( pCSV->zRow ){
pCSV->maxRow = 100;
}
}
if( !pCSV->zRow ) {
sqlite3_log(SQLITE_NOMEM, "Error while reading CSV line (1)");
return 0;
}
/* read until eol */
while( !bEol ){
/* grow row buffer as needed */
if( n+100>pCSV->maxRow ){
int newSize = pCSV->maxRow*2 + 100;
if( newSize>= sqlite3_limit(pCSV->db, SQLITE_LIMIT_LENGTH, -1) ){
sqlite3_log(SQLITE_ERROR, "CSV row is too long (> %d)", pCSV->maxRow);
return 0;
}
char *p = sqlite3_realloc(pCSV->zRow, newSize);
if( !p ) {
sqlite3_log(SQLITE_NOMEM, "Error while reading CSV line (2)");
return 0;
}
pCSV->maxRow = newSize;
pCSV->zRow = p;
bShrink = -1;
}
if( fgets(&pCSV->zRow[n], pCSV->maxRow-n, pCSV->f)==0 ){
if( n==0 ){
break;
}
pCSV->zRow[n] = '\0';
bEol = -1;
break;
}
/* look for line delimiter */
while( pCSV->zRow[n] ){
if( pCSV->zRow[n]=='\"' ){
if( bQuotedCol ) {
if( pCSV->zRow[n+1]=='\"' ) { /* escaped */
n++;
}else{
bQuotedCol = 0;
}
}else if( n==0 || pCSV->zRow[n-1]==pCSV->cDelim ){
bQuotedCol = 1;
}
}
n++;
}
/* uniform line ending */
if( (n>0) && ((pCSV->zRow[n-1]=='\n') || (pCSV->zRow[n-1]=='\r')) && !bQuotedCol ){
if( (n>1) && ((pCSV->zRow[n-1]=='\n') && (pCSV->zRow[n-2]=='\r')) ){
pCSV->zRow[n-2] = '\n';
pCSV->zRow[n-1] = '\0';
bEol = -1;
}else{
pCSV->zRow[n-1] = '\n';
pCSV->zRow[n] = '\0';
bEol = -1;
}
}
}
if( bShrink ){
pCSV->zRow = sqlite3_realloc( pCSV->zRow, n+1 );
pCSV->maxRow = n+1;
}
return bEol ? pCSV->zRow : 0;
}
DLL_FUNCTION(int32_t) BU_SQLite_Limit(sqlite3* db, int32_t id, int32_t newVal) {
#pragma comment(linker, "/EXPORT:BU_SQLite_Limit=_BU_SQLite_Limit@12")
return sqlite3_limit(db, id, newVal);
}
