/*
** Insert a new row into the FTS content table.
*/
int sqlite3Fts5StorageContentInsert(
Fts5Storage *p,
sqlite3_value **apVal,
i64 *piRowid
){
Fts5Config *pConfig = p->pConfig;
int rc = SQLITE_OK;
/* Insert the new row into the %_content table. */
if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
*piRowid = sqlite3_value_int64(apVal[1]);
}else{
rc = fts5StorageNewRowid(p, piRowid);
}
}else{
sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
int i; /* Counter variable */
rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
rc = sqlite3_bind_value(pInsert, i, apVal[i]);
}
if( rc==SQLITE_OK ){
sqlite3_step(pInsert);
rc = sqlite3_reset(pInsert);
}
*piRowid = sqlite3_last_insert_rowid(pConfig->db);
}
return rc;
}
int sqlite3Fts5StorageConfigValue(
Fts5Storage *p,
const char *z,
sqlite3_value *pVal,
int iVal
){
sqlite3_stmt *pReplace = 0;
int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
if( pVal ){
sqlite3_bind_value(pReplace, 2, pVal);
}else{
sqlite3_bind_int(pReplace, 2, iVal);
}
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
}
if( rc==SQLITE_OK && pVal ){
int iNew = p->pConfig->iCookie + 1;
rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
if( rc==SQLITE_OK ){
p->pConfig->iCookie = iNew;
}
}
return rc;
}
/*
** Argument aCol points to an array of integers containing one entry for
** each table column. This function reads the %_docsize record for the
** specified rowid and populates aCol[] with the results.
**
** An SQLite error code is returned if an error occurs, or SQLITE_OK
** otherwise.
*/
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
int nCol = p->pConfig->nCol; /* Number of user columns in table */
sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */
int rc; /* Return Code */
assert( p->pConfig->bColumnsize );
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
if( rc==SQLITE_OK ){
int bCorrupt = 1;
sqlite3_bind_int64(pLookup, 1, iRowid);
if( SQLITE_ROW==sqlite3_step(pLookup) ){
const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
int nBlob = sqlite3_column_bytes(pLookup, 0);
if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
bCorrupt = 0;
}
}
rc = sqlite3_reset(pLookup);
if( bCorrupt && rc==SQLITE_OK ){
rc = FTS5_CORRUPT;
}
}
return rc;
}
/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
Fts5Config *pConfig = p->pConfig;
int rc;
sqlite3_stmt *pDel = 0;
assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
rc = fts5StorageLoadTotals(p, 1);
/* Delete the index records */
if( rc==SQLITE_OK ){
rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
}
/* Delete the %_docsize record */
if( rc==SQLITE_OK && pConfig->bColumnsize ){
rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pDel, 1, iDel);
sqlite3_step(pDel);
rc = sqlite3_reset(pDel);
}
}
/* Delete the %_content record */
if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
}
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pDel, 1, iDel);
sqlite3_step(pDel);
rc = sqlite3_reset(pDel);
}
}
/* Write the averages record */
if( rc==SQLITE_OK ){
rc = fts5StorageSaveTotals(p);
}
return rc;
}
int sqlite3Fts5StorageRebuild(Fts5Storage *p){
Fts5Buffer buf = {0,0,0};
Fts5Config *pConfig = p->pConfig;
sqlite3_stmt *pScan = 0;
Fts5InsertCtx ctx;
int rc, rc2;
memset(&ctx, 0, sizeof(Fts5InsertCtx));
ctx.pStorage = p;
rc = sqlite3Fts5StorageDeleteAll(p);
if( rc==SQLITE_OK ){
rc = fts5StorageLoadTotals(p, 1);
}
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
i64 iRowid = sqlite3_column_int64(pScan, 0);
sqlite3Fts5BufferZero(&buf);
rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, ctx.iCol+1),
sqlite3_column_bytes(pScan, ctx.iCol+1),
(void*)&ctx,
fts5StorageInsertCallback
);
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
}
p->nTotalRow++;
if( rc==SQLITE_OK ){
rc = fts5StorageInsertDocsize(p, iRowid, &buf);
}
}
sqlite3_free(buf.p);
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
/* Write the averages record */
if( rc==SQLITE_OK ){
rc = fts5StorageSaveTotals(p);
}
return rc;
}
/*
** If a row with rowid iDel is present in the %_content table, add the
** delete-markers to the FTS index necessary to delete it. Do not actually
** remove the %_content row at this time though.
*/
static int fts5StorageDeleteFromIndex(
Fts5Storage *p,
i64 iDel,
sqlite3_value **apVal
){
Fts5Config *pConfig = p->pConfig;
sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */
int rc; /* Return code */
int rc2; /* sqlite3_reset() return code */
int iCol;
Fts5InsertCtx ctx;
if( apVal==0 ){
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)!=SQLITE_ROW ){
return sqlite3_reset(pSeek);
}
}
ctx.pStorage = p;
ctx.iCol = -1;
rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
if( pConfig->abUnindexed[iCol-1]==0 ){
const char *zText;
int nText;
if( pSeek ){
zText = (const char*)sqlite3_column_text(pSeek, iCol);
nText = sqlite3_column_bytes(pSeek, iCol);
}else{
zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
nText = sqlite3_value_bytes(apVal[iCol-1]);
}
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
zText, nText, (void*)&ctx, fts5StorageInsertCallback
);
p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
}
}
p->nTotalRow--;
rc2 = sqlite3_reset(pSeek);
if( rc==SQLITE_OK ) rc = rc2;
return rc;
}
/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
** by inserting a dummy row into the %_docsize table. The dummy will be
** overwritten later.
**
** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
** this case the user is required to provide a rowid explicitly.
*/
static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
int rc = SQLITE_MISMATCH;
if( p->pConfig->bColumnsize ){
sqlite3_stmt *pReplace = 0;
rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_null(pReplace, 1);
sqlite3_bind_null(pReplace, 2);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
}
if( rc==SQLITE_OK ){
*piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
}
}
return rc;
}
/*
** Obtain an SQLite statement handle that may be used to read data from the
** %_content table.
*/
int sqlite3Fts5StorageStmt(
Fts5Storage *p,
int eStmt,
sqlite3_stmt **pp,
char **pzErrMsg
){
int rc;
assert( eStmt==FTS5_STMT_SCAN_ASC
|| eStmt==FTS5_STMT_SCAN_DESC
|| eStmt==FTS5_STMT_LOOKUP
);
rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
if( rc==SQLITE_OK ){
assert( p->aStmt[eStmt]==*pp );
p->aStmt[eStmt] = 0;
}
return rc;
}
/*
** Insert a record into the %_docsize table. Specifically, do:
**
** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
**
** If there is no %_docsize table (as happens if the columnsize=0 option
** is specified when the FTS5 table is created), this function is a no-op.
*/
static int fts5StorageInsertDocsize(
Fts5Storage *p, /* Storage module to write to */
i64 iRowid, /* id value */
Fts5Buffer *pBuf /* sz value */
){
int rc = SQLITE_OK;
if( p->pConfig->bColumnsize ){
sqlite3_stmt *pReplace = 0;
rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pReplace, 1, iRowid);
sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
}
}
return rc;
}
/*
** Check that the contents of the FTS index match that of the %_content
** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
** some other SQLite error code if an error occurs while attempting to
** determine this.
*/
int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
Fts5Config *pConfig = p->pConfig;
int rc; /* Return code */
int *aColSize; /* Array of size pConfig->nCol */
i64 *aTotalSize; /* Array of size pConfig->nCol */
Fts5IntegrityCtx ctx;
sqlite3_stmt *pScan;
memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
ctx.pConfig = p->pConfig;
aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
if( !aTotalSize ) return SQLITE_NOMEM;
aColSize = (int*)&aTotalSize[pConfig->nCol];
memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
/* Generate the expected index checksum based on the contents of the
** %_content table. This block stores the checksum in ctx.cksum. */
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
if( rc==SQLITE_OK ){
int rc2;
while( SQLITE_ROW==sqlite3_step(pScan) ){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
ctx.szCol = 0;
if( pConfig->bColumnsize ){
rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
}
if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
(void*)&ctx,
fts5StorageIntegrityCallback
);
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
if( rc!=SQLITE_OK ) break;
}
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
}
/* Test that the "totals" (sometimes called "averages") record looks Ok */
if( rc==SQLITE_OK ){
int i;
rc = fts5StorageLoadTotals(p, 0);
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT;
}
}
/* Check that the %_docsize and %_content tables contain the expected
** number of rows. */
if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
i64 nRow = 0;
rc = fts5StorageCount(p, "content", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
}
if( rc==SQLITE_OK && pConfig->bColumnsize ){
i64 nRow = 0;
rc = fts5StorageCount(p, "docsize", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
}
/* Pass the expected checksum down to the FTS index module. It will
** verify, amongst other things, that it matches the checksum generated by
** inspecting the index itself. */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum);
}
sqlite3_free(aTotalSize);
return rc;
}
