/*
** Implementation of the scalar function fts3_tokenizer_internal_test().
** This function is used for testing only, it is not included in the
** build unless SQLITE_TEST is defined.
**
** The purpose of this is to test that the fts3_tokenizer() function
** can be used as designed by the C-code in the queryTokenizer and
** registerTokenizer() functions above. These two functions are repeated
** in the README.tokenizer file as an example, so it is important to
** test them.
**
** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
** function with no arguments. An assert() will fail if a problem is
** detected. i.e.:
**
** SELECT fts3_tokenizer_internal_test();
**
*/
static void intTestFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
int rc;
const sqlite3_tokenizer_module *p1;
const sqlite3_tokenizer_module *p2;
sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
UNUSED_PARAMETER(argc);
UNUSED_PARAMETER(argv);
/* Test the query function */
sqlite3Fts3SimpleTokenizerModule(&p1);
rc = queryTokenizer(db, "simple", &p2);
assert( rc==SQLITE_OK );
assert( p1==p2 );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_ERROR );
assert( p2==0 );
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
/* Test the storage function */
#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
#endif
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
int sqlite3_unicodesn_register_tokenizer(sqlite3 *db)
{
static const sqlite3_tokenizer_module *tokenizer = 0;
if (!tokenizer)
sqlite3Fts3UnicodeSnTokenizer(&tokenizer);
return registerTokenizer(db, TOKENIZER_NAME, tokenizer);
}
int sqlite3_extension_init (
sqlite3 *db, /* The database connection */
char **pzErrMsg, /* Write error messages here */
const sqlite3_api_routines *pApi /* API methods */
) {
SQLITE_EXTENSION_INIT2(pApi)
printf("YES! Load ictclas module\n");
return registerTokenizer(db, "ictclas", &ictlasTokenizerModule);
}
int register_character_tokenizer(sqlite3 *db) {
const sqlite3_tokenizer_module *ptr;
char token_name[] = "character";
// get the tokenizer
get_character_tokenizer_module(&ptr);
// register character tokenizer, note that you need to register it everytime the database is opened
return registerTokenizer(db, token_name, ptr);
}
/* SQLite invokes this routine once when it loads the extension.
** Create new functions, collating sequences, and virtual table
** modules here. This is usually the only exported symbol in
** the shared library.
*/
int sqlite3_extension_init(
sqlite3 *db, /* The database connection */
char **pzErrMsg, /* Write error messages here */
const sqlite3_api_routines *pApi /* API methods */
)
{
const sqlite3_tokenizer_module *tokenizer;
SQLITE_EXTENSION_INIT2(pApi)
sqlite3Fts3UnicodeSnTokenizer(&tokenizer);
registerTokenizer(db, TOKENIZER_NAME, tokenizer);
return 0;
}
int main(void)
{
int i, j;
int rows, cols;
sqlite3 *db;
char *errMsg = NULL;
char **result;
const sqlite3_tokenizer_module *ptr;
char token_name[] = "character";
// create the database
if (sqlite3_open_v2("example.db", &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) {
return 0;
}
// get the tokenizer
get_character_tokenizer_module(&ptr);
// register character tokenizer, note that you need to register it everytime the database is opened
registerTokenizer(db, token_name, ptr);
// remove existing table
sqlite3_exec(db, removesql, 0, 0, &errMsg);
// create table with sqlite FTS3 support
sqlite3_exec(db, createsql, 0, 0, &errMsg);
// insert test data
sqlite3_exec(db, "INSERT INTO Book VALUES('Biography of Barenziah, v1', 'Stern Gamboge');", 0, 0, &errMsg);
sqlite3_exec(db, "INSERT INTO Book VALUES('Biography of Barenziah, v2', 'Stern Gamboge');", 0, 0, &errMsg);
sqlite3_exec(db, "INSERT INTO Book VALUES('Biography of Barenziah, v3', 'Stern Gamboge');", 0, 0, &errMsg);
sqlite3_exec(db, "INSERT INTO Book VALUES('The Alduin-Akatosh Dichotomy', 'Alexandre Simon');", 0, 0, &errMsg);
sqlite3_exec(db, "INSERT INTO Book VALUES('The Legendary Sancre Tor', 'Matera Chapel');", 0, 0, &errMsg);
sqlite3_exec(db, "INSERT INTO Book VALUES('The Legendary Scourge, v3', 'Marobar Sul');", 0, 0, &errMsg);
// test prefix matches
sqlite3_get_table(db , querysql_prefix, &result , &rows, &cols, &errMsg);
printf("query results for %s\n", querysql_prefix);
for (i=0;i<=rows;i++) {
for (j=0;j< cols;j++) {
printf("%s\t", result[i*cols+j]);
}
printf("\n");
}
sqlite3_free_table(result);
printf("\n");
// test suffix matches
sqlite3_get_table(db , querysql_suffix, &result , &rows, &cols, &errMsg);
printf("query results for %s\n", querysql_suffix);
for (i=0;i<=rows;i++) {
for (j=0;j< cols;j++) {
printf("%s\t", result[i*cols+j]);
}
printf("\n");
}
sqlite3_free_table(result);
printf("\n");
// test infix matches
sqlite3_get_table(db , querysql_infix, &result , &rows, &cols, &errMsg);
printf("query results for %s\n", querysql_infix);
for (i=0;i<=rows;i++) {
for (j=0;j< cols;j++) {
printf("%s\t", result[i*cols+j]);
}
printf("\n");
}
sqlite3_free_table(result);
// close database
sqlite3_close(db);
getchar();
return 0;
}
