/* ** Create a "unicode61" tokenizer. */ static int fts5UnicodeCreate( void *pCtx, const char **azArg, int nArg, Fts5Tokenizer **ppOut ){ int rc = SQLITE_OK; /* Return code */ Unicode61Tokenizer *p = 0; /* New tokenizer object */ if( nArg%2 ){ rc = SQLITE_ERROR; }else{ p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); if( p ){ int i; memset(p, 0, sizeof(Unicode61Tokenizer)); memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); p->bRemoveDiacritic = 1; p->nFold = 64; p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); if( p->aFold==0 ){ rc = SQLITE_NOMEM; } for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ const char *zArg = azArg[i+1]; if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){ rc = SQLITE_ERROR; } p->bRemoveDiacritic = (zArg[0]=='1'); }else if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ rc = fts5UnicodeAddExceptions(p, zArg, 1); }else if( 0==sqlite3_stricmp(azArg[i], "separators") ){ rc = fts5UnicodeAddExceptions(p, zArg, 0); }else{ rc = SQLITE_ERROR; } } }else{ rc = SQLITE_NOMEM; } if( rc!=SQLITE_OK ){ fts5UnicodeDelete((Fts5Tokenizer*)p); p = 0; } *ppOut = (Fts5Tokenizer*)p; } return rc; }
/* ** Create a "unicode61" tokenizer. */ static int fts5UnicodeCreate( void *pUnused, const char **azArg, int nArg, Fts5Tokenizer **ppOut ){ int rc = SQLITE_OK; /* Return code */ Unicode61Tokenizer *p = 0; /* New tokenizer object */ UNUSED_PARAM(pUnused); if( nArg%2 ){ rc = SQLITE_ERROR; }else{ p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); if( p ){ const char *zCat = "L* N* Co"; int i; memset(p, 0, sizeof(Unicode61Tokenizer)); p->bRemoveDiacritic = 1; p->nFold = 64; p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); if( p->aFold==0 ){ rc = SQLITE_NOMEM; } /* Search for a "categories" argument */ for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ if( 0==sqlite3_stricmp(azArg[i], "categories") ){ zCat = azArg[i+1]; } } if( rc==SQLITE_OK ){ rc = unicodeSetCategories(p, zCat); } for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ const char *zArg = azArg[i+1]; if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){ rc = SQLITE_ERROR; } p->bRemoveDiacritic = (zArg[0]=='1'); }else if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ rc = fts5UnicodeAddExceptions(p, zArg, 1); }else if( 0==sqlite3_stricmp(azArg[i], "separators") ){ rc = fts5UnicodeAddExceptions(p, zArg, 0); }else if( 0==sqlite3_stricmp(azArg[i], "categories") ){ /* no-op */ }else{ rc = SQLITE_ERROR; } } }else{ rc = SQLITE_NOMEM; } if( rc!=SQLITE_OK ){ fts5UnicodeDelete((Fts5Tokenizer*)p); p = 0; } *ppOut = (Fts5Tokenizer*)p; } return rc; }