/* Open the tag index in redis */
TagIndex *TagIndex_Open(RedisSearchCtx *sctx, RedisModuleString *formattedKey, int openWrite,
RedisModuleKey **keyp) {
TagIndex *ret = NULL;
if (!sctx->spec->keysDict) {
RedisModuleKey *key_s = NULL;
if (!keyp) {
keyp = &key_s;
}
*keyp = RedisModule_OpenKey(sctx->redisCtx, formattedKey,
REDISMODULE_READ | (openWrite ? REDISMODULE_WRITE : 0));
int type = RedisModule_KeyType(*keyp);
if (type != REDISMODULE_KEYTYPE_EMPTY && RedisModule_ModuleTypeGetType(*keyp) != TagIndexType) {
return NULL;
}
/* Create an empty value object if the key is currently empty. */
if (type == REDISMODULE_KEYTYPE_EMPTY) {
if (openWrite) {
ret = NewTagIndex();
RedisModule_ModuleTypeSetValue((*keyp), TagIndexType, ret);
}
} else {
ret = RedisModule_ModuleTypeGetValue(*keyp);
}
} else {
ret = openTagKeyDict(sctx, formattedKey, openWrite);
}
return ret;
}
int UniqueLenCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc != 2) {
return RedisModule_WrongArity(ctx);
}
RedisModule_AutoMemory(ctx);
// open the key and make sure it is indeed a Hash and not empty
RedisModuleKey *key =
RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ);
int type = RedisModule_KeyType(key);
if (type == REDISMODULE_KEYTYPE_EMPTY) {
RedisModule_ReplyWithNull(ctx);
return REDISMODULE_OK;
}
if (RedisModule_ModuleTypeGetType(key) != UniqueType) {
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
unique *unique = RedisModule_ModuleTypeGetValue(key);
size_t len = uniqueLen(unique);
RedisModule_ReplyWithLongLong(ctx, len);
return REDISMODULE_OK;
}
int UniqueGetAllCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc != 2) {
return RedisModule_WrongArity(ctx);
}
RedisModule_AutoMemory(ctx);
// open the key and make sure it is indeed a Hash and not empty
RedisModuleKey *key =
RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ);
int type = RedisModule_KeyType(key);
if (type == REDISMODULE_KEYTYPE_EMPTY) {
RedisModule_ReplyWithNull(ctx);
return REDISMODULE_OK;
}
if (RedisModule_ModuleTypeGetType(key) != UniqueType) {
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
unique *unique = RedisModule_ModuleTypeGetValue(key);
size_t len = uniqueLen(unique);
RedisModule_ReplyWithArray(ctx, len*2);
size_t i;
listIter *it = listGetIterator(unique->l, AL_START_HEAD);
listNode *node;
for (node = listNext(it), i=0; node && i<len; node = listNext(it),i++) {
dictEntry *en = node->value;
RedisModule_ReplyWithStringBuffer(ctx, en->key, sdslen(en->key));
RedisModule_ReplyWithStringBuffer(ctx, en->v.val, sdslen(en->v.val));
}
return REDISMODULE_OK;
}
/* HELLOTYPE.RANGE key first count */
int HelloTypeRange_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
if (argc != 4) return RedisModule_WrongArity(ctx);
RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1],
REDISMODULE_READ|REDISMODULE_WRITE);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY &&
RedisModule_ModuleTypeGetType(key) != HelloType)
{
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
long long first, count;
if (RedisModule_StringToLongLong(argv[2],&first) != REDISMODULE_OK ||
RedisModule_StringToLongLong(argv[3],&count) != REDISMODULE_OK ||
first < 0 || count < 0)
{
return RedisModule_ReplyWithError(ctx,
"ERR invalid first or count parameters");
}
struct HelloTypeObject *hto = RedisModule_ModuleTypeGetValue(key);
struct HelloTypeNode *node = hto ? hto->head : NULL;
RedisModule_ReplyWithArray(ctx,REDISMODULE_POSTPONED_ARRAY_LEN);
long long arraylen = 0;
while(node && count--) {
RedisModule_ReplyWithLongLong(ctx,node->value);
arraylen++;
node = node->next;
}
RedisModule_ReplySetArrayLength(ctx,arraylen);
return REDISMODULE_OK;
}
int UniquePopCommandHelper(RedisModuleCtx *ctx, RedisModuleString **argv, int argc,
PopReplyCB replycb) {
if (argc != 2) {
return RedisModule_WrongArity(ctx);
}
RedisModule_AutoMemory(ctx);
// open the key and make sure it is indeed a Hash and not empty
RedisModuleKey *key =
RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ);
int type = RedisModule_KeyType(key);
if (type == REDISMODULE_KEYTYPE_EMPTY) {
RedisModule_ReplyWithNull(ctx);
return REDISMODULE_OK;
}
if (RedisModule_ModuleTypeGetType(key) != UniqueType) {
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
unique *unique = RedisModule_ModuleTypeGetValue(key);
sds skey, sval;
int r = uniquePop(unique, &skey, &sval);
if (r == 0) {
replycb(ctx, skey, sval);
sdsfree(skey);
sdsfree(sval);
} else {
RedisModule_ReplyWithNull(ctx);
}
return REDISMODULE_OK;
}
/* HELLOTYPE.INSERT key value */
int HelloTypeInsert_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
if (argc != 3) return RedisModule_WrongArity(ctx);
RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1],
REDISMODULE_READ|REDISMODULE_WRITE);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY &&
RedisModule_ModuleTypeGetType(key) != HelloType)
{
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
long long value;
if ((RedisModule_StringToLongLong(argv[2],&value) != REDISMODULE_OK)) {
return RedisModule_ReplyWithError(ctx,"ERR invalid value: must be a signed 64 bit integer");
}
/* Create an empty value object if the key is currently empty. */
struct HelloTypeObject *hto;
if (type == REDISMODULE_KEYTYPE_EMPTY) {
hto = createHelloTypeObject();
RedisModule_ModuleTypeSetValue(key,HelloType,hto);
} else {
hto = RedisModule_ModuleTypeGetValue(key);
}
/* Insert the new element. */
HelloTypeInsert(hto,value);
RedisModule_ReplyWithLongLong(ctx,hto->len);
RedisModule_ReplicateVerbatim(ctx);
return REDISMODULE_OK;
}
// Create and populate index for specified property
// (This function will create separate string and numeric indices if property has mixed types)
void Index_Create(RedisModuleCtx *ctx, const char *graphName, AST_IndexNode *indexOp) {
const char *label = indexOp->target.label;
const char *prop_str = indexOp->target.property;
LabelStore *store = LabelStore_Get(ctx, STORE_NODE, graphName, label);
LabelStoreIterator it;
LabelStore_Scan(store, &it);
char *nodeId;
uint16_t nodeIdLen;
Node *node;
EntityProperty *prop;
RedisModuleKey *key = Index_LookupKey(ctx, graphName, label, prop_str);
// Do nothing if this index already exists
if (RedisModule_ModuleTypeGetType(key) == IndexRedisModuleType) {
RedisModule_CloseKey(key);
return;
}
Index *index = malloc(sizeof(Index));
RedisModule_ModuleTypeSetValue(key, IndexRedisModuleType, index);
RedisModule_CloseKey(key);
index->target.label = strdup(label);
index->target.property = strdup(prop_str);
index->string_sl = skiplistCreate(compareStrings, NULL, compareNodes, cloneKey, freeKey);
index->numeric_sl = skiplistCreate(compareNumerics, NULL, compareNodes, cloneKey, freeKey);
int prop_index = 0;
while(LabelStoreIterator_Next(&it, &nodeId, &nodeIdLen, (void**)&node)) {
// If the sought property is at a different offset than it occupied in the previous node,
// then seek and update
if (strcmp(prop_str, node->properties[prop_index].name)) {
for (int i = 0; i < node->prop_count; i ++) {
prop = node->properties + i;
if (!strcmp(prop_str, prop->name)) {
prop_index = i;
break;
}
}
}
prop = node->properties + prop_index;
// This value will be cloned within the skiplistInsert routine if necessary
SIValue *key = &prop->value;
if (key->type == T_STRING) {
skiplistInsert(index->string_sl, key, node);
} else if (key->type & SI_NUMERIC) {
skiplistInsert(index->numeric_sl, key, node);
} else { // This property was neither a string nor numeric value; raise a run-time error.
assert(0);
}
}
}
Index* Index_Get(RedisModuleCtx *ctx, const char *graph, const char *label, const char *property) {
RedisModuleKey *key = Index_LookupKey(ctx, graph, label, property);
Index *idx = NULL;
if (RedisModule_ModuleTypeGetType(key) == IndexRedisModuleType) {
idx = RedisModule_ModuleTypeGetValue(key);
}
RedisModule_CloseKey(key);
return idx;
}
int UniquePushIVCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc < 4) {
return RedisModule_WrongArity(ctx);
}
RedisModule_AutoMemory(ctx);
// open the key and make sure it is indeed a Hash and not empty
RedisModuleKey *key =
RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ | REDISMODULE_WRITE);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY &&
RedisModule_ModuleTypeGetType(key) != UniqueType)
{
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
unique *unique;
if (type == REDISMODULE_KEYTYPE_EMPTY) {
unique = uniqueCreate();
RedisModule_ModuleTypeSetValue(key,UniqueType,unique);
} else {
unique = RedisModule_ModuleTypeGetValue(key);
}
sds skey, sval;
size_t skeylen, svallen;
const char *pkey;
unsigned int lvec = argc-3;
pkey = RedisModule_StringPtrLen(argv[2], &skeylen);
svallen = lvec *8;
skey = sdsnewlen(pkey, skeylen);
sval = sdsnewlen(NULL, svallen);
long long *vec = (long long*)sval;
int i;
for (i=3; i<argc; i++) {
if (REDISMODULE_OK != RedisModule_StringToLongLong(argv[i], vec+i-3)) {
return RedisModule_WrongArity(ctx);
}
}
int n = uniquePush(unique, skey, sval, merge_int64);
if (n == -1) {
RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
} else {
RedisModule_ReplyWithLongLong(ctx, n);
}
RedisModule_ReplicateVerbatim(ctx);
return REDISMODULE_OK;
}
/*
## FT.SUGLEN key
Get the size of an autoc-complete suggestion dictionary
### Parameters:
- key: the suggestion dictionary key.
### Returns:
Integer reply: the current size of the suggestion dictionary.
*/
int RSSuggestLenCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
if (argc != 2) return RedisModule_WrongArity(ctx);
RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY && RedisModule_ModuleTypeGetType(key) != TrieType) {
return RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
}
Trie *tree = RedisModule_ModuleTypeGetValue(key);
return RedisModule_ReplyWithLongLong(ctx, tree ? tree->size : 0);
}
/*
## FT.SUGGADD key string score [INCR] [PAYLOAD {payload}]
Add a suggestion string to an auto-complete suggestion dictionary. This is
disconnected from the
index definitions, and leaves creating and updating suggestino dictionaries to
the user.
### Parameters:
- key: the suggestion dictionary key.
- string: the suggestion string we index
- score: a floating point number of the suggestion string's weight
-INCR: if set, we increment the existing entry of the suggestion by the
given
score, instead
of
replacing the score. This is useful for updating the dictionary based on
user queries in
real
time
- PAYLOAD: Add a payload to the suggestion string that will be used as additional information.
### Returns:
Integer reply: the current size of the suggestion dictionary.
*/
int RSSuggestAddCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc < 4 || argc > 7) {
return RedisModule_WrongArity(ctx);
}
int incr = 0, rv = AC_OK;
RSPayload payload = {0};
ArgsCursor ac = {0};
ArgsCursor_InitRString(&ac, argv + 4, argc - 4);
while (!AC_IsAtEnd(&ac)) {
const char *s = AC_GetStringNC(&ac, NULL);
if (!strcasecmp(s, "INCR")) {
incr = 1;
} else if (!strcasecmp(s, "PAYLOAD")) {
if ((rv = AC_GetString(&ac, (const char **)&payload.data, &payload.len, 0)) != AC_OK) {
return RMUtil_ReplyWithErrorFmt(ctx, "Invalid payload: %s", AC_Strerror(rv));
}
} else {
return RMUtil_ReplyWithErrorFmt(ctx, "Unknown argument `%s`", s);
}
}
RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ | REDISMODULE_WRITE);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY && RedisModule_ModuleTypeGetType(key) != TrieType) {
return RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
}
RedisModuleString *val = argv[2];
double score;
if ((RedisModule_StringToDouble(argv[3], &score) != REDISMODULE_OK)) {
return RedisModule_ReplyWithError(ctx, "ERR invalid score");
}
/* Create an empty value object if the key is currently empty. */
Trie *tree;
if (type == REDISMODULE_KEYTYPE_EMPTY) {
tree = NewTrie();
RedisModule_ModuleTypeSetValue(key, TrieType, tree);
} else {
tree = RedisModule_ModuleTypeGetValue(key);
}
/* Insert the new element. */
Trie_Insert(tree, val, score, incr, &payload);
RedisModule_ReplyWithLongLong(ctx, tree->size);
RedisModule_ReplicateVerbatim(ctx);
return REDISMODULE_OK;
}
/* HELLOTYPE.LEN key */
int HelloTypeLen_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
if (argc != 2) return RedisModule_WrongArity(ctx);
RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1],
REDISMODULE_READ|REDISMODULE_WRITE);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY &&
RedisModule_ModuleTypeGetType(key) != HelloType)
{
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
struct HelloTypeObject *hto = RedisModule_ModuleTypeGetValue(key);
RedisModule_ReplyWithLongLong(ctx,hto ? hto->len : 0);
return REDISMODULE_OK;
}
static int UniquePushCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, mergefn fn) {
if (argc != 4) {
return RedisModule_WrongArity(ctx);
}
RedisModule_AutoMemory(ctx);
// open the key and make sure it is indeed a Hash and not empty
RedisModuleKey *key =
RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ | REDISMODULE_WRITE);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY &&
RedisModule_ModuleTypeGetType(key) != UniqueType)
{
return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
}
unique *unique;
if (type == REDISMODULE_KEYTYPE_EMPTY) {
unique = uniqueCreate();
RedisModule_ModuleTypeSetValue(key,UniqueType,unique);
} else {
unique = RedisModule_ModuleTypeGetValue(key);
}
sds skey, sval;
size_t skeylen, svallen;
const char *pkey, *pval;
pkey = RedisModule_StringPtrLen(argv[2], &skeylen);
pval = RedisModule_StringPtrLen(argv[3], &svallen);
skey = sdsnewlen(pkey, skeylen);
sval = sdsnewlen(pval, svallen);
int n = uniquePush(unique, skey, sval, fn);
if (n == -1) {
RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
} else {
RedisModule_ReplyWithLongLong(ctx, n);
}
RedisModule_ReplicateVerbatim(ctx);
return REDISMODULE_OK;
}
/*
## FT.SUGDEL key str
Delete a string from a suggestion index.
### Parameters:
- key: the suggestion dictionary key.
- str: the string to delete
### Returns:
Integer reply: 1 if the string was found and deleted, 0 otherwise.
*/
int RSSuggestDelCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
if (argc != 3) return RedisModule_WrongArity(ctx);
RedisModule_ReplicateVerbatim(ctx);
RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ);
int type = RedisModule_KeyType(key);
if (type != REDISMODULE_KEYTYPE_EMPTY && RedisModule_ModuleTypeGetType(key) != TrieType) {
return RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
}
Trie *tree = RedisModule_ModuleTypeGetValue(key);
if (!tree) {
return RedisModule_ReplyWithLongLong(ctx, 0);
}
size_t len;
const char *str = RedisModule_StringPtrLen(argv[2], &len);
return RedisModule_ReplyWithLongLong(ctx, Trie_Delete(tree, str, len));
}
/* Ecode a single docId into a specific tag value */
static inline size_t tagIndex_Put(TagIndex *idx, const char *value, size_t len, t_docId docId) {
InvertedIndex *iv = TrieMap_Find(idx->values, (char *)value, len);
if (iv == TRIEMAP_NOTFOUND) {
iv = NewInvertedIndex(Index_DocIdsOnly, 1);
TrieMap_Add(idx->values, (char *)value, len, iv, NULL);
}
IndexEncoder enc = InvertedIndex_GetEncoder(Index_DocIdsOnly);
RSIndexResult rec = {.type = RSResultType_Virtual, .docId = docId, .offsetsSz = 0, .freq = 0};
return InvertedIndex_WriteEntryGeneric(iv, enc, docId, &rec);
}
/* Index a vector of pre-processed tags for a docId */
size_t TagIndex_Index(TagIndex *idx, const char **values, size_t n, t_docId docId) {
if (!values) return 0;
size_t ret = 0;
for (size_t ii = 0; ii < n; ++ii) {
const char *tok = values[ii];
if (tok && *tok != '\0') {
ret += tagIndex_Put(idx, tok, strlen(tok), docId);
}
}
return ret;
}
typedef struct IndexIterator **TagConcCtx;
static void TagReader_OnReopen(RedisModuleKey *k, void *privdata) {
TagConcCtx *ctx = privdata;
IndexIterator **its = privdata;
size_t nits = array_len(its);
// If the key has been deleted we'll get a NULL here, so we just mark ourselves as EOF
if (k == NULL || RedisModule_ModuleTypeGetType(k) != TagIndexType) {
for (size_t ii = 0; ii < nits; ++ii) {
its[ii]->Abort(its[ii]->ctx);
}
return;
}
// If the key is valid, we just reset the reader's buffer reader to the current block pointer
TagIndex *idx = RedisModule_ModuleTypeGetValue(k);
for (size_t ii = 0; ii < nits; ++ii) {
IndexReader *ir = its[ii]->ctx;
// the gc marker tells us if there is a chance the keys has undergone GC while we were asleep
if (ir->gcMarker == ir->idx->gcMarker) {
// no GC - we just go to the same offset we were at
size_t offset = ir->br.pos;
ir->br = NewBufferReader(&ir->idx->blocks[ir->currentBlock].buf);
ir->br.pos = offset;
} else {
// if there has been a GC cycle on this key while we were asleep, the offset might not be
// valid anymore. This means that we need to seek to last docId we were at
// reset the state of the reader
t_docId lastId = ir->lastId;
ir->br = NewBufferReader(&ir->idx->blocks[ir->currentBlock].buf);
ir->lastId = 0;
// seek to the previous last id
RSIndexResult *dummy = NULL;
IR_SkipTo(ir, lastId, &dummy);
}
}
}