int DXDB_rdbLoad(FILE *fp) { //printf("DXDB_rdbLoad\n");
uint32 ntbl, nindx;
if ((ntbl = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return -1;
if ((nindx = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return -1;
init_DXDB_PersistentStorageItems(ntbl, nindx);
while (1) {
int type;
if ((type = rdbLoadType(fp)) == -1) return -1;
if (type == REDIS_EOF) break; /* SQL delim REDIS_EOF */
if (type == REDIS_BTREE) {
if (!rdbLoadBT(fp)) return -1;
} else if (type == REDIS_LUA_TRIGGER) {
if (!rdbLoadLuaTrigger(fp)) return -1;
}
}
CLEAR_LUA_STACK lua_getglobal(server.lua, "load_lua_universe");
int r = DXDB_lua_pcall(server.lua, 0, 0, 0);
if (r) {
redisLog(REDIS_WARNING, "ERROR GETTING LUA UNIVERSE: %s",
lua_tostring(server.lua, -1)); //return -1;
}
CLEAR_LUA_STACK
rdbLoadFinished(); // -> build Indexes
return 0;
}
static sds rdbGenericLoadStringObject(FILE*fp, int encode) {
int isencoded;
uint32_t len;
sds val;
len = rdbLoadLen(fp,&isencoded);
if (isencoded) {
switch(len) {
case REDIS_RDB_ENC_INT8:
case REDIS_RDB_ENC_INT16:
case REDIS_RDB_ENC_INT32:
return rdbLoadIntegerObject(fp,len,encode);
case REDIS_RDB_ENC_LZF:
return rdbLoadLzfStringObject(fp);
default:
parsePanic("Unknown RDB encoding type");
}
}
if (len == REDIS_RDB_LENERR) return NULL;
val = sdsnewlen(NULL,len);
if (len && freadCheck(val,len,1,fp) == 0) {
sdsfree(val);
return NULL;
}
return val;
}
static sds rdbLoadLzfStringObject(FILE*fp) {
unsigned int len, clen;
unsigned char *c = NULL;
sds val = NULL;
if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
if ((c = zmalloc(clen)) == NULL) goto err;
if ((val = sdsnewlen(NULL,len)) == NULL) goto err;
if (freadCheck(c,clen,1,fp) == 0) goto err;
if (lzf_decompress(c,clen,val,len) == 0) goto err;
zfree(c);
return val;
err:
zfree(c);
sdsfree(val);
return NULL;
}
robj *rdbLoadNRL(FILE *fp) {
robj *iname;
unsigned int u;
d_l_t *nrlind = malloc(sizeof(d_l_t));
nrlind->l1 = listCreate();
list *nrltoks = nrlind->l1;
nrlind->l2 = listCreate();
list *nrlcols = nrlind->l2;
if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
nrlind->num = (int)u;
int imatch = nrlind->num;
if (!(iname = rdbLoadStringObject(fp))) return NULL;
if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
int tmatch = (int)u;
unsigned int ssize;
if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
for (uint32 i = 0; i < ssize; i++) {
robj *r;
if (!(r = rdbLoadStringObject(fp))) return NULL;
listAddNodeTail(nrltoks, sdsdup(r->ptr));
decrRefCount(r);
}
if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
for (uint32 i = 0; i < ssize; i++) {
uint32 col;
if ((col = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
listAddNodeTail(nrlcols, (void *)(long)col);
}
Index[server.dbid][imatch].obj = iname;
Index[server.dbid][imatch].table = tmatch;
Index[server.dbid][imatch].column = -1;
Index[server.dbid][imatch].type = COL_TYPE_NONE;
Index[server.dbid][imatch].virt = 0;
Index[server.dbid][imatch].nrl = 1;
int dbid = server.dbid;
if (Num_indx[dbid] < (imatch + 1)) Num_indx[dbid] = imatch + 1;
return createObject(REDIS_NRL_INDEX, nrlind);
}
static int rdbLoadRow(FILE *fp, bt *btr) {
void *UUbuf;
uint32 ssize;
if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return -1;
void *bt_val = UU(btr) ? &UUbuf : bt_malloc(ssize, btr);
if (fread(bt_val, ssize, 1, fp) == 0) return -1;
if (btr->numkeys == TRANSITION_ONE_MAX) {
btr = abt_resize(btr, TRANSITION_TWO_BTREE);
}
if UU(btr) bt_insert(btr, UUbuf);
else bt_insert(btr, bt_val);
/* Check the specified RDB file. Return 0 if the RDB looks sane, otherwise
* 1 is returned. */
int redis_check_rdb(char *rdbfilename) {
uint64_t dbid;
int type, rdbver;
char buf[1024];
long long expiretime, now = mstime();
FILE *fp;
static rio rdb; /* Pointed by global struct riostate. */
if ((fp = fopen(rdbfilename,"r")) == NULL) return 1;
rioInitWithFile(&rdb,fp);
rdbstate.rio = &rdb;
rdb.update_cksum = rdbLoadProgressCallback;
if (rioRead(&rdb,buf,9) == 0) goto eoferr;
buf[9] = '\0';
if (memcmp(buf,"REDIS",5) != 0) {
rdbCheckError("Wrong signature trying to load DB from file");
return 1;
}
rdbver = atoi(buf+5);
if (rdbver < 1 || rdbver > RDB_VERSION) {
rdbCheckError("Can't handle RDB format version %d",rdbver);
return 1;
}
startLoading(fp);
while(1) {
robj *key, *val;
expiretime = -1;
/* Read type. */
rdbstate.doing = RDB_CHECK_DOING_READ_TYPE;
if ((type = rdbLoadType(&rdb)) == -1) goto eoferr;
/* Handle special types. */
if (type == RDB_OPCODE_EXPIRETIME) {
rdbstate.doing = RDB_CHECK_DOING_READ_EXPIRE;
/* EXPIRETIME: load an expire associated with the next key
* to load. Note that after loading an expire we need to
* load the actual type, and continue. */
if ((expiretime = rdbLoadTime(&rdb)) == -1) goto eoferr;
/* We read the time so we need to read the object type again. */
rdbstate.doing = RDB_CHECK_DOING_READ_TYPE;
if ((type = rdbLoadType(&rdb)) == -1) goto eoferr;
/* the EXPIRETIME opcode specifies time in seconds, so convert
* into milliseconds. */
expiretime *= 1000;
} else if (type == RDB_OPCODE_EXPIRETIME_MS) {
/* EXPIRETIME_MS: milliseconds precision expire times introduced
* with RDB v3. Like EXPIRETIME but no with more precision. */
rdbstate.doing = RDB_CHECK_DOING_READ_EXPIRE;
if ((expiretime = rdbLoadMillisecondTime(&rdb)) == -1) goto eoferr;
/* We read the time so we need to read the object type again. */
rdbstate.doing = RDB_CHECK_DOING_READ_TYPE;
if ((type = rdbLoadType(&rdb)) == -1) goto eoferr;
} else if (type == RDB_OPCODE_EOF) {
/* EOF: End of file, exit the main loop. */
break;
} else if (type == RDB_OPCODE_SELECTDB) {
/* SELECTDB: Select the specified database. */
rdbstate.doing = RDB_CHECK_DOING_READ_LEN;
if ((dbid = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
goto eoferr;
rdbCheckInfo("Selecting DB ID %d", dbid);
continue; /* Read type again. */
} else if (type == RDB_OPCODE_RESIZEDB) {
/* RESIZEDB: Hint about the size of the keys in the currently
* selected data base, in order to avoid useless rehashing. */
uint64_t db_size, expires_size;
rdbstate.doing = RDB_CHECK_DOING_READ_LEN;
if ((db_size = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
goto eoferr;
if ((expires_size = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
goto eoferr;
continue; /* Read type again. */
} else if (type == RDB_OPCODE_AUX) {
/* AUX: generic string-string fields. Use to add state to RDB
* which is backward compatible. Implementations of RDB loading
* are requierd to skip AUX fields they don't understand.
*
* An AUX field is composed of two strings: key and value. */
robj *auxkey, *auxval;
rdbstate.doing = RDB_CHECK_DOING_READ_AUX;
if ((auxkey = rdbLoadStringObject(&rdb)) == NULL) goto eoferr;
if ((auxval = rdbLoadStringObject(&rdb)) == NULL) goto eoferr;
rdbCheckInfo("AUX FIELD %s = '%s'",
(char*)auxkey->ptr, (char*)auxval->ptr);
decrRefCount(auxkey);
decrRefCount(auxval);
continue; /* Read type again. */
} else {
if (!rdbIsObjectType(type)) {
rdbCheckError("Invalid object type: %d", type);
return 1;
}
rdbstate.key_type = type;
}
/* Read key */
rdbstate.doing = RDB_CHECK_DOING_READ_KEY;
if ((key = rdbLoadStringObject(&rdb)) == NULL) goto eoferr;
rdbstate.key = key;
rdbstate.keys++;
/* Read value */
rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE;
if ((val = rdbLoadObject(type,&rdb)) == NULL) goto eoferr;
/* Check if the key already expired. This function is used when loading
* an RDB file from disk, either at startup, or when an RDB was
* received from the master. In the latter case, the master is
* responsible for key expiry. If we would expire keys here, the
* snapshot taken by the master may not be reflected on the slave. */
if (server.masterhost == NULL && expiretime != -1 && expiretime < now)
rdbstate.already_expired++;
if (expiretime != -1) rdbstate.expires++;
rdbstate.key = NULL;
decrRefCount(key);
decrRefCount(val);
rdbstate.key_type = -1;
}
/* Verify the checksum if RDB version is >= 5 */
if (rdbver >= 5 && server.rdb_checksum) {
uint64_t cksum, expected = rdb.cksum;
rdbstate.doing = RDB_CHECK_DOING_CHECK_SUM;
if (rioRead(&rdb,&cksum,8) == 0) goto eoferr;
memrev64ifbe(&cksum);
if (cksum == 0) {
rdbCheckInfo("RDB file was saved with checksum disabled: no check performed.");
} else if (cksum != expected) {
rdbCheckError("RDB CRC error");
} else {
rdbCheckInfo("Checksum OK");
}
}
fclose(fp);
return 0;
eoferr: /* unexpected end of file is handled here with a fatal exit */
if (rdbstate.error_set) {
rdbCheckError(rdbstate.error);
} else {
rdbCheckError("Unexpected EOF reading RDB file");
}
return 1;
}
int rdbParse(char *rdbFile, keyValueHandler handler) {
int type, loops = 0, dbid, valType;
unsigned int rlen;
char buf[1024];
time_t expiretime = -1;
FILE *fp;
sds key, sval; /* sval is simple string value.*/
sds *cval; /*complicatae value*/
/* Double constants initialization */
R_Zero = 0.0;
R_PosInf = 1.0/R_Zero;
R_NegInf = -1.0/R_Zero;
R_Nan = R_Zero/R_Zero;
if((fp = fopen(rdbFile, "r")) == NULL) {
return PARSE_ERR;
}
if (freadCheck(buf, 9, 1, fp) == 0) {
fclose(fp);
fprintf(stderr, "fread err :%s\n", strerror(errno));
return PARSE_ERR;
}
buf[9] = '\0';
if (memcmp(buf, "REDIS", 5) != 0) {
fclose(fp);
fprintf(stderr, "Wrong signature trying to load DB from file\n");
return PARSE_ERR;
}
rdb_version = atoi(buf+5);
if (rdb_version > 6) {
fclose(fp);
fprintf(stderr, "Can't handle RDB format version %d\n", rdb_version);
return PARSE_ERR;
}
startParse(fp);
while(1) {
if(!(loops++ % 1000)) {
/* record parse progress every 1000 loops. */
parseProgress(ftello(fp));
}
if((type = rdbLoadType(fp)) == -1) return PARSE_ERR;
if(type == REDIS_EXPIRETIME) {
if ((expiretime = rdbLoadTime(fp)) == -1) return PARSE_ERR;
if((type = rdbLoadType(fp)) == -1) return PARSE_ERR;
}
/* file end. */
if(type == REDIS_EOF) {
break;
}
/* select db */
if(type == REDIS_SELECTDB) {
dbid = rdbLoadLen(fp,NULL);
continue;
}
/* load key. */
if ((key = rdbLoadStringObject(fp)) == NULL) {
return PARSE_ERR;
}
if(type == REDIS_HASH_ZIPMAP) {
valType = REDIS_HASH;
} else if(type == REDIS_LIST_ZIPLIST) {
valType = REDIS_LIST;
} else if(type == REDIS_SET_INTSET) {
valType = REDIS_SET;
} else if(type == REDIS_ZSET_ZIPLIST) {
valType = REDIS_ZSET;
} else {
valType = type;
}
/* load value. */
if(type == REDIS_STRING) {
sval = rdbLoadValueObject(fp, type, &rlen);
handler(valType, key, sval, rlen, expiretime);
} else {
cval = rdbLoadValueObject(fp, type, &rlen);
handler(valType, key, cval, rlen, expiretime);
}
/* clean*/
sdsfree(key);
if(valType == REDIS_STRING) {
sdsfree(sval);
} else {
unsigned int k;
for(k = 0; k < rlen; k++) {
sdsfree(cval[k]);
}
zfree(cval);
}
}
/* checksum */
uint64_t checksum = 0;
int ret;
ret = fread(&checksum, sizeof(checksum), 1, fp);
if (ret == 1) {
if ((long long)checksum != digest) {
fprintf(stderr, "DB load failed, checksum does not match: %016llx != %016llx\n", (long long)checksum, digest);
exit(1);
}
fprintf(stderr, "DB loaded, checksum: %016llx\n", digest);
}
parser_stats.stop_time = time(NULL);
fclose(fp);
return PARSE_OK;
}
static void* rdbLoadValueObject(FILE *fp, int type, unsigned int *rlen) {
unsigned int i, j, len;
int buf_len;
sds ele;
sds *results = NULL;
char buf[128];
if(type == REDIS_STRING) {
/* value type is string. */
parser_stats.parse_num[STRING] += 1;
ele = rdbLoadEncodedStringObject(fp);
*rlen = sdslen(ele);
return ele;
} else if(type == REDIS_LIST) {
/* value type is list. */
parser_stats.parse_num[LIST] += 1;
if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
j = 0;
*rlen = len;
results = zmalloc(len * sizeof(*results));
while(len--) {
if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
results[j++] = ele;
}
return results;
} else if(type == REDIS_SET) {
/* value type is set. */
parser_stats.parse_num[SET] += 1;
if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
*rlen = len;
results = zmalloc(len * sizeof(*results));
for (i = 0; i < len; i++) {
if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
results[i] = ele;
}
return results;
} else if (type == REDIS_ZSET) {
/* value type is zset */
parser_stats.parse_num[ZSET] += 1;
size_t zsetlen;
double score;
j = 0;
if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
if(rdb_version < 2) {
*rlen = zsetlen * 2;
} else {
*rlen = zsetlen;
}
results = zmalloc( *rlen * sizeof(*results));
while(zsetlen--) {
if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
if (rdbLoadDoubleValue(fp,&score) == -1) return NULL;
buf_len = snprintf(buf, 128, "%f", score);
results[j] = ele;
results[j+1] = sdsnewlen(buf, buf_len);
j += 2;
}
return results;
} else if (type == REDIS_HASH) {
/* value type is hash */
parser_stats.parse_num[HASH] += 1;
size_t hashlen;
if ((hashlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
sds key, val;
j = 0;
*rlen = hashlen * 2;
results = zmalloc(*rlen * sizeof(*results));
while(hashlen--) {
if ((key = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
if ((val = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
results[j] = key;
results[j + 1] = val;
j += 2;
}
return results;
} else if(type == REDIS_HASH_ZIPMAP ||
type == REDIS_LIST_ZIPLIST ||
type == REDIS_SET_INTSET ||
type == REDIS_ZSET_ZIPLIST ||
type == REDIS_LSET) {
sds aux = rdbLoadStringObject(fp);
switch(type) {
case REDIS_HASH_ZIPMAP:
parser_stats.parse_num[HASH] += 1;
results = loadHashZipMapObject((unsigned char*)aux, rlen);
break;
case REDIS_LIST_ZIPLIST:
parser_stats.parse_num[LIST] += 1;
results = loadListZiplistObject((unsigned char *)aux, rlen);
break;
case REDIS_SET_INTSET:
parser_stats.parse_num[SET] += 1;
results = loadSetIntsetObject((unsigned char *)aux, rlen);
break;
case REDIS_ZSET_ZIPLIST:
parser_stats.parse_num[ZSET] += 1;
results = loadZsetZiplistObject((unsigned char *)aux, rlen);
break;
}
sdsfree(aux);
return results;
} else {
parsePanic("Unknown object type");
}
}