Exemplo n.º 1
0
int DXDB_rdbLoad(FILE *fp) { //printf("DXDB_rdbLoad\n");
   uint32 ntbl, nindx;
   if ((ntbl  = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return -1;
   if ((nindx = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return -1;
    init_DXDB_PersistentStorageItems(ntbl, nindx);
    while (1) {
        int type;
        if ((type = rdbLoadType(fp))  == -1)               return -1;
        if (type == REDIS_EOF)              break;    /* SQL delim REDIS_EOF */
        if        (type == REDIS_BTREE) {
            if (!rdbLoadBT(fp))                            return -1;
        } else if (type == REDIS_LUA_TRIGGER) {
            if (!rdbLoadLuaTrigger(fp))                    return -1;
        }
    }
    CLEAR_LUA_STACK lua_getglobal(server.lua, "load_lua_universe");
    int r = DXDB_lua_pcall(server.lua, 0, 0, 0);
    if (r) {
        redisLog(REDIS_WARNING, "ERROR GETTING LUA UNIVERSE: %s",
                                 lua_tostring(server.lua, -1)); //return -1;
    }
    CLEAR_LUA_STACK
    rdbLoadFinished(); // -> build Indexes
    return 0;
}
Exemplo n.º 2
0
static sds rdbGenericLoadStringObject(FILE*fp, int encode) {
    int isencoded;
    uint32_t len; 
    sds val; 

    len = rdbLoadLen(fp,&isencoded);
    if (isencoded) {
        switch(len) {
            case REDIS_RDB_ENC_INT8:
            case REDIS_RDB_ENC_INT16:
            case REDIS_RDB_ENC_INT32:
                return rdbLoadIntegerObject(fp,len,encode);
            case REDIS_RDB_ENC_LZF:
                return rdbLoadLzfStringObject(fp);
            default:
                parsePanic("Unknown RDB encoding type");
        }    
    }    

    if (len == REDIS_RDB_LENERR) return NULL;
    val = sdsnewlen(NULL,len);
    if (len && freadCheck(val,len,1,fp) == 0) { 
        sdsfree(val);
        return NULL;
    }    
    return val; 
}
Exemplo n.º 3
0
static sds rdbLoadLzfStringObject(FILE*fp) {
    unsigned int len, clen;
    unsigned char *c = NULL;
    sds val = NULL;

    if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
    if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
    if ((c = zmalloc(clen)) == NULL) goto err;
    if ((val = sdsnewlen(NULL,len)) == NULL) goto err;
    if (freadCheck(c,clen,1,fp) == 0) goto err;
    if (lzf_decompress(c,clen,val,len) == 0) goto err;
    zfree(c);
    return val;
err:
    zfree(c);
    sdsfree(val);
    return NULL;
}
Exemplo n.º 4
0
robj *rdbLoadNRL(FILE *fp) {
    robj         *iname;
    unsigned int  u;
    d_l_t *nrlind  = malloc(sizeof(d_l_t));
    nrlind->l1     = listCreate();
    list  *nrltoks = nrlind->l1;
    nrlind->l2     = listCreate();
    list  *nrlcols = nrlind->l2;

    if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
    nrlind->num = (int)u;
    int imatch  = nrlind->num;
    if (!(iname = rdbLoadStringObject(fp))) return NULL;
    if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
    int tmatch  = (int)u;

    unsigned int ssize;
    if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
    for (uint32 i = 0; i < ssize; i++) {
        robj *r;
        if (!(r = rdbLoadStringObject(fp))) return NULL;
        listAddNodeTail(nrltoks, sdsdup(r->ptr));
        decrRefCount(r);
    }

    if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
    for (uint32 i = 0; i < ssize; i++) {
        uint32 col;
        if ((col = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
        listAddNodeTail(nrlcols, (void *)(long)col);
    }
    Index[server.dbid][imatch].obj     = iname;
    Index[server.dbid][imatch].table   = tmatch;
    Index[server.dbid][imatch].column  = -1;
    Index[server.dbid][imatch].type    = COL_TYPE_NONE;
    Index[server.dbid][imatch].virt    = 0;
    Index[server.dbid][imatch].nrl     = 1;
    int dbid = server.dbid;
    if (Num_indx[dbid] < (imatch + 1)) Num_indx[dbid] = imatch + 1;

    return createObject(REDIS_NRL_INDEX, nrlind);
}
Exemplo n.º 5
0
static int rdbLoadRow(FILE *fp, bt *btr) {
    void   *UUbuf;
    uint32  ssize;
    if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return -1;
    void *bt_val = UU(btr) ? &UUbuf : bt_malloc(ssize, btr);
    if (fread(bt_val, ssize, 1, fp) == 0) return -1;
    if (btr->numkeys == TRANSITION_ONE_MAX) {
        btr = abt_resize(btr, TRANSITION_TWO_BTREE);
    }
    if UU(btr) bt_insert(btr, UUbuf);
    else       bt_insert(btr, bt_val);
Exemplo n.º 6
0
/* Check the specified RDB file. Return 0 if the RDB looks sane, otherwise
 * 1 is returned. */
int redis_check_rdb(char *rdbfilename) {
    uint64_t dbid;
    int type, rdbver;
    char buf[1024];
    long long expiretime, now = mstime();
    FILE *fp;
    static rio rdb; /* Pointed by global struct riostate. */

    if ((fp = fopen(rdbfilename,"r")) == NULL) return 1;

    rioInitWithFile(&rdb,fp);
    rdbstate.rio = &rdb;
    rdb.update_cksum = rdbLoadProgressCallback;
    if (rioRead(&rdb,buf,9) == 0) goto eoferr;
    buf[9] = '\0';
    if (memcmp(buf,"REDIS",5) != 0) {
        rdbCheckError("Wrong signature trying to load DB from file");
        return 1;
    }
    rdbver = atoi(buf+5);
    if (rdbver < 1 || rdbver > RDB_VERSION) {
        rdbCheckError("Can't handle RDB format version %d",rdbver);
        return 1;
    }

    startLoading(fp);
    while(1) {
        robj *key, *val;
        expiretime = -1;

        /* Read type. */
        rdbstate.doing = RDB_CHECK_DOING_READ_TYPE;
        if ((type = rdbLoadType(&rdb)) == -1) goto eoferr;

        /* Handle special types. */
        if (type == RDB_OPCODE_EXPIRETIME) {
            rdbstate.doing = RDB_CHECK_DOING_READ_EXPIRE;
            /* EXPIRETIME: load an expire associated with the next key
             * to load. Note that after loading an expire we need to
             * load the actual type, and continue. */
            if ((expiretime = rdbLoadTime(&rdb)) == -1) goto eoferr;
            /* We read the time so we need to read the object type again. */
            rdbstate.doing = RDB_CHECK_DOING_READ_TYPE;
            if ((type = rdbLoadType(&rdb)) == -1) goto eoferr;
            /* the EXPIRETIME opcode specifies time in seconds, so convert
             * into milliseconds. */
            expiretime *= 1000;
        } else if (type == RDB_OPCODE_EXPIRETIME_MS) {
            /* EXPIRETIME_MS: milliseconds precision expire times introduced
             * with RDB v3. Like EXPIRETIME but no with more precision. */
            rdbstate.doing = RDB_CHECK_DOING_READ_EXPIRE;
            if ((expiretime = rdbLoadMillisecondTime(&rdb)) == -1) goto eoferr;
            /* We read the time so we need to read the object type again. */
            rdbstate.doing = RDB_CHECK_DOING_READ_TYPE;
            if ((type = rdbLoadType(&rdb)) == -1) goto eoferr;
        } else if (type == RDB_OPCODE_EOF) {
            /* EOF: End of file, exit the main loop. */
            break;
        } else if (type == RDB_OPCODE_SELECTDB) {
            /* SELECTDB: Select the specified database. */
            rdbstate.doing = RDB_CHECK_DOING_READ_LEN;
            if ((dbid = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
                goto eoferr;
            rdbCheckInfo("Selecting DB ID %d", dbid);
            continue; /* Read type again. */
        } else if (type == RDB_OPCODE_RESIZEDB) {
            /* RESIZEDB: Hint about the size of the keys in the currently
             * selected data base, in order to avoid useless rehashing. */
            uint64_t db_size, expires_size;
            rdbstate.doing = RDB_CHECK_DOING_READ_LEN;
            if ((db_size = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
                goto eoferr;
            if ((expires_size = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
                goto eoferr;
            continue; /* Read type again. */
        } else if (type == RDB_OPCODE_AUX) {
            /* AUX: generic string-string fields. Use to add state to RDB
             * which is backward compatible. Implementations of RDB loading
             * are requierd to skip AUX fields they don't understand.
             *
             * An AUX field is composed of two strings: key and value. */
            robj *auxkey, *auxval;
            rdbstate.doing = RDB_CHECK_DOING_READ_AUX;
            if ((auxkey = rdbLoadStringObject(&rdb)) == NULL) goto eoferr;
            if ((auxval = rdbLoadStringObject(&rdb)) == NULL) goto eoferr;

            rdbCheckInfo("AUX FIELD %s = '%s'",
                (char*)auxkey->ptr, (char*)auxval->ptr);
            decrRefCount(auxkey);
            decrRefCount(auxval);
            continue; /* Read type again. */
        } else {
            if (!rdbIsObjectType(type)) {
                rdbCheckError("Invalid object type: %d", type);
                return 1;
            }
            rdbstate.key_type = type;
        }

        /* Read key */
        rdbstate.doing = RDB_CHECK_DOING_READ_KEY;
        if ((key = rdbLoadStringObject(&rdb)) == NULL) goto eoferr;
        rdbstate.key = key;
        rdbstate.keys++;
        /* Read value */
        rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE;
        if ((val = rdbLoadObject(type,&rdb)) == NULL) goto eoferr;
        /* Check if the key already expired. This function is used when loading
         * an RDB file from disk, either at startup, or when an RDB was
         * received from the master. In the latter case, the master is
         * responsible for key expiry. If we would expire keys here, the
         * snapshot taken by the master may not be reflected on the slave. */
        if (server.masterhost == NULL && expiretime != -1 && expiretime < now)
            rdbstate.already_expired++;
        if (expiretime != -1) rdbstate.expires++;
        rdbstate.key = NULL;
        decrRefCount(key);
        decrRefCount(val);
        rdbstate.key_type = -1;
    }
    /* Verify the checksum if RDB version is >= 5 */
    if (rdbver >= 5 && server.rdb_checksum) {
        uint64_t cksum, expected = rdb.cksum;

        rdbstate.doing = RDB_CHECK_DOING_CHECK_SUM;
        if (rioRead(&rdb,&cksum,8) == 0) goto eoferr;
        memrev64ifbe(&cksum);
        if (cksum == 0) {
            rdbCheckInfo("RDB file was saved with checksum disabled: no check performed.");
        } else if (cksum != expected) {
            rdbCheckError("RDB CRC error");
        } else {
            rdbCheckInfo("Checksum OK");
        }
    }

    fclose(fp);
    return 0;

eoferr: /* unexpected end of file is handled here with a fatal exit */
    if (rdbstate.error_set) {
        rdbCheckError(rdbstate.error);
    } else {
        rdbCheckError("Unexpected EOF reading RDB file");
    }
    return 1;
}
Exemplo n.º 7
0
int rdbParse(char *rdbFile, keyValueHandler handler) {
    int type, loops = 0, dbid, valType;
    unsigned int rlen;
    char buf[1024];
    time_t expiretime = -1;
    FILE *fp;
    sds key, sval; /* sval is simple string value.*/
    sds *cval; /*complicatae value*/

    /* Double constants initialization */
    R_Zero = 0.0;
    R_PosInf = 1.0/R_Zero;
    R_NegInf = -1.0/R_Zero;
    R_Nan = R_Zero/R_Zero;

    if((fp = fopen(rdbFile, "r")) == NULL) {
        return PARSE_ERR;
    }

    if (freadCheck(buf, 9, 1, fp) == 0) {
        fclose(fp);
        fprintf(stderr, "fread err :%s\n", strerror(errno));
        return PARSE_ERR;
    }
    buf[9] = '\0';
    if (memcmp(buf, "REDIS", 5) != 0) {
        fclose(fp);
        fprintf(stderr, "Wrong signature trying to load DB from file\n");
        return PARSE_ERR;
    }
    rdb_version = atoi(buf+5);
    if (rdb_version > 6) {
        fclose(fp);
        fprintf(stderr, "Can't handle RDB format version %d\n", rdb_version);
        return PARSE_ERR;
    }

    startParse(fp);
    while(1) {
        if(!(loops++ % 1000)) {
            /* record parse progress every 1000 loops. */
            parseProgress(ftello(fp));
        }
        if((type = rdbLoadType(fp)) == -1) return PARSE_ERR;

        if(type == REDIS_EXPIRETIME) {
            if ((expiretime = rdbLoadTime(fp)) == -1) return PARSE_ERR;
            if((type = rdbLoadType(fp)) == -1) return PARSE_ERR;
        }
        /* file end. */
        if(type == REDIS_EOF) {
            break;
        }
        /* select db */
        if(type == REDIS_SELECTDB) {
            dbid = rdbLoadLen(fp,NULL);
            continue;
        }

        /* load key. */
        if ((key = rdbLoadStringObject(fp)) == NULL) {
            return PARSE_ERR;
        } 

        if(type == REDIS_HASH_ZIPMAP) {
            valType = REDIS_HASH;
        } else if(type == REDIS_LIST_ZIPLIST) {
            valType = REDIS_LIST;
        } else if(type == REDIS_SET_INTSET) {
            valType = REDIS_SET;
        } else if(type == REDIS_ZSET_ZIPLIST) {
            valType = REDIS_ZSET;
        } else {
            valType = type;
        }
        /* load value. */
        if(type == REDIS_STRING) {
            sval = rdbLoadValueObject(fp, type, &rlen);
            handler(valType, key, sval, rlen, expiretime);
        } else {
            cval = rdbLoadValueObject(fp, type, &rlen);
            handler(valType, key, cval, rlen, expiretime);
        }

        /* clean*/
        sdsfree(key);
        if(valType == REDIS_STRING) {
            sdsfree(sval);
        } else {
            unsigned int k;
            for(k = 0; k < rlen; k++) {
                sdsfree(cval[k]);
            }
            zfree(cval);
        }
    }

    /* checksum */
    uint64_t checksum = 0;
    int ret;
    ret = fread(&checksum, sizeof(checksum), 1, fp);
    if (ret == 1) {
        if ((long long)checksum != digest) {
            fprintf(stderr, "DB load failed, checksum does not match: %016llx != %016llx\n", (long long)checksum, digest);
            exit(1);
        }
        fprintf(stderr, "DB loaded, checksum: %016llx\n", digest);
    }
    parser_stats.stop_time = time(NULL);
    fclose(fp);

    return PARSE_OK;
}
Exemplo n.º 8
0
static void* rdbLoadValueObject(FILE *fp, int type, unsigned int *rlen) {
    unsigned int i, j, len;
    int buf_len;
    sds ele;
    sds *results = NULL;
    char buf[128];

    if(type == REDIS_STRING) {
        /* value type is string. */
        parser_stats.parse_num[STRING] += 1;
        ele = rdbLoadEncodedStringObject(fp);
        *rlen = sdslen(ele);
        return ele;

    } else if(type == REDIS_LIST) {
        /* value type is list. */
        parser_stats.parse_num[LIST] += 1;
        if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
        j = 0;
        *rlen = len;
        results = zmalloc(len * sizeof(*results));
        while(len--) {
            if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
            results[j++] = ele;
        }
        return results;

    } else if(type == REDIS_SET) {
        /* value type is set. */
        parser_stats.parse_num[SET] += 1;
        if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
        *rlen = len;
        results = zmalloc(len * sizeof(*results));
        for (i = 0; i < len; i++) {
            if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
            results[i] = ele; 
        }
        return results;

    } else if (type == REDIS_ZSET) {
        /* value type is zset */
        parser_stats.parse_num[ZSET] += 1;
        size_t zsetlen;
        double score;
        j = 0;    
        if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;

        if(rdb_version < 2) {
            *rlen = zsetlen * 2;
        } else {
            *rlen = zsetlen;
        }
        results = zmalloc( *rlen * sizeof(*results));
        while(zsetlen--) {
            if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
            if (rdbLoadDoubleValue(fp,&score) == -1) return NULL;
            buf_len = snprintf(buf, 128, "%f", score);
            results[j] = ele;
            results[j+1] = sdsnewlen(buf, buf_len);
            j += 2;
        }
        return results;

    } else if (type == REDIS_HASH) {
        /* value type is hash */
        parser_stats.parse_num[HASH] += 1;
        size_t hashlen;
        if ((hashlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
        sds key, val;
        j = 0;
        *rlen = hashlen * 2;
        results = zmalloc(*rlen * sizeof(*results));
        while(hashlen--) {
            if ((key = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
            if ((val = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
            results[j] = key;
            results[j + 1] = val;
            j += 2;
        }
        return results;

    } else if(type == REDIS_HASH_ZIPMAP || 
            type == REDIS_LIST_ZIPLIST ||
            type == REDIS_SET_INTSET ||
            type == REDIS_ZSET_ZIPLIST ||
            type == REDIS_LSET) {
        sds aux = rdbLoadStringObject(fp); 
        switch(type) {
            case REDIS_HASH_ZIPMAP:
                parser_stats.parse_num[HASH] += 1;
                results = loadHashZipMapObject((unsigned char*)aux, rlen);
                break;
            case REDIS_LIST_ZIPLIST:
                parser_stats.parse_num[LIST] += 1;
                results = loadListZiplistObject((unsigned char *)aux, rlen);
                break;
            case REDIS_SET_INTSET:
                parser_stats.parse_num[SET] += 1;
                results = loadSetIntsetObject((unsigned char *)aux, rlen);
                break;
            case REDIS_ZSET_ZIPLIST:
                parser_stats.parse_num[ZSET] += 1;
                results = loadZsetZiplistObject((unsigned char *)aux, rlen);
                break;
        }
        sdsfree(aux);
        return results;
    } else {
        parsePanic("Unknown object type");
    }
}