/* Emit the commands needed to rebuild a S3 object. * The function returns 0 on error, 1 on success */ int rewriteSssObject(rio *r, robj *key, robj *o) { sssTypeIterator *si; sssEntry *se; si = sssTypeInitIterator(o); while ((se = sssIterNext(si)) != NULL) { robj *ks, *svc, *skey, *val; long long idx, vll; int kv_mode; char *cmd; int cmdlen; if (sssIterPeek(se, &ks, &svc, &skey, &idx, &val, &vll, &kv_mode) == REDIS_ERR) return 0; if(kv_mode == SSS_KV_LIST) { cmd = "S3LADDAT"; cmdlen = 8; } else { redisAssert(kv_mode == SSS_KV_SET); cmd = "S3SADDAT"; cmdlen = 8; } if (rioWriteBulkCount(r, '*', 7) == 0) return 0; if (rioWriteBulkString(r, cmd, cmdlen) == 0) return 0; if (rioWriteBulkObject(r, ks) == 0) return 0; if (rioWriteBulkObject(r, key) == 0) return 0; if (rioWriteBulkObject(r, svc) == 0) return 0; if (rioWriteBulkObject(r, skey) == 0) return 0; /* idx is not used */ if (rioWriteBulkObject(r, val) == 0) return 0; if (rioWriteBulkLongLong(r, vll) == 0) return 0; } sssTypeReleaseIterator(si); return 1; }
/* Emit the commands needed to rebuild a list object. * The function returns 0 on error, 1 on success. */ int rewriteListObject(rio *r, robj *key, robj *o) { long long count = 0, items = listTypeLength(o); if (o->encoding == REDIS_ENCODING_ZIPLIST) { unsigned char *zl = o->ptr; unsigned char *p = ziplistIndex(zl,0); unsigned char *vstr; unsigned int vlen; long long vlong; while(ziplistGet(p,&vstr,&vlen,&vlong)) { if (count == 0) { int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; } if (vstr) { if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0; } else { if (rioWriteBulkLongLong(r,vlong) == 0) return 0; } p = ziplistNext(zl,p); if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { list *list = o->ptr; listNode *ln; listIter li; listRewind(list,&li); while((ln = listNext(&li))) { robj *eleobj = listNodeValue(ln); if (count == 0) { int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; } if (rioWriteBulkObject(r,eleobj) == 0) return 0; if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } } else { redisPanic("Unknown list encoding"); } return 1; }
/* Delegate writing an object to writing a bulk string or bulk long long. * This is not placed in rio.c since that adds the redis.h dependency. */ int rioWriteBulkObject(rio *r, robj *obj) { /* Avoid using getDecodedObject to help copy-on-write (we are often * in a child process when this function is called). */ if (obj->encoding == REDIS_ENCODING_INT) { return rioWriteBulkLongLong(r,(long)obj->ptr); } else if (obj->encoding == REDIS_ENCODING_RAW) { return rioWriteBulkString(r,obj->ptr,sdslen(obj->ptr)); } else { redisPanic("Unknown string encoding"); } }
/* Emit the commands needed to rebuild a set object. * The function returns 0 on error, 1 on success. */ int rewriteSetObject(rio *r, robj *key, robj *o) { long long count = 0, items = setTypeSize(o); if (o->encoding == REDIS_ENCODING_INTSET) { int ii = 0; int64_t llval; while(intsetGet(o->ptr,ii++,&llval)) { if (count == 0) { int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; if (rioWriteBulkString(r,"SADD",4) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; } if (rioWriteBulkLongLong(r,llval) == 0) return 0; if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } } else if (o->encoding == REDIS_ENCODING_HT) { dictIterator *di = dictGetIterator(o->ptr); dictEntry *de; while((de = dictNext(di)) != NULL) { robj *eleobj = dictGetKey(de); if (count == 0) { int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; if (rioWriteBulkString(r,"SADD",4) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; } if (rioWriteBulkObject(r,eleobj) == 0) return 0; if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } dictReleaseIterator(di); } else { redisPanic("Unknown set encoding"); } return 1; }
/* Write either the key or the value of the currently selected item of an hash. * The 'hi' argument passes a valid Redis hash iterator. * The 'what' filed specifies if to write a key or a value and can be * either REDIS_HASH_KEY or REDIS_HASH_VALUE. * * The function returns 0 on error, non-zero on success. */ static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) { if (hi->encoding == REDIS_ENCODING_ZIPLIST) { unsigned char *vstr = NULL; unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll); if (vstr) { return rioWriteBulkString(r, (char*)vstr, vlen); } else { return rioWriteBulkLongLong(r, vll); } } else if (hi->encoding == REDIS_ENCODING_HT) { robj *value; hashTypeCurrentFromHashTable(hi, what, &value); return rioWriteBulkObject(r, value); } redisPanic("Unknown hash encoding"); return 0; }
/* Write a sequence of commands able to fully rebuild the dataset into * "filename". Used both by REWRITEAOF and BGREWRITEAOF. * * In order to minimize the number of commands needed in the rewritten * log Redis uses variadic commands when possible, such as RPUSH, SADD * and ZADD. However at max REDIS_AOF_REWRITE_ITEMS_PER_CMD items per time * are inserted using a single command. */ int rewriteAppendOnlyFile(char *filename) { dictIterator *di = NULL; dictEntry *de; rio aof; FILE *fp; char tmpfile[256]; int j; long long now = mstime(); /* Note that we have to use a different temp name here compared to the * one used by rewriteAppendOnlyFileBackground() function. */ snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid()); fp = fopen(tmpfile,"w"); if (!fp) { redisLog(REDIS_WARNING, "Opening the temp file for AOF rewrite in rewriteAppendOnlyFile(): %s", strerror(errno)); return REDIS_ERR; } rioInitWithFile(&aof,fp); for (j = 0; j < server.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; redisDb *db = server.db+j; dict *d = db->dict; if (dictSize(d) == 0) continue; di = dictGetSafeIterator(d); if (!di) { fclose(fp); return REDIS_ERR; } /* SELECT the new DB */ if (rioWrite(&aof,selectcmd,sizeof(selectcmd)-1) == 0) goto werr; if (rioWriteBulkLongLong(&aof,j) == 0) goto werr; /* Iterate this DB writing every entry */ while((de = dictNext(di)) != NULL) { sds keystr; robj key, *o; long long expiretime; keystr = dictGetKey(de); o = dictGetVal(de); initStaticStringObject(key,keystr); expiretime = getExpire(db,&key); /* Save the key and associated value */ if (o->type == REDIS_STRING) { /* Emit a SET command */ char cmd[]="*3\r\n$3\r\nSET\r\n"; if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; /* Key and value */ if (rioWriteBulkObject(&aof,&key) == 0) goto werr; if (rioWriteBulkObject(&aof,o) == 0) goto werr; } else if (o->type == REDIS_LIST) { if (rewriteListObject(&aof,&key,o) == 0) goto werr; } else if (o->type == REDIS_SET) { if (rewriteSetObject(&aof,&key,o) == 0) goto werr; } else if (o->type == REDIS_ZSET) { if (rewriteSortedSetObject(&aof,&key,o) == 0) goto werr; } else if (o->type == REDIS_HASH) { if (rewriteHashObject(&aof,&key,o) == 0) goto werr; } else { redisPanic("Unknown object type"); } /* Save the expire time */ if (expiretime != -1) { char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n"; /* If this key is already expired skip it */ if (expiretime < now) continue; if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; if (rioWriteBulkObject(&aof,&key) == 0) goto werr; if (rioWriteBulkLongLong(&aof,expiretime) == 0) goto werr; } } dictReleaseIterator(di); } /* Make sure data will not remain on the OS's output buffers */ fflush(fp); aof_fsync(fileno(fp)); fclose(fp); /* Use RENAME to make sure the DB file is changed atomically only * if the generate DB file is ok. */ if (rename(tmpfile,filename) == -1) { redisLog(REDIS_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno)); unlink(tmpfile); return REDIS_ERR; } redisLog(REDIS_NOTICE,"SYNC append only file rewrite performed"); return REDIS_OK; werr: fclose(fp); unlink(tmpfile); redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); if (di) dictReleaseIterator(di); return REDIS_ERR; }
/* Emit the commands needed to rebuild a sorted set object. * The function returns 0 on error, 1 on success. */ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { long long count = 0, items = zsetLength(o); if (o->encoding == REDIS_ENCODING_ZIPLIST) { unsigned char *zl = o->ptr; unsigned char *eptr, *sptr; unsigned char *vstr; unsigned int vlen; long long vll; double score; eptr = ziplistIndex(zl,0); redisAssert(eptr != NULL); sptr = ziplistNext(zl,eptr); redisAssert(sptr != NULL); while (eptr != NULL) { redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll)); score = zzlGetScore(sptr); if (count == 0) { int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; } if (rioWriteBulkDouble(r,score) == 0) return 0; if (vstr != NULL) { if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0; } else { if (rioWriteBulkLongLong(r,vll) == 0) return 0; } zzlNext(zl,&eptr,&sptr); if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } } else if (o->encoding == REDIS_ENCODING_SKIPLIST) { zset *zs = o->ptr; dictIterator *di = dictGetIterator(zs->dict); dictEntry *de; while((de = dictNext(di)) != NULL) { robj *eleobj = dictGetKey(de); double *score = dictGetVal(de); if (count == 0) { int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; } if (rioWriteBulkDouble(r,*score) == 0) return 0; if (rioWriteBulkObject(r,eleobj) == 0) return 0; if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } dictReleaseIterator(di); } else { redisPanic("Unknown sorted zset encoding"); } return 1; }
static int slotsmgrt(redisClient *c, sds host, sds port, int fd, int dbid, int timeout, robj *keys[], robj *vals[], int n) { rio cmd; rioInitWithBuffer(&cmd, sdsempty()); redisAssertWithInfo(c, NULL, rioWriteBulkCount(&cmd, '*', 2)); redisAssertWithInfo(c, NULL, rioWriteBulkString(&cmd, "SELECT", 6)); redisAssertWithInfo(c, NULL, rioWriteBulkLongLong(&cmd, dbid)); redisAssertWithInfo(c, NULL, rioWriteBulkCount(&cmd, '*', 1 + 3 * n)); redisAssertWithInfo(c, NULL, rioWriteBulkString(&cmd, "SLOTSRESTORE", 12)); sds onekey = NULL; for (int i = 0; i < n; i ++) { robj *key = keys[i], *val = vals[i]; long long ttl = 0, expireat = getExpire(c->db, key); if (expireat != -1) { ttl = expireat - mstime(); if (ttl < 1) { ttl = 1; } } sds skey = key->ptr; redisAssertWithInfo(c, NULL, rioWriteBulkString(&cmd, skey, sdslen(skey))); redisAssertWithInfo(c, NULL, rioWriteBulkLongLong(&cmd, ttl)); do { rio pld; createDumpPayload(&pld, val); sds buf = pld.io.buffer.ptr; redisAssertWithInfo(c, NULL, rioWriteBulkString(&cmd, buf, sdslen(buf))); sdsfree(buf); } while (0); if (onekey == NULL) { onekey = skey; } } do { sds buf = cmd.io.buffer.ptr; size_t pos = 0, towrite; int nwritten = 0; while ((towrite = sdslen(buf) - pos) > 0) { towrite = (towrite > (64 * 1024) ? (64 * 1024) : towrite); nwritten = syncWrite(fd, buf + pos, towrite, timeout); if (nwritten != (signed)towrite) { redisLog(REDIS_WARNING, "slotsmgrt: writing to target %s:%s, error '%s', " "nkeys = %d, onekey = '%s', cmd.len = %ld, pos = %ld, towrite = %ld", host, port, server.neterr, n, onekey, sdslen(buf), pos, towrite); addReplySds(c, sdsnew("-IOERR error or timeout writing to target\r\n")); sdsfree(buf); return -1; } pos += nwritten; } sdsfree(buf); } while (0); do { char buf1[1024]; char buf2[1024]; if (syncReadLine(fd, buf1, sizeof(buf1), timeout) <= 0 || syncReadLine(fd, buf2, sizeof(buf2), timeout) <= 0) { redisLog(REDIS_WARNING, "slotsmgrt: reading from target %s:%s, error '%s', nkeys = %d, onekey = '%s'", host, port, server.neterr, n, onekey); addReplySds(c, sdsnew("-IOERR error or timeout reading from target\r\n")); return -1; } if (buf1[0] == '-' || buf2[0] == '-') { redisLog(REDIS_WARNING, "slotsmgrt: response from target %s:%s: rsp1 = '%s', rsp2 = '%s', nkeys = %d, onekey = '%s'", host, port, buf1, buf2, n, onekey); addReplyError(c, "error on slotsrestore"); return -1; } } while (0); redisLog(REDIS_VERBOSE, "slotsmgrt: migrate to %s:%s, nkeys = %d, onekey = '%s'", host, port, n, onekey); return 0; }