char *loadIntegerObject(int enctype) {
uint32_t offset = CURR_OFFSET;
unsigned char enc[4];
long long val;
char *buf;
if (enctype == REDIS_RDB_ENC_INT8) {
uint8_t v;
if (!readBytes(enc, 1)) return NULL;
v = enc[0];
val = (int8_t)v;
} else if (enctype == REDIS_RDB_ENC_INT16) {
uint16_t v;
if (!readBytes(enc, 2)) return NULL;
v = enc[0]|(enc[1]<<8);
val = (int16_t)v;
} else if (enctype == REDIS_RDB_ENC_INT32) {
uint32_t v;
if (!readBytes(enc, 4)) return NULL;
v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24);
val = (int32_t)v;
} else {
SHIFT_ERROR(offset, "Unknown integer encoding (0x%02x)", enctype);
return NULL;
}
/* convert val into string */
buf = malloc(sizeof(char) * 128);
sprintf(buf, "%lld", val);
return buf;
}
/* returns NULL when not processable, char* when valid */
char* loadStringObject() {
uint32_t offset = CURR_OFFSET;
int isencoded;
uint32_t len;
char *buf;
len = loadLength(&isencoded);
if (isencoded) {
switch(len) {
case REDIS_RDB_ENC_INT8:
case REDIS_RDB_ENC_INT16:
case REDIS_RDB_ENC_INT32:
return loadIntegerObject(len);
case REDIS_RDB_ENC_LZF:
return loadLzfStringObject();
default:
/* unknown encoding */
SHIFT_ERROR(offset, "Unknown string encoding (0x%02x)", len);
return NULL;
}
}
if (len == REDIS_RDB_LENERR) return NULL;
buf = malloc(sizeof(char) * (len+1));
buf[len] = '\0';
if (!readBytes(buf, len)) {
free(buf);
return NULL;
}
return buf;
}
int loadType(entry *e) {
uint32_t offset = CURR_OFFSET;
/* this byte needs to qualify as type */
unsigned char t;
if (readBytes(&t, 1)) {
if (checkType(t)) {
e->type = t;
return 1;
} else {
SHIFT_ERROR(offset, "Unknown type (0x%02x)", t);
}
} else {
SHIFT_ERROR(offset, "Could not read type");
}
/* failure */
return 0;
}
/* discard time, just consume the bytes */
int processTime() {
uint32_t offset = CURR_OFFSET;
unsigned char t[4];
if (readBytes(t, 4)) {
return 1;
} else {
SHIFT_ERROR(offset, "Could not read time");
}
/* failure */
return 0;
}
entry loadEntry() {
entry e = { NULL, -1, 0 };
uint32_t length, offset[4];
/* reset error container */
errors.level = 0;
offset[0] = CURR_OFFSET;
if (!loadType(&e)) {
return e;
}
offset[1] = CURR_OFFSET;
if (e.type == REDIS_SELECTDB) {
if ((length = loadLength(NULL)) == REDIS_RDB_LENERR) {
SHIFT_ERROR(offset[1],
"Error reading database number");
return e;
}
if (length > 63) {
SHIFT_ERROR(offset[1],
"Database number out of range (%d)", length);
return e;
}
} else if (e.type == REDIS_EOF) {
if (positions[level].offset < positions[level].size) {
SHIFT_ERROR(offset[0],
"Unexpected EOF");
} else {
e.success = 1;
}
return e;
} else {
/* optionally consume expire */
if (e.type == REDIS_EXPIRETIME) {
if (!processTime()) return e;
if (!loadType(&e)) return e;
db_stats.total_expires++;
}
offset[1] = CURR_OFFSET;
if (!loadPair(&e)) {
SHIFT_ERROR(offset[1], "Error for type %s", types[e.type]);
return e;
}
}
/* all entries are followed by a valid type:
* e.g. a new entry, SELECTDB, EXPIRE, EOF */
offset[2] = CURR_OFFSET;
if (peekType() == -1) {
SHIFT_ERROR(offset[2], "Followed by invalid type");
SHIFT_ERROR(offset[0], "Error for type %s", types[e.type]);
e.success = 0;
} else {
e.success = 1;
}
return e;
}
/* discard time, just consume the bytes */
int processTime(int type) {
uint32_t offset = CURR_OFFSET;
unsigned char t[8];
int timelen = (type == REDIS_EXPIRETIME_MS) ? 8 : 4;
if (readBytes(t,timelen)) {
return 1;
} else {
SHIFT_ERROR(offset, "Could not read time");
}
/* failure */
return 0;
}
int processDoubleValue(double** store) {
unsigned long offset = CURR_OFFSET;
double *val = loadDoubleValue();
if (val == NULL) {
SHIFT_ERROR(offset, "Error reading double value");
free(val);
return 0;
}
if (store != NULL) {
*store = val;
} else {
free(val);
}
return 1;
}
int processStringObject(char** store) {
unsigned long offset = CURR_OFFSET;
char *key = loadStringObject();
if (key == NULL) {
SHIFT_ERROR(offset, "Error reading string object");
free(key);
return 0;
}
if (store != NULL) {
*store = key;
} else {
free(key);
}
return 1;
}
void process() {
uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
int dump_version = processHeader();
/* Exclude the final checksum for RDB >= 5. Will be checked at the end. */
if (dump_version >= 5) {
if (positions[0].size < 8) {
printf("RDB version >= 5 but no room for checksum.\n");
exit(1);
}
positions[0].size -= 8;;
}
level = 1;
while(positions[0].offset < positions[0].size) {
positions[1] = positions[0];
entry = loadEntry();
if (!entry.success) {
uint64_t offset;
int i;
printValid(num_valid_ops, num_valid_bytes);
printErrorStack(&entry);
num_errors++;
num_valid_ops = 0;
num_valid_bytes = 0;
/* search for next valid entry */
offset = positions[0].offset + 1;
i = 0;
while (!entry.success && offset < positions[0].size) {
positions[1].offset = (size_t)offset;
/* find 3 consecutive valid entries */
for (i = 0; i < 3; i++) {
entry = loadEntry();
if (!entry.success) break;
}
/* check if we found 3 consecutive valid entries */
if (i < 3) {
offset++;
}
}
/* print how many bytes we have skipped to find a new valid opcode */
if (offset < positions[0].size) {
printSkipped(offset - positions[0].offset, offset);
}
positions[0].offset = (size_t)offset;
} else {
num_valid_ops++;
num_valid_bytes += positions[1].offset - positions[0].offset;
/* advance position */
positions[0] = positions[1];
}
free(entry.key);
}
/* because there is another potential error,
* print how many valid ops we have processed */
printValid(num_valid_ops, num_valid_bytes);
/* expect an eof */
if (entry.type != REDIS_EOF) {
/* last byte should be EOF, add error */
errors.level = 0;
SHIFT_ERROR(positions[0].offset, "Expected EOF, got %s", types[entry.type]);
/* this is an EOF error so reset type */
entry.type = -1;
printErrorStack(&entry);
num_errors++;
}
/* Verify checksum */
if (dump_version >= 5) {
uint64_t crc = crc64(0,positions[0].data,positions[0].size);
uint64_t crc2;
unsigned char *p = (unsigned char*)positions[0].data+positions[0].size;
crc2 = ((uint64_t)p[0] << 0) |
((uint64_t)p[1] << 8) |
((uint64_t)p[2] << 16) |
((uint64_t)p[3] << 24) |
((uint64_t)p[4] << 32) |
((uint64_t)p[5] << 40) |
((uint64_t)p[6] << 48) |
((uint64_t)p[7] << 56);
if (crc != crc2) {
SHIFT_ERROR(positions[0].offset, "RDB CRC64 does not match.");
} else {
printf("CRC64 checksum is OK\n");
}
}
/* print summary on errors */
if (num_errors) {
printf("\n");
printf("Total unprocessable opcodes: %llu\n",
(unsigned long long) num_errors);
}
}
int loadPair(entry *e) {
uint32_t offset = CURR_OFFSET;
uint32_t i;
uint32_t length = 0;
/* read key first */
char *key;
if (processStringObject(&key)) {
e->key = key;
} else {
SHIFT_ERROR(offset, "Error reading entry key");
return 0;
}
if (e->type == REDIS_LIST ||
e->type == REDIS_SET ||
e->type == REDIS_ZSET ||
e->type == REDIS_HASH) {
if ((length = loadLength(NULL)) == REDIS_RDB_LENERR) {
SHIFT_ERROR(offset, "Error reading %s length", types[e->type]);
return 0;
}
}
switch(e->type) {
case REDIS_STRING:
case REDIS_HASH_ZIPMAP:
case REDIS_LIST_ZIPLIST:
case REDIS_SET_INTSET:
case REDIS_ZSET_ZIPLIST:
case REDIS_HASH_ZIPLIST:
if (!processStringObject(NULL)) {
SHIFT_ERROR(offset, "Error reading entry value");
return 0;
}
break;
case REDIS_LIST:
case REDIS_SET:
for (i = 0; i < length; i++) {
offset = CURR_OFFSET;
if (!processStringObject(NULL)) {
SHIFT_ERROR(offset, "Error reading element at index %d (length: %d)", i, length);
return 0;
}
}
break;
case REDIS_ZSET:
for (i = 0; i < length; i++) {
offset = CURR_OFFSET;
if (!processStringObject(NULL)) {
SHIFT_ERROR(offset, "Error reading element key at index %d (length: %d)", i, length);
return 0;
}
offset = CURR_OFFSET;
if (!processDoubleValue(NULL)) {
SHIFT_ERROR(offset, "Error reading element value at index %d (length: %d)", i, length);
return 0;
}
}
break;
case REDIS_HASH:
for (i = 0; i < length; i++) {
offset = CURR_OFFSET;
if (!processStringObject(NULL)) {
SHIFT_ERROR(offset, "Error reading element key at index %d (length: %d)", i, length);
return 0;
}
offset = CURR_OFFSET;
if (!processStringObject(NULL)) {
SHIFT_ERROR(offset, "Error reading element value at index %d (length: %d)", i, length);
return 0;
}
}
break;
default:
SHIFT_ERROR(offset, "Type not implemented");
return 0;
}
/* because we're done, we assume success */
e->success = 1;
return 1;
}
void process() {
uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
processHeader();
level = 1;
while(positions[0].offset < positions[0].size) {
positions[1] = positions[0];
entry = loadEntry();
if (!entry.success) {
printValid(num_valid_ops, num_valid_bytes);
printErrorStack(&entry);
num_errors++;
num_valid_ops = 0;
num_valid_bytes = 0;
/* search for next valid entry */
uint64_t offset = positions[0].offset + 1;
int i = 0;
while (!entry.success && offset < positions[0].size) {
positions[1].offset = offset;
/* find 3 consecutive valid entries */
for (i = 0; i < 3; i++) {
entry = loadEntry();
if (!entry.success) break;
}
/* check if we found 3 consecutive valid entries */
if (i < 3) {
offset++;
}
}
/* print how many bytes we have skipped to find a new valid opcode */
if (offset < positions[0].size) {
printSkipped(offset - positions[0].offset, offset);
}
positions[0].offset = offset;
} else {
num_valid_ops++;
num_valid_bytes += positions[1].offset - positions[0].offset;
/* advance position */
positions[0] = positions[1];
}
}
/* because there is another potential error,
* print how many valid ops we have processed */
//printValid(num_valid_ops, num_valid_bytes);
db_stats.num_valid_ops = num_valid_ops;
db_stats.num_valid_bytes = num_valid_bytes;
/* expect an eof */
if (entry.type != REDIS_EOF) {
/* last byte should be EOF, add error */
errors.level = 0;
SHIFT_ERROR(positions[0].offset,
"Expected EOF, got %s", types[entry.type]);
/* this is an EOF error so reset type */
entry.type = -1;
printErrorStack(&entry);
num_errors++;
}
/* print summary on errors */
if (num_errors) {
printf("\n");
printf("Total unprocessable opcodes: %llu\n",
(unsigned long long) num_errors);
}
}
