/* Replay the append log file. On error REDIS_OK is returned. On non fatal * error (the append only file is zero-length) REDIS_ERR is returned. On * fatal error an error message is logged and the program exists. */ int loadAppendOnlyFile(char *filename) { struct redisClient *fakeClient; FILE *fp = fopen(filename,"r"); struct redis_stat sb; int old_aof_state = server.aof_state; long loops = 0; if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) { server.aof_current_size = 0; fclose(fp); return REDIS_ERR; } if (fp == NULL) { redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno)); exit(1); } /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI * to the same file we're about to read. */ server.aof_state = REDIS_AOF_OFF; fakeClient = createFakeClient(); startLoading(fp); while(1) { int argc, j; unsigned long len; robj **argv; char buf[128]; sds argsds; struct redisCommand *cmd; /* Serve the clients from time to time */ if (!(loops++ % 1000)) { loadingProgress(ftello(fp)); aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); } if (fgets(buf,sizeof(buf),fp) == NULL) { if (feof(fp)) break; else goto readerr; } if (buf[0] != '*') goto fmterr; argc = atoi(buf+1); if (argc < 1) goto fmterr; argv = zmalloc(sizeof(robj*)*argc); for (j = 0; j < argc; j++) { if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr; if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); argsds = sdsnewlen(NULL,len); if (len && fread(argsds,len,1,fp) == 0) goto fmterr; argv[j] = createObject(REDIS_STRING,argsds); if (fread(buf,2,1,fp) == 0) goto fmterr; /* discard CRLF */ } /* Command lookup */ cmd = lookupCommand(argv[0]->ptr); if (!cmd) { redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr); exit(1); } /* Run the command in the context of a fake client */ fakeClient->argc = argc; fakeClient->argv = argv; cmd->proc(fakeClient); /* The fake client should not have a reply */ redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0); /* The fake client should never get blocked */ redisAssert((fakeClient->flags & REDIS_BLOCKED) == 0); /* Clean up. Command code may have changed argv/argc so we use the * argv/argc of the client instead of the local variables. */ for (j = 0; j < fakeClient->argc; j++) decrRefCount(fakeClient->argv[j]); zfree(fakeClient->argv); } /* This point can only be reached when EOF is reached without errors. * If the client is in the middle of a MULTI/EXEC, log error and quit. */ if (fakeClient->flags & REDIS_MULTI) goto readerr; fclose(fp); freeFakeClient(fakeClient); server.aof_state = old_aof_state; stopLoading(); aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; return REDIS_OK; readerr: if (feof(fp)) { redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file"); } else { redisLog(REDIS_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); } exit(1); fmterr: redisLog(REDIS_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>"); exit(1); }
int redis_check_aof_main(int argc, char **argv) { char *filename; int fix = 0; if (argc < 2) { printf("Usage: %s [--fix] <file.aof>\n", argv[0]); exit(1); } else if (argc == 2) { filename = argv[1]; } else if (argc == 3) { if (strcmp(argv[1],"--fix") != 0) { printf("Invalid argument: %s\n", argv[1]); exit(1); } filename = argv[2]; fix = 1; } else { printf("Invalid arguments\n"); exit(1); } FILE *fp = fopen(filename,"r+"); if (fp == NULL) { printf("Cannot open file: %s\n", filename); exit(1); } struct redis_stat sb; if (redis_fstat(fileno(fp),&sb) == -1) { printf("Cannot stat file: %s\n", filename); exit(1); } off_t size = sb.st_size; if (size == 0) { printf("Empty file: %s\n", filename); exit(1); } /* This AOF file may have an RDB preamble. Check this to start, and if this * is the case, start processing the RDB part. */ if (size >= 8) { /* There must be at least room for the RDB header. */ char sig[5]; int has_preamble = fread(sig,sizeof(sig),1,fp) == 1 && memcmp(sig,"REDIS",sizeof(sig)) == 0; rewind(fp); if (has_preamble) { printf("The AOF appears to start with an RDB preamble.\n" "Checking the RDB preamble to start:\n"); if (redis_check_rdb_main(argc,argv,fp) == C_ERR) { printf("RDB preamble of AOF file is not sane, aborting.\n"); exit(1); } else { printf("RDB preamble is OK, proceeding with AOF tail...\n"); } } } off_t pos = process(fp); off_t diff = size-pos; printf("AOF analyzed: size=%lld, ok_up_to=%lld, diff=%lld\n", (long long) size, (long long) pos, (long long) diff); if (diff > 0) { if (fix) { char buf[2]; printf("This will shrink the AOF from %lld bytes, with %lld bytes, to %lld bytes\n",(long long)size,(long long)diff,(long long)pos); printf("Continue? [y/N]: "); if (fgets(buf,sizeof(buf),stdin) == NULL || strncasecmp(buf,"y",1) != 0) { printf("Aborting...\n"); exit(1); } if (ftruncate(fileno(fp), pos) == -1) { printf("Failed to truncate AOF\n"); exit(1); } else { printf("Successfully truncated AOF\n"); } } else { printf("AOF is not valid. " "Use the --fix option to try fixing it.\n"); exit(1); } } else { printf("AOF is valid\n"); } fclose(fp); exit(0); }
/* Replay the append log file. On error REDIS_OK is returned. On non fatal * error (the append only file is zero-length) REDIS_ERR is returned. On * fatal error an error message is logged and the program exists. */ int loadAppendOnlyFile(char *filename) { struct redisClient *fakeClient; FILE *fp = fopen(filename,"r"); struct redis_stat sb; int old_aof_state = server.aof_state; long loops = 0; if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) { server.aof_current_size = 0; fclose(fp); return REDIS_ERR; } if (fp == NULL) { redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno)); exit(1); } /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI * to the same file we're about to read. */ server.aof_state = REDIS_AOF_OFF; fakeClient = createFakeClient(); startLoading(fp); while(1) { int argc, j; unsigned long len; robj **argv; char buf[128]; sds argsds; struct redisCommand *cmd; /* Serve the clients from time to time */ if (!(loops++ % 1000)) { loadingProgress(ftello(fp)); processEventsWhileBlocked(); } if (fgets(buf,sizeof(buf),fp) == NULL) { if (feof(fp)) break; else goto readerr; } if (buf[0] != '*') goto fmterr; if (buf[1] == '\0') goto readerr; argc = atoi(buf+1); if (argc < 1) goto fmterr; argv = zmalloc(sizeof(robj*)*argc); fakeClient->argc = argc; fakeClient->argv = argv; for (j = 0; j < argc; j++) { if (fgets(buf,sizeof(buf),fp) == NULL) { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); goto readerr; } if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); argsds = sdsnewlen(NULL,len); if (len && fread(argsds,len,1,fp) == 0) { sdsfree(argsds); fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); goto readerr; } argv[j] = createObject(REDIS_STRING,argsds); if (fread(buf,2,1,fp) == 0) { fakeClient->argc = j+1; /* Free up to j. */ freeFakeClientArgv(fakeClient); goto readerr; /* discard CRLF */ } } /* Command lookup */ cmd = lookupCommand(argv[0]->ptr); if (!cmd) { redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", (char*)argv[0]->ptr); exit(1); } /* Run the command in the context of a fake client */ cmd->proc(fakeClient); /* The fake client should not have a reply */ redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0); /* The fake client should never get blocked */ redisAssert((fakeClient->flags & REDIS_BLOCKED) == 0); /* Clean up. Command code may have changed argv/argc so we use the * argv/argc of the client instead of the local variables. */ freeFakeClientArgv(fakeClient); } /* This point can only be reached when EOF is reached without errors. * If the client is in the middle of a MULTI/EXEC, log error and quit. */ if (fakeClient->flags & REDIS_MULTI) goto uxeof; loaded_ok: /* DB loaded, cleanup and return REDIS_OK to the caller. */ fclose(fp); freeFakeClient(fakeClient); server.aof_state = old_aof_state; stopLoading(); aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; return REDIS_OK; readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ if (!feof(fp)) { redisLog(REDIS_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); exit(1); } uxeof: /* Unexpected AOF end of file. */ if (server.aof_load_truncated) { redisLog(REDIS_WARNING,"!!! Warning: short read while loading the AOF file !!!"); redisLog(REDIS_WARNING, "AOF loaded anyway because aof-load-truncated is enabled"); goto loaded_ok; } redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename>. 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); fmterr: /* Format error. */ redisLog(REDIS_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>"); exit(1); }
int main(int argc, char **argv) { char *filename; int fix = 0; #ifdef _WIN32 _fmode = _O_BINARY; setmode(_fileno(stdin), _O_BINARY); setmode(_fileno(stdout), _O_BINARY); setmode(_fileno(stderr), _O_BINARY); #endif if (argc < 2) { printf("Usage: %s [--fix] <file.aof>\n", argv[0]); exit(1); } else if (argc == 2) { filename = argv[1]; } else if (argc == 3) { if (strcmp(argv[1],"--fix") != 0) { printf("Invalid argument: %s\n", argv[1]); exit(1); } filename = argv[2]; fix = 1; } else { printf("Invalid arguments\n"); exit(1); } FILE *fp = fopen(filename,IF_WIN32("r+b","r+")); if (fp == NULL) { printf("Cannot open file: %s\n", filename); exit(1); } struct redis_stat sb; if (redis_fstat(fileno(fp),&sb) == -1) { printf("Cannot stat file: %s\n", filename); exit(1); } off_t size = sb.st_size; if (size == 0) { printf("Empty file: %s\n", filename); exit(1); } off_t pos = process(fp); off_t diff = size-pos; printf("AOF analyzed: size=%lld, ok_up_to=%lld, diff=%lld\n", (PORT_LONGLONG) size, (PORT_LONGLONG) pos, (PORT_LONGLONG) diff); if (diff > 0) { if (fix) { char buf[2]; printf("This will shrink the AOF from %lld bytes, with %lld bytes, to %lld bytes\n",(PORT_LONGLONG)size,(PORT_LONGLONG)diff,(PORT_LONGLONG)pos); printf("Continue? [y/N]: "); if (fgets(buf,sizeof(buf),stdin) == NULL || strncasecmp(buf,"y",1) != 0) { printf("Aborting...\n"); exit(1); } if (ftruncate(fileno(fp), pos) == -1) { printf("Failed to truncate AOF\n"); exit(1); } else { printf("Successfully truncated AOF\n"); } } else { printf("AOF is not valid\n"); exit(1); } } else { printf("AOF is valid\n"); } fclose(fp); return 0; }
/* Replay the append log file. On error REDIS_OK is returned. On non fatal * error (the append only file is zero-length) REDIS_ERR is returned. On * fatal error an error message is logged and the program exists. */ int loadAppendOnlyFile(char *filename) { struct redisClient *fakeClient; FILE *fp = fopen(filename,"r"); struct redis_stat sb; unsigned long long loadedkeys = 0; if (redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) return REDIS_ERR; fakeClient = createFakeClient(); while(1) { int argc, j; unsigned long len; robj **argv; char buf[128]; sds argsds; struct redisCommand *cmd; if (fgets(buf,sizeof(buf),fp) == NULL) { if (feof(fp)) break; else goto readerr; } if (buf[0] != '*') goto fmterr; argc = atoi(buf+1); argv = zmalloc(sizeof(robj*)*argc); for (j = 0; j < argc; j++) { if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr; if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); argsds = sdsnewlen(NULL,len); if (len && fread(argsds,len,1,fp) == 0) goto fmterr; argv[j] = createObject(REDIS_STRING,argsds); if (fread(buf,2,1,fp) == 0) goto fmterr; /* discard CRLF */ } /* Command lookup */ cmd = lookupCommand(argv[0]->ptr); /* Try object sharing and encoding */ if (server.shareobjects) { int j; for(j = 1; j < argc; j++) argv[j] = tryObjectSharing(argv[j]); } if (cmd->flags & REDIS_CMD_BULK) tryObjectEncoding(argv[argc-1]); /* Run the command in the context of a fake client */ fakeClient->argc = argc; fakeClient->argv = argv; cmd->proc(fakeClient); /* Discard the reply objects list from the fake client */ while(listLength(fakeClient->reply)) listDelNode(fakeClient->reply,listFirst(fakeClient->reply)); /* Clean up, ready for the next command */ for (j = 0; j < argc; j++) decrRefCount(argv[j]); zfree(argv); /* Handle swapping while loading big datasets when VM is on */ loadedkeys++; if (server.vm_enabled && (loadedkeys % 5000) == 0) { while (zmalloc_used_memory() > server.vm_max_memory) { if (vmSwapOneObjectBlocking() == REDIS_ERR) break; } } } fclose(fp); freeFakeClient(fakeClient); return REDIS_OK; }