void file_info::info_set_int(const char * name,__int64 value)
{
assert(is_valid_utf8(name));
char temp[32];
_i64toa(value,temp,10);
info_set(name,temp);
}
void file_info::info_set_wfx_chanMask(uint32_t val) {
switch(val) {
case 0:
case 4:
case 3:
break;
default:
info_set ("WAVEFORMATEXTENSIBLE_CHANNEL_MASK", pfc::string_formatter() << "0x" << pfc::format_hex(val) );
break;
}
}
void file_info::copy(const file_info * src)
{
meta_remove_all();
info_remove_all();
set_location(src->get_location());
set_length(src->get_length());
int n;
for(n=0;n<src->meta_get_count();n++)
meta_add(src->meta_enum_name(n),src->meta_enum_value(n));
for(n=0;n<src->info_get_count();n++)
info_set(src->info_enum_name(n),src->info_enum_value(n));
}
void file_info::info_set_float(const char * name,double value,unsigned precision,bool force_sign,const char * unit)
{
assert(is_valid_utf8(name));
assert(unit==0 || strlen(unit) <= 64);
char temp[128];
pfc_float_to_string(temp,value,precision,force_sign);
if (unit)
{
strcat(temp," ");
strcat(temp,unit);
}
info_set(name,temp);
}
void file_info::merge(const pfc::list_base_const_t<const file_info*> & p_in)
{
t_size in_count = p_in.get_count();
if (in_count == 0)
{
meta_remove_all();
return;
}
else if (in_count == 1)
{
const file_info * info = p_in[0];
copy_meta(*info);
set_replaygain(replaygain_info::g_merge(get_replaygain(),info->get_replaygain()));
overwrite_info(*info);
//copy_info_single_by_name(*info,"tagtype");
return;
}
merge_meta(*this,p_in);
{
pfc::string8_fastalloc tagtype;
replaygain_info rg = get_replaygain();
t_size in_ptr;
for(in_ptr = 0; in_ptr < in_count; in_ptr++ )
{
const file_info * info = p_in[in_ptr];
rg = replaygain_info::g_merge(rg, info->get_replaygain());
t_size field_ptr, field_max = info->info_get_count();
for(field_ptr = 0; field_ptr < field_max; field_ptr++ )
{
const char * field_name = info->info_enum_name(field_ptr), * field_value = info->info_enum_value(field_ptr);
if (*field_value)
{
if (!pfc::stricmp_ascii(field_name,"tagtype"))
{
if (!tagtype.is_empty()) tagtype += "|";
tagtype += field_value;
}
}
}
}
if (!tagtype.is_empty()) info_set("tagtype",tagtype);
set_replaygain(rg);
}
}
void file_info::info_set_float(const char * name,double value,unsigned precision,bool force_sign,const char * unit)
{
PFC_ASSERT(pfc::is_valid_utf8(name));
PFC_ASSERT(unit==0 || strlen(unit) <= 64);
char temp[128];
pfc::float_to_string(temp,64,value,precision,force_sign);
temp[63] = 0;
if (unit)
{
strcat_s(temp," ");
strcat_s(temp,unit);
}
info_set(name,temp);
}
void
mgv_file_info(Widget w, XtPointer closure, XtPointer call)
{
MGV *m = (MGV *)closure;
char pages[64], *media, *type;
USEUP(w); USEUP(call);
assert(MgvOK(m));
if(m->docinfo == NULL) {
m->docinfo = wl_create(m->main, info_dlg);
wl_callback(XtParent(m->docinfo), &info_cb[0], (XtPointer)m);
}
/*
* The document information can change whenever the file
* is scanned for DSC. This can happen if we open a new
* file or re-open the current file. Since I'm a little
* lazy, I'll just update the dialog every time and avoid
* some hassles.
*/
sprintf(pages, "%d", m->dsc->n_pages);
switch(m->dsc->type) {
case DSC_PS: type = mgv_s(m->main, MgvSPS); break;
case DSC_EPS: type = mgv_s(m->main, MgvSEPS); break;
case DSC_PDF: type = mgv_s(m->main, MgvSPDF); break;
default: type = mgv_s(m->main, MgvSunknownType); break;
}
if(m->dsc->defmedia != NULL
&& m->dsc->defmedia->name != NULL)
media = m->dsc->defmedia->name;
else
media = mgv_s(m->main, MgvSunknownMedia);
info_set(m->docinfo, "*title", m->dsc->title);
info_set(m->docinfo, "*date", m->dsc->date);
info_set(m->docinfo, "*for", m->dsc->forwho);
info_set(m->docinfo, "*creator", m->dsc->creator);
info_set(m->docinfo, "*media", media);
info_set(m->docinfo, "*pages", pages);
info_set(m->docinfo, "*type", type);
wl_dlg_popup(m->docinfo);
}
static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity** parity, block_off_t blockstart, block_off_t blockmax, time_t timelimit, block_off_t countlimit, time_t now)
{
struct snapraid_handle* handle;
void* rehandle_alloc;
struct snapraid_rehash* rehandle;
unsigned diskmax;
block_off_t i;
unsigned j;
void* buffer_alloc;
void** buffer;
unsigned buffermax;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
block_off_t recountmax;
block_off_t autosavedone;
block_off_t autosavelimit;
block_off_t autosavemissing;
int ret;
unsigned error;
unsigned silent_error;
unsigned l;
/* maps the disks to handles */
handle = handle_map(state, &diskmax);
/* rehash buffers */
rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
/* we need disk + 2 for each parity level buffers */
buffermax = diskmax + state->level * 2;
buffer = malloc_nofail_vector_align(diskmax, buffermax, state->block_size, &buffer_alloc);
if (!state->opt.skip_self)
mtest_vector(buffermax, state->block_size, buffer);
error = 0;
silent_error = 0;
/* first count the number of blocks to process */
countmax = 0;
for(i=blockstart;i<blockmax;++i) {
time_t blocktime;
snapraid_info info;
/* if it's unused */
info = info_get(&state->infoarr, i);
if (info == 0) {
/* skip it */
continue;
}
/* blocks marked as bad are always checked */
if (!info_get_bad(info)) {
/* if it's too new */
blocktime = info_get_time(info);
if (blocktime > timelimit) {
/* skip it */
continue;
}
/* skip odd blocks, used only for testing */
if (state->opt.force_scrub_even && (i % 2) != 0) {
/* skip it */
continue;
}
/* if the time is less than the limit, always include */
/* otherwise, check if we reached the max count */
if (blocktime == timelimit) {
/* if we reached the count limit */
if (countmax >= countlimit) {
/* skip it */
continue;
}
}
}
++countmax;
}
/* compute the autosave size for all disk, even if not read */
/* this makes sense because the speed should be almost the same */
/* if the disks are read in parallel */
autosavelimit = state->autosave / (diskmax * state->block_size);
autosavemissing = countmax; /* blocks to do */
autosavedone = 0; /* blocks done */
countsize = 0;
countpos = 0;
state_progress_begin(state, blockstart, blockmax, countmax);
recountmax = 0;
for(i=blockstart;i<blockmax;++i) {
time_t blocktime;
snapraid_info info;
int error_on_this_block;
int silent_error_on_this_block;
int block_is_unsynced;
int rehash;
/* if it's unused */
info = info_get(&state->infoarr, i);
if (info == 0) {
/* skip it */
continue;
}
/* blocks marked as bad are always checked */
if (!info_get_bad(info)) {
/* if it's too new */
blocktime = info_get_time(info);
if (blocktime > timelimit) {
/* skip it */
continue;
}
/* skip odd blocks, used only for testing */
if (state->opt.force_scrub_even && (i % 2) != 0) {
/* skip it */
continue;
}
/* if the time is less than the limit, always include */
/* otherwise, check if we reaced the count max */
if (blocktime == timelimit) {
/* if we reached the count limit */
if (recountmax >= countlimit) {
/* skip it */
continue;
}
}
}
++recountmax;
/* one more block processed for autosave */
++autosavedone;
--autosavemissing;
/* by default process the block, and skip it if something goes wrong */
error_on_this_block = 0;
silent_error_on_this_block = 0;
/* if all the blocks at this address are synced */
/* if not, parity is not even checked */
block_is_unsynced = 0;
/* if we have to use the old hash */
rehash = info_get_rehash(info);
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
int file_is_unsynced;
/* if the file on this disk is synced */
/* if not, silent errors are assumed as expected error */
file_is_unsynced = 0;
/* by default no rehash in case of "continue" */
rehandle[j].block = 0;
/* if the disk position is not used */
if (!handle[j].disk) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* if the block is not used */
block = disk_block_get(handle[j].disk, i);
if (!block_has_file(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* if the block is unsynced, errors are expected */
if (block_has_invalid_parity(block)) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* if the file is different than the current one, close it */
if (handle[j].file != 0 && handle[j].file != block_file_get(block)) {
/* keep a pointer at the file we are going to close for error reporting */
struct snapraid_file* file = handle[j].file;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
fprintf(stdlog, "error:%u:%s:%s: Close error. %s\n", i, handle[j].disk->name, file->sub, strerror(errno));
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to scrub.\n");
printf("Stopping at block %u\n", i);
++error;
goto bail;
/* LCOV_EXCL_STOP */
}
}
ret = handle_open(&handle[j], block_file_get(block), state->opt.skip_sequential, stderr);
if (ret == -1) {
/* file we have tried to open for error reporting */
struct snapraid_file* file = block_file_get(block);
fprintf(stdlog, "error:%u:%s:%s: Open error. %s\n", i, handle[j].disk->name, file->sub, strerror(errno));
++error;
error_on_this_block = 1;
continue;
}
/* check if the file is changed */
if (handle[j].st.st_size != block_file_get(block)->size
|| handle[j].st.st_mtime != block_file_get(block)->mtime_sec
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
|| handle[j].st.st_ino != block_file_get(block)->inode
) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* note that we intentionally don't abort if the file has different attributes */
/* from the last sync, as we are expected to return errors if running */
/* in an unsynced array. This is just like the check command. */
read_size = handle_read(&handle[j], block, buffer[j], state->block_size, stderr);
if (read_size == -1) {
fprintf(stdlog, "error:%u:%s:%s: Read error at position %u\n", i, handle[j].disk->name, handle[j].file->sub, block_file_pos(block));
++error;
error_on_this_block = 1;
continue;
}
countsize += read_size;
/* now compute the hash */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
/* compute the new hash, and store it */
rehandle[j].block = block;
memhash(state->hash, state->hashseed, rehandle[j].hash, buffer[j], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
}
if (block_has_updated_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, handle[j].file->sub, block_file_pos(block));
/* it's a silent error only if we are dealing with synced files */
if (file_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
fprintf(stderr, "Data error in file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
fprintf(stderr, "WARNING! Unexpected data error in a data disk! The block is now marked as bad!\n");
fprintf(stderr, "Try with 'snapraid -e fix' to recover!\n");
++silent_error;
silent_error_on_this_block = 1;
}
continue;
}
}
}
/* if we have read all the data required and it's correct, proceed with the parity check */
if (!error_on_this_block && !silent_error_on_this_block) {
unsigned char* buffer_recov[LEV_MAX];
/* buffers for parity read and not computed */
for(l=0;l<state->level;++l)
buffer_recov[l] = buffer[diskmax + state->level + l];
for(;l<LEV_MAX;++l)
buffer_recov[l] = 0;
/* read the parity */
for(l=0;l<state->level;++l) {
ret = parity_read(parity[l], i, buffer_recov[l], state->block_size, stdlog);
if (ret == -1) {
buffer_recov[l] = 0;
fprintf(stdlog, "parity_error:%u:%s: Read error\n", i, lev_config_name(l));
++error;
error_on_this_block = 1;
/* follow */
}
}
/* compute the parity */
raid_gen(diskmax, state->level, state->block_size, buffer);
/* compare the parity */
for(l=0;l<state->level;++l) {
if (buffer_recov[l] && memcmp(buffer[diskmax + l], buffer_recov[l], state->block_size) != 0) {
fprintf(stdlog, "parity_error:%u:%s: Data error\n", i, lev_config_name(l));
/* it's a silent error only if we are dealing with synced blocks */
if (block_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
fprintf(stderr, "Data error in parity '%s' at position '%u'\n", lev_config_name(l), i);
fprintf(stderr, "WARNING! Unexpected data error in a parity disk! The block is now marked as bad!\n");
fprintf(stderr, "Try with 'snapraid -e fix' to recover!\n");
++silent_error;
silent_error_on_this_block = 1;
}
}
}
}
if (silent_error_on_this_block) {
/* set the error status keeping the existing time and hash */
info_set(&state->infoarr, i, info_set_bad(info));
} else if (error_on_this_block) {
/* do nothing, as this is a generic error */
/* likely caused by a not synced array */
} else {
/* if rehash is needed */
if (rehash) {
/* store all the new hash already computed */
for(j=0;j<diskmax;++j) {
if (rehandle[j].block)
memcpy(rehandle[j].block->hash, rehandle[j].hash, HASH_SIZE);
}
}
/* update the time info of the block */
/* and clear any other flag */
info_set(&state->infoarr, i, info_make(now, 0, 0));
}
/* mark the state as needing write */
state->need_write = 1;
/* count the number of processed block */
++countpos;
/* progress */
if (state_progress(state, i, countpos, countmax, countsize)) {
/* LCOV_EXCL_START */
break;
/* LCOV_EXCL_STOP */
}
/* autosave */
if (state->autosave != 0
&& autosavedone >= autosavelimit /* if we have reached the limit */
&& autosavemissing >= autosavelimit /* if we have at least a full step to do */
) {
autosavedone = 0; /* restart the counter */
state_progress_stop(state);
printf("Autosaving...\n");
state_write(state);
state_progress_restart(state);
}
}
state_progress_end(state, countpos, countmax, countsize);
if (error || silent_error) {
printf("\n");
printf("%8u read errors\n", error);
printf("%8u data errors\n", silent_error);
printf("WARNING! There are errors!\n");
} else {
/* print the result only if processed something */
if (countpos != 0)
printf("Everything OK\n");
}
fprintf(stdlog, "summary:error_read:%u\n", error);
fprintf(stdlog, "summary:error_data:%u\n", silent_error);
if (error + silent_error == 0)
fprintf(stdlog, "summary:exit:ok\n");
else
fprintf(stdlog, "summary:exit:error\n");
fflush(stdlog);
bail:
for(j=0;j<diskmax;++j) {
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
fprintf(stderr, "DANGER! Unexpected close error in a data disk.\n");
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
free(handle);
free(buffer_alloc);
free(buffer);
free(rehandle_alloc);
if (state->opt.expect_recoverable) {
if (error + silent_error == 0)
return -1;
} else {
if (error + silent_error != 0)
return -1;
}
return 0;
}
static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity_handle* parity_handle, block_off_t blockstart, block_off_t blockmax, struct snapraid_plan* plan, time_t now)
{
struct snapraid_io io;
struct snapraid_handle* handle;
void* rehandle_alloc;
struct snapraid_rehash* rehandle;
unsigned diskmax;
block_off_t blockcur;
unsigned j;
unsigned buffermax;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
block_off_t autosavedone;
block_off_t autosavelimit;
block_off_t autosavemissing;
int ret;
unsigned error;
unsigned silent_error;
unsigned io_error;
unsigned l;
unsigned* waiting_map;
unsigned waiting_mac;
char esc_buffer[ESC_MAX];
/* maps the disks to handles */
handle = handle_mapping(state, &diskmax);
/* rehash buffers */
rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
/* we need 1 * data + 2 * parity */
buffermax = diskmax + 2 * state->level;
/* initialize the io threads */
io_init(&io, state, state->opt.io_cache, buffermax, scrub_data_reader, handle, diskmax, scrub_parity_reader, 0, parity_handle, state->level);
/* possibly waiting disks */
waiting_mac = diskmax > RAID_PARITY_MAX ? diskmax : RAID_PARITY_MAX;
waiting_map = malloc_nofail(waiting_mac * sizeof(unsigned));
error = 0;
silent_error = 0;
io_error = 0;
/* first count the number of blocks to process */
countmax = 0;
plan->countlast = 0;
for (blockcur = blockstart; blockcur < blockmax; ++blockcur) {
if (!block_is_enabled(plan, blockcur))
continue;
++countmax;
}
/* compute the autosave size for all disk, even if not read */
/* this makes sense because the speed should be almost the same */
/* if the disks are read in parallel */
autosavelimit = state->autosave / (diskmax * state->block_size);
autosavemissing = countmax; /* blocks to do */
autosavedone = 0; /* blocks done */
/* drop until now */
state_usage_waste(state);
countsize = 0;
countpos = 0;
plan->countlast = 0;
/* start all the worker threads */
io_start(&io, blockstart, blockmax, &block_is_enabled, plan);
state_progress_begin(state, blockstart, blockmax, countmax);
while (1) {
unsigned char* buffer_recov[LEV_MAX];
snapraid_info info;
int error_on_this_block;
int silent_error_on_this_block;
int io_error_on_this_block;
int block_is_unsynced;
int rehash;
void** buffer;
/* go to the next block */
blockcur = io_read_next(&io, &buffer);
if (blockcur >= blockmax)
break;
/* until now is scheduling */
state_usage_sched(state);
/* one more block processed for autosave */
++autosavedone;
--autosavemissing;
/* by default process the block, and skip it if something goes wrong */
error_on_this_block = 0;
silent_error_on_this_block = 0;
io_error_on_this_block = 0;
/* if all the blocks at this address are synced */
/* if not, parity is not even checked */
block_is_unsynced = 0;
/* get block specific info */
info = info_get(&state->infoarr, blockcur);
/* if we have to use the old hash */
rehash = info_get_rehash(info);
/* for each disk, process the block */
for (j = 0; j < diskmax; ++j) {
struct snapraid_task* task;
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
int file_is_unsynced;
struct snapraid_disk* disk;
struct snapraid_file* file;
block_off_t file_pos;
unsigned diskcur;
/* if the file on this disk is synced */
/* if not, silent errors are assumed as expected error */
file_is_unsynced = 0;
/* until now is misc */
state_usage_misc(state);
/* get the next task */
task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
/* until now is disk */
state_usage_disk(state, handle, waiting_map, waiting_mac);
/* get the task results */
disk = task->disk;
block = task->block;
file = task->file;
file_pos = task->file_pos;
read_size = task->read_size;
/* by default no rehash in case of "continue" */
rehandle[diskcur].block = 0;
/* if the disk position is not used */
if (!disk)
continue;
/* if the block is unsynced, errors are expected */
if (block_has_invalid_parity(block)) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* if the block is not used */
if (!block_has_file(block))
continue;
/* if the block is unsynced, errors are expected */
if (task->is_timestamp_different) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* handle error conditions */
if (task->state == TASK_STATE_IOERROR) {
++io_error;
goto bail;
}
if (task->state == TASK_STATE_ERROR) {
++error;
goto bail;
}
if (task->state == TASK_STATE_ERROR_CONTINUE) {
++error;
error_on_this_block = 1;
continue;
}
if (task->state == TASK_STATE_IOERROR_CONTINUE) {
++io_error;
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, task->path);
log_fatal("Stopping at block %u\n", blockcur);
goto bail;
/* LCOV_EXCL_STOP */
}
/* otherwise continue */
io_error_on_this_block = 1;
continue;
}
if (task->state != TASK_STATE_DONE) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in task state\n");
os_abort();
/* LCOV_EXCL_STOP */
}
countsize += read_size;
/* now compute the hash */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
/* compute the new hash, and store it */
rehandle[diskcur].block = block;
memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
}
/* until now is hash */
state_usage_hash(state);
if (block_has_updated_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
unsigned diff = memdiff(hash, block->hash, HASH_SIZE);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", blockcur, disk->name, esc(file->sub, esc_buffer), file_pos, diff);
/* it's a silent error only if we are dealing with synced files */
if (file_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
log_error("Data error in file '%s' at position '%u', diff bits %u\n", task->path, file_pos, diff);
++silent_error;
silent_error_on_this_block = 1;
}
continue;
}
}
}
/* buffers for parity read and not computed */
for (l = 0; l < state->level; ++l)
buffer_recov[l] = buffer[diskmax + state->level + l];
for (; l < LEV_MAX; ++l)
buffer_recov[l] = 0;
/* until now is misc */
state_usage_misc(state);
/* read the parity */
for (l = 0; l < state->level; ++l) {
struct snapraid_task* task;
unsigned levcur;
task = io_parity_read(&io, &levcur, waiting_map, &waiting_mac);
/* until now is parity */
state_usage_parity(state, waiting_map, waiting_mac);
/* handle error conditions */
if (task->state == TASK_STATE_IOERROR) {
++io_error;
goto bail;
}
if (task->state == TASK_STATE_ERROR) {
++error;
goto bail;
}
if (task->state == TASK_STATE_ERROR_CONTINUE) {
++error;
error_on_this_block = 1;
/* if continuing on error, clear the missing buffer */
buffer_recov[levcur] = 0;
continue;
}
if (task->state == TASK_STATE_IOERROR_CONTINUE) {
++io_error;
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in the %s disk, it isn't possible to scrub.\n", lev_name(levcur));
log_fatal("Ensure that disk '%s' is sane and can be read.\n", lev_config_name(levcur));
log_fatal("Stopping at block %u\n", blockcur);
goto bail;
/* LCOV_EXCL_STOP */
}
/* otherwise continue */
io_error_on_this_block = 1;
/* if continuing on error, clear the missing buffer */
buffer_recov[levcur] = 0;
continue;
}
if (task->state != TASK_STATE_DONE) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in task state\n");
os_abort();
/* LCOV_EXCL_STOP */
}
}
/* if we have read all the data required and it's correct, proceed with the parity check */
if (!error_on_this_block && !silent_error_on_this_block && !io_error_on_this_block) {
/* compute the parity */
raid_gen(diskmax, state->level, state->block_size, buffer);
/* compare the parity */
for (l = 0; l < state->level; ++l) {
if (buffer_recov[l] && memcmp(buffer[diskmax + l], buffer_recov[l], state->block_size) != 0) {
unsigned diff = memdiff(buffer[diskmax + l], buffer_recov[l], state->block_size);
log_tag("parity_error:%u:%s: Data error, diff bits %u\n", blockcur, lev_config_name(l), diff);
/* it's a silent error only if we are dealing with synced blocks */
if (block_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
log_fatal("Data error in parity '%s' at position '%u', diff bits %u\n", lev_config_name(l), blockcur, diff);
++silent_error;
silent_error_on_this_block = 1;
}
}
}
/* until now is raid */
state_usage_raid(state);
}
if (silent_error_on_this_block || io_error_on_this_block) {
/* set the error status keeping other info */
info_set(&state->infoarr, blockcur, info_set_bad(info));
} else if (error_on_this_block) {
/* do nothing, as this is a generic error */
/* likely caused by a not synced array */
} else {
/* if rehash is needed */
if (rehash) {
/* store all the new hash already computed */
for (j = 0; j < diskmax; ++j) {
if (rehandle[j].block)
memcpy(rehandle[j].block->hash, rehandle[j].hash, HASH_SIZE);
}
}
/* update the time info of the block */
/* and clear any other flag */
info_set(&state->infoarr, blockcur, info_make(now, 0, 0, 0));
}
/* mark the state as needing write */
state->need_write = 1;
/* count the number of processed block */
++countpos;
/* progress */
if (state_progress(state, &io, blockcur, countpos, countmax, countsize)) {
/* LCOV_EXCL_START */
break;
/* LCOV_EXCL_STOP */
}
/* autosave */
if (state->autosave != 0
&& autosavedone >= autosavelimit /* if we have reached the limit */
&& autosavemissing >= autosavelimit /* if we have at least a full step to do */
) {
autosavedone = 0; /* restart the counter */
/* until now is misc */
state_usage_misc(state);
state_progress_stop(state);
msg_progress("Autosaving...\n");
state_write(state);
state_progress_restart(state);
/* drop until now */
state_usage_waste(state);
}
}
state_progress_end(state, countpos, countmax, countsize);
state_usage_print(state);
if (error || silent_error || io_error) {
msg_status("\n");
msg_status("%8u file errors\n", error);
msg_status("%8u io errors\n", io_error);
msg_status("%8u data errors\n", silent_error);
} else {
/* print the result only if processed something */
if (countpos != 0)
msg_status("Everything OK\n");
}
if (error)
log_fatal("WARNING! Unexpected file errors!\n");
if (io_error)
log_fatal("DANGER! Unexpected input/output errors! The failing blocks are now marked as bad!\n");
if (silent_error)
log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
if (io_error || silent_error) {
log_fatal("Use 'snapraid status' to list the bad blocks.\n");
log_fatal("Use 'snapraid -e fix' to recover.\n");
}
log_tag("summary:error_file:%u\n", error);
log_tag("summary:error_io:%u\n", io_error);
log_tag("summary:error_data:%u\n", silent_error);
if (error + silent_error + io_error == 0)
log_tag("summary:exit:ok\n");
else
log_tag("summary:exit:error\n");
log_flush();
bail:
/* stop all the worker threads */
io_stop(&io);
for (j = 0; j < diskmax; ++j) {
struct snapraid_file* file = handle[j].file;
struct snapraid_disk* disk = handle[j].disk;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Close error. %s\n", blockcur, disk->name, esc(file->sub, esc_buffer), strerror(errno));
log_fatal("DANGER! Unexpected close error in a data disk.\n");
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
free(handle);
free(rehandle_alloc);
free(waiting_map);
io_done(&io);
if (state->opt.expect_recoverable) {
if (error + silent_error + io_error == 0)
return -1;
} else {
if (error + silent_error + io_error != 0)
return -1;
}
return 0;
}
void file_info::overwrite_info(const file_info & p_source) {
t_size count = p_source.info_get_count();
for(t_size n=0;n<count;n++) {
info_set(p_source.info_enum_name(n),p_source.info_enum_value(n));
}
}
static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity_handle** parity, block_off_t blockstart, block_off_t blockmax, struct snapraid_plan* plan, time_t now)
{
struct snapraid_handle* handle;
void* rehandle_alloc;
struct snapraid_rehash* rehandle;
unsigned diskmax;
block_off_t i;
unsigned j;
void* buffer_alloc;
void** buffer;
unsigned buffermax;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
block_off_t autosavedone;
block_off_t autosavelimit;
block_off_t autosavemissing;
int ret;
unsigned error;
unsigned silent_error;
unsigned io_error;
unsigned l;
/* maps the disks to handles */
handle = handle_map(state, &diskmax);
/* rehash buffers */
rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
/* we need disk + 2 for each parity level buffers */
buffermax = diskmax + state->level * 2;
buffer = malloc_nofail_vector_align(diskmax, buffermax, state->block_size, &buffer_alloc);
if (!state->opt.skip_self)
mtest_vector(buffermax, state->block_size, buffer);
error = 0;
silent_error = 0;
io_error = 0;
/* first count the number of blocks to process */
countmax = 0;
plan->countlast = 0;
for (i = blockstart; i < blockmax; ++i) {
if (!block_is_enabled(state, i, plan))
continue;
++countmax;
}
/* compute the autosave size for all disk, even if not read */
/* this makes sense because the speed should be almost the same */
/* if the disks are read in parallel */
autosavelimit = state->autosave / (diskmax * state->block_size);
autosavemissing = countmax; /* blocks to do */
autosavedone = 0; /* blocks done */
/* drop until now */
state_usage_waste(state);
countsize = 0;
countpos = 0;
plan->countlast = 0;
state_progress_begin(state, blockstart, blockmax, countmax);
for (i = blockstart; i < blockmax; ++i) {
snapraid_info info;
int error_on_this_block;
int silent_error_on_this_block;
int io_error_on_this_block;
int block_is_unsynced;
int rehash;
if (!block_is_enabled(state, i, plan))
continue;
/* one more block processed for autosave */
++autosavedone;
--autosavemissing;
/* by default process the block, and skip it if something goes wrong */
error_on_this_block = 0;
silent_error_on_this_block = 0;
io_error_on_this_block = 0;
/* if all the blocks at this address are synced */
/* if not, parity is not even checked */
block_is_unsynced = 0;
/* get block specific info */
info = info_get(&state->infoarr, i);
/* if we have to use the old hash */
rehash = info_get_rehash(info);
/* for each disk, process the block */
for (j = 0; j < diskmax; ++j) {
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
int file_is_unsynced;
struct snapraid_disk* disk = handle[j].disk;
struct snapraid_file* file;
block_off_t file_pos;
/* if the file on this disk is synced */
/* if not, silent errors are assumed as expected error */
file_is_unsynced = 0;
/* by default no rehash in case of "continue" */
rehandle[j].block = 0;
/* if the disk position is not used */
if (!disk) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* if the block is not used */
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* get the file of this block */
file = fs_par2file_get(disk, i, &file_pos);
/* if the block is unsynced, errors are expected */
if (block_has_invalid_parity(block)) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* until now is CPU */
state_usage_cpu(state);
/* if the file is different than the current one, close it */
if (handle[j].file != 0 && handle[j].file != file) {
/* keep a pointer at the file we are going to close for error reporting */
struct snapraid_file* report = handle[j].file;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
if (errno == EIO) {
log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", i);
++io_error;
goto bail;
}
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
log_fatal("Stopping at block %u\n", i);
++error;
goto bail;
/* LCOV_EXCL_STOP */
}
}
ret = handle_open(&handle[j], file, state->file_mode, log_error, 0);
if (ret == -1) {
if (errno == EIO) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_fatal("DANGER! Unexpected input/output open error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", i);
++io_error;
goto bail;
/* LCOV_EXCL_STOP */
}
log_tag("error:%u:%s:%s: Open error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
++error;
error_on_this_block = 1;
continue;
}
/* check if the file is changed */
if (handle[j].st.st_size != file->size
|| handle[j].st.st_mtime != file->mtime_sec
|| STAT_NSEC(&handle[j].st) != file->mtime_nsec
/* don't check the inode to support filesystem without persistent inodes */
) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* note that we intentionally don't abort if the file has different attributes */
/* from the last sync, as we are expected to return errors if running */
/* in an unsynced array. This is just like the check command. */
read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size, log_error, 0);
if (read_size == -1) {
if (errno == EIO) {
log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", i);
++io_error;
goto bail;
/* LCOV_EXCL_STOP */
}
log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, file_pos);
++io_error;
io_error_on_this_block = 1;
continue;
}
log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
++error;
error_on_this_block = 1;
continue;
}
/* until now is disk */
state_usage_disk(state, disk);
countsize += read_size;
/* now compute the hash */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
/* compute the new hash, and store it */
rehandle[j].block = block;
memhash(state->hash, state->hashseed, rehandle[j].hash, buffer[j], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
}
if (block_has_updated_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
unsigned diff = memdiff(hash, block->hash, HASH_SIZE);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), file_pos, diff);
/* it's a silent error only if we are dealing with synced files */
if (file_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, file_pos, diff);
++silent_error;
silent_error_on_this_block = 1;
}
continue;
}
}
}
/* if we have read all the data required and it's correct, proceed with the parity check */
if (!error_on_this_block && !silent_error_on_this_block && !io_error_on_this_block) {
unsigned char* buffer_recov[LEV_MAX];
/* until now is CPU */
state_usage_cpu(state);
/* buffers for parity read and not computed */
for (l = 0; l < state->level; ++l)
buffer_recov[l] = buffer[diskmax + state->level + l];
for (; l < LEV_MAX; ++l)
buffer_recov[l] = 0;
/* read the parity */
for (l = 0; l < state->level; ++l) {
ret = parity_read(parity[l], i, buffer_recov[l], state->block_size, log_error);
if (ret == -1) {
buffer_recov[l] = 0;
if (errno == EIO) {
log_tag("parity_error:%u:%s: Read EIO error. %s\n", i, lev_config_name(l), strerror(errno));
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in the %s disk, it isn't possible to scrub.\n", lev_name(l));
log_fatal("Ensure that disk '%s' is sane and can be read.\n", lev_config_name(l));
log_fatal("Stopping at block %u\n", i);
++io_error;
goto bail;
/* LCOV_EXCL_STOP */
}
log_error("Input/Output error in parity '%s' at position '%u'\n", lev_config_name(l), i);
++io_error;
io_error_on_this_block = 1;
continue;
}
log_tag("parity_error:%u:%s: Read error. %s\n", i, lev_config_name(l), strerror(errno));
++error;
error_on_this_block = 1;
continue;
}
/* until now is parity */
state_usage_parity(state, l);
}
/* compute the parity */
raid_gen(diskmax, state->level, state->block_size, buffer);
/* compare the parity */
for (l = 0; l < state->level; ++l) {
if (buffer_recov[l] && memcmp(buffer[diskmax + l], buffer_recov[l], state->block_size) != 0) {
unsigned diff = memdiff(buffer[diskmax + l], buffer_recov[l], state->block_size);
log_tag("parity_error:%u:%s: Data error, diff bits %u\n", i, lev_config_name(l), diff);
/* it's a silent error only if we are dealing with synced blocks */
if (block_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
log_fatal("Data error in parity '%s' at position '%u', diff bits %u\n", lev_config_name(l), i, diff);
++silent_error;
silent_error_on_this_block = 1;
}
}
}
}
if (silent_error_on_this_block || io_error_on_this_block) {
/* set the error status keeping other info */
info_set(&state->infoarr, i, info_set_bad(info));
} else if (error_on_this_block) {
/* do nothing, as this is a generic error */
/* likely caused by a not synced array */
} else {
/* if rehash is needed */
if (rehash) {
/* store all the new hash already computed */
for (j = 0; j < diskmax; ++j) {
if (rehandle[j].block)
memcpy(rehandle[j].block->hash, rehandle[j].hash, HASH_SIZE);
}
}
/* update the time info of the block */
/* and clear any other flag */
info_set(&state->infoarr, i, info_make(now, 0, 0, 0));
}
/* mark the state as needing write */
state->need_write = 1;
/* count the number of processed block */
++countpos;
/* progress */
if (state_progress(state, i, countpos, countmax, countsize)) {
/* LCOV_EXCL_START */
break;
/* LCOV_EXCL_STOP */
}
/* autosave */
if (state->autosave != 0
&& autosavedone >= autosavelimit /* if we have reached the limit */
&& autosavemissing >= autosavelimit /* if we have at least a full step to do */
) {
autosavedone = 0; /* restart the counter */
/* until now is CPU */
state_usage_cpu(state);
state_progress_stop(state);
msg_progress("Autosaving...\n");
state_write(state);
state_progress_restart(state);
/* drop until now */
state_usage_waste(state);
}
}
state_progress_end(state, countpos, countmax, countsize);
state_usage_print(state);
if (error || silent_error || io_error) {
msg_status("\n");
msg_status("%8u file errors\n", error);
msg_status("%8u io errors\n", io_error);
msg_status("%8u data errors\n", silent_error);
} else {
/* print the result only if processed something */
if (countpos != 0)
msg_status("Everything OK\n");
}
if (error)
log_fatal("WARNING! Unexpected file errors!\n");
if (io_error)
log_fatal("DANGER! Unexpected input/output errors! The failing blocks are now marked as bad!\n");
if (silent_error)
log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
if (io_error || silent_error) {
log_fatal("Use 'snapraid status' to list the bad blocks.\n");
log_fatal("Use 'snapraid -e fix' to recover.\n");
}
log_tag("summary:error_file:%u\n", error);
log_tag("summary:error_io:%u\n", io_error);
log_tag("summary:error_data:%u\n", silent_error);
if (error + silent_error + io_error == 0)
log_tag("summary:exit:ok\n");
else
log_tag("summary:exit:error\n");
log_flush();
bail:
for (j = 0; j < diskmax; ++j) {
struct snapraid_file* file = handle[j].file;
struct snapraid_disk* disk = handle[j].disk;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_fatal("DANGER! Unexpected close error in a data disk.\n");
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
free(handle);
free(buffer_alloc);
free(buffer);
free(rehandle_alloc);
if (state->opt.expect_recoverable) {
if (error + silent_error + io_error == 0)
return -1;
} else {
if (error + silent_error + io_error != 0)
return -1;
}
return 0;
}
void file_info::info_set_int(const char * name,t_int64 value)
{
PFC_ASSERT(pfc::is_valid_utf8(name));
info_set(name,pfc::format_int(value));
}
void file_info::copy_info_single(const file_info & p_source,t_size p_index)
{
info_set(p_source.info_enum_name(p_index),p_source.info_enum_value(p_index));
}
