struct gpio *gpio_open(int pin, const char *usage, int type, ...)
{
if(!usage)
return NULL; // no usage specified
int valid = 0;
int i;
for(i=0; i<PINS_COUNT; i++)
{
if(pins[i] == pin)
{
valid = 1;
break;
}
}
if(!valid)
return NULL; // invalid pin number
struct pin_lookup_data data_pin;
data_pin.pin = pin;
data_pin.result = NULL;
list_foreach(&gpios, pin_lookup, &data_pin);
if(data_pin.result)
return NULL; // already in use
struct type_lookup_data data;
data.type = type;
data.result = NULL;
list_foreach(&types, type_lookup, &data);
if(!data.result)
return NULL; // type not found
struct gpio *gpio;
malloc_nofail(gpio);
gpio->pin = pin;
strdup_nofail(gpio->usage, usage);
gpio->type = data.result;
gpio->gpio = gpio_from_pin[gpio->pin];
va_list args;
va_start(args, type);
int ret = gpio->type->open(gpio, args);
va_end(args);
if(!ret)
{
free(gpio->usage);
free(gpio);
return NULL; // type open failed
}
list_add(&gpios, gpio);
++(gpio->type->refcount);
return gpio;
}
struct snapraid_dir* dir_alloc(const char* sub)
{
struct snapraid_dir* dir;
dir = malloc_nofail(sizeof(struct snapraid_dir));
dir->sub = strdup_nofail(sub);
dir->flag = 0;
return dir;
}
struct snapraid_content* content_alloc(const char* path, uint64_t dev)
{
struct snapraid_content* content;
content = malloc_nofail(sizeof(struct snapraid_content));
pathimport(content->content, sizeof(content->content), path);
content->device = dev;
return content;
}
struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag)
{
struct snapraid_link* slink;
slink = malloc_nofail(sizeof(struct snapraid_link));
slink->sub = strdup_nofail(sub);
slink->linkto = strdup_nofail(linkto);
slink->flag = link_flag;
return slink;
}
struct snapraid_chunk* chunk_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count)
{
struct snapraid_chunk* chunk;
chunk = malloc_nofail(sizeof(struct snapraid_chunk));
chunk->parity_pos = parity_pos;
chunk->file = file;
chunk->file_pos = file_pos;
chunk->count = count;
return chunk;
}
struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid)
{
struct snapraid_map* map;
map = malloc_nofail(sizeof(struct snapraid_map));
pathcpy(map->name, sizeof(map->name), name);
map->position = position;
map->total_blocks = total_blocks;
map->free_blocks = free_blocks;
pathcpy(map->uuid, sizeof(map->uuid), uuid);
return map;
}
struct snapraid_file* file_dup(struct snapraid_file* copy)
{
struct snapraid_file* file;
block_off_t i;
file = malloc_nofail(sizeof(struct snapraid_file));
file->sub = strdup_nofail(copy->sub);
file->size = copy->size;
file->blockmax = copy->blockmax;
file->mtime_sec = copy->mtime_sec;
file->mtime_nsec = copy->mtime_nsec;
file->inode = copy->inode;
file->physical = copy->physical;
file->flag = copy->flag;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
for (i = 0; i < file->blockmax; ++i) {
file->blockvec[i].state = copy->blockvec[i].state;
memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE);
}
return file;
}
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical)
{
struct snapraid_file* file;
block_off_t i;
file = malloc_nofail(sizeof(struct snapraid_file));
file->sub = strdup_nofail(sub);
file->size = size;
file->blockmax = (size + block_size - 1) / block_size;
file->mtime_sec = mtime_sec;
file->mtime_nsec = mtime_nsec;
file->inode = inode;
file->physical = physical;
file->flag = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
for (i = 0; i < file->blockmax; ++i) {
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);
hash_invalid_set(file->blockvec[i].hash);
}
return file;
}
static void search_file(struct snapraid_state* state, const char* path, data_off_t size, int64_t mtime_sec, int mtime_nsec)
{
struct snapraid_search_file* file;
tommy_uint32_t file_hash;
file = malloc_nofail(sizeof(struct snapraid_search_file));
file->path = strdup_nofail(path);
file->size = size;
file->mtime_sec = mtime_sec;
file->mtime_nsec = mtime_nsec;
file_hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
tommy_hashdyn_insert(&state->searchset, &file->node, file, file_hash);
}
struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, int skip)
{
struct snapraid_disk* disk;
disk = malloc_nofail(sizeof(struct snapraid_disk));
pathcpy(disk->name, sizeof(disk->name), name);
pathimport(disk->dir, sizeof(disk->dir), dir);
/* ensure that the dir terminate with "/" if it isn't empty */
pathslash(disk->dir, sizeof(disk->dir));
disk->smartctl[0] = 0;
disk->device = dev;
disk->tick = 0;
disk->total_blocks = 0;
disk->free_blocks = 0;
disk->first_free_block = 0;
disk->has_volatile_inodes = 0;
disk->has_unreliable_physical = 0;
disk->has_different_uuid = 0;
disk->has_unsupported_uuid = 0;
disk->had_empty_uuid = 0;
disk->mapping_idx = -1;
disk->skip_access = skip;
tommy_list_init(&disk->filelist);
tommy_list_init(&disk->deletedlist);
tommy_hashdyn_init(&disk->inodeset);
tommy_hashdyn_init(&disk->pathset);
tommy_hashdyn_init(&disk->stampset);
tommy_list_init(&disk->linklist);
tommy_hashdyn_init(&disk->linkset);
tommy_list_init(&disk->dirlist);
tommy_hashdyn_init(&disk->dirset);
tommy_tree_init(&disk->fs_parity, chunk_parity_compare);
tommy_tree_init(&disk->fs_file, chunk_file_compare);
disk->fs_last = 0;
return disk;
}
struct snapraid_filter* filter_alloc_disk(int direction, const char* pattern)
{
struct snapraid_filter* filter;
filter = malloc_nofail(sizeof(struct snapraid_filter));
pathimport(filter->pattern, sizeof(filter->pattern), pattern);
filter->direction = direction;
/* it's a disk filter */
filter->is_disk = 1;
filter->is_path = 0;
filter->is_dir = 0;
/* no slash allowed in disk names */
if (strchr(filter->pattern, '/') != 0) {
/* LCOV_EXCL_START */
free(filter);
return 0;
/* LCOV_EXCL_STOP */
}
return filter;
}
int state_scrub(struct snapraid_state* state, int plan, int olderthan)
{
block_off_t blockmax;
block_off_t countlimit;
block_off_t i;
block_off_t count;
time_t recentlimit;
int ret;
struct snapraid_parity_handle parity_handle[LEV_MAX];
struct snapraid_plan ps;
time_t* timemap;
unsigned error;
time_t now;
unsigned l;
/* get the present time */
now = time(0);
msg_progress("Initializing...\n");
if ((plan == SCRUB_BAD || plan == SCRUB_NEW || plan == SCRUB_FULL)
&& olderthan >= 0) {
/* LCOV_EXCL_START */
log_fatal("You can specify -o, --older-than only with a numeric percentage.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
blockmax = parity_allocated_size(state);
/* preinitialize to avoid warnings */
countlimit = 0;
recentlimit = 0;
ps.state = state;
if (state->opt.force_scrub_even) {
ps.plan = SCRUB_EVEN;
} else if (plan == SCRUB_FULL) {
ps.plan = SCRUB_FULL;
} else if (plan == SCRUB_NEW) {
ps.plan = SCRUB_NEW;
} else if (plan == SCRUB_BAD) {
ps.plan = SCRUB_BAD;
} else if (state->opt.force_scrub_at) {
/* scrub the specified amount of blocks */
ps.plan = SCRUB_AUTO;
countlimit = state->opt.force_scrub_at;
recentlimit = now;
} else {
ps.plan = SCRUB_AUTO;
if (plan >= 0) {
countlimit = md(blockmax, plan, 100);
} else {
/* by default scrub 8.33% of the array (100/12=8.(3)) */
countlimit = md(blockmax, 1, 12);
}
if (olderthan >= 0) {
recentlimit = now - olderthan * 24 * 3600;
} else {
/* by default use a 10 day time limit */
recentlimit = now - 10 * 24 * 3600;
}
}
/* identify the time limit */
/* we sort all the block times, and we identify the time limit for which we reach the quota */
/* this allow to process first the oldest blocks */
timemap = malloc_nofail(blockmax * sizeof(time_t));
/* copy the info in the temp vector */
count = 0;
log_tag("block_count:%u\n", blockmax);
for (i = 0; i < blockmax; ++i) {
snapraid_info info = info_get(&state->infoarr, i);
/* skip unused blocks */
if (info == 0)
continue;
timemap[count++] = info_get_time(info);
}
if (!count) {
/* LCOV_EXCL_START */
log_fatal("The array appears to be empty.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* sort it */
qsort(timemap, count, sizeof(time_t), time_compare);
/* output the info map */
i = 0;
log_tag("info_count:%u\n", count);
while (i < count) {
unsigned j = i + 1;
while (j < count && timemap[i] == timemap[j])
++j;
log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)timemap[i], j - i);
i = j;
}
/* compute the limits from count/recentlimit */
if (ps.plan == SCRUB_AUTO) {
/* no more than the full count */
if (countlimit > count)
countlimit = count;
/* decrease until we reach the specific recentlimit */
while (countlimit > 0 && timemap[countlimit - 1] > recentlimit)
--countlimit;
/* if there is something to scrub */
if (countlimit > 0) {
/* get the most recent time we want to scrub */
ps.timelimit = timemap[countlimit - 1];
/* count how many entries for this exact time we have to scrub */
/* if the blocks have all the same time, we end with countlimit == lastlimit */
ps.lastlimit = 1;
while (countlimit > ps.lastlimit && timemap[countlimit - ps.lastlimit - 1] == ps.timelimit)
++ps.lastlimit;
} else {
/* if nothing to scrub, disable also other limits */
ps.timelimit = 0;
ps.lastlimit = 0;
}
log_tag("count_limit:%u\n", countlimit);
log_tag("time_limit:%" PRIu64 "\n", (uint64_t)ps.timelimit);
log_tag("last_limit:%u\n", ps.lastlimit);
}
/* free the temp vector */
free(timemap);
/* open the file for reading */
for (l = 0; l < state->level; ++l) {
ret = parity_open(&parity_handle[l], l, state->parity[l].path, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to scrub.\n", lev_name(l));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
msg_progress("Scrubbing...\n");
error = 0;
ret = state_scrub_process(state, parity_handle, 0, blockmax, &ps, now);
if (ret == -1) {
++error;
/* continue, as we are already exiting */
}
for (l = 0; l < state->level; ++l) {
ret = parity_close(&parity_handle[l]);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l));
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
/* abort if required */
if (error != 0)
return -1;
return 0;
}
static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity_handle* parity_handle, block_off_t blockstart, block_off_t blockmax, struct snapraid_plan* plan, time_t now)
{
struct snapraid_io io;
struct snapraid_handle* handle;
void* rehandle_alloc;
struct snapraid_rehash* rehandle;
unsigned diskmax;
block_off_t blockcur;
unsigned j;
unsigned buffermax;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
block_off_t autosavedone;
block_off_t autosavelimit;
block_off_t autosavemissing;
int ret;
unsigned error;
unsigned silent_error;
unsigned io_error;
unsigned l;
unsigned* waiting_map;
unsigned waiting_mac;
char esc_buffer[ESC_MAX];
/* maps the disks to handles */
handle = handle_mapping(state, &diskmax);
/* rehash buffers */
rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
/* we need 1 * data + 2 * parity */
buffermax = diskmax + 2 * state->level;
/* initialize the io threads */
io_init(&io, state, state->opt.io_cache, buffermax, scrub_data_reader, handle, diskmax, scrub_parity_reader, 0, parity_handle, state->level);
/* possibly waiting disks */
waiting_mac = diskmax > RAID_PARITY_MAX ? diskmax : RAID_PARITY_MAX;
waiting_map = malloc_nofail(waiting_mac * sizeof(unsigned));
error = 0;
silent_error = 0;
io_error = 0;
/* first count the number of blocks to process */
countmax = 0;
plan->countlast = 0;
for (blockcur = blockstart; blockcur < blockmax; ++blockcur) {
if (!block_is_enabled(plan, blockcur))
continue;
++countmax;
}
/* compute the autosave size for all disk, even if not read */
/* this makes sense because the speed should be almost the same */
/* if the disks are read in parallel */
autosavelimit = state->autosave / (diskmax * state->block_size);
autosavemissing = countmax; /* blocks to do */
autosavedone = 0; /* blocks done */
/* drop until now */
state_usage_waste(state);
countsize = 0;
countpos = 0;
plan->countlast = 0;
/* start all the worker threads */
io_start(&io, blockstart, blockmax, &block_is_enabled, plan);
state_progress_begin(state, blockstart, blockmax, countmax);
while (1) {
unsigned char* buffer_recov[LEV_MAX];
snapraid_info info;
int error_on_this_block;
int silent_error_on_this_block;
int io_error_on_this_block;
int block_is_unsynced;
int rehash;
void** buffer;
/* go to the next block */
blockcur = io_read_next(&io, &buffer);
if (blockcur >= blockmax)
break;
/* until now is scheduling */
state_usage_sched(state);
/* one more block processed for autosave */
++autosavedone;
--autosavemissing;
/* by default process the block, and skip it if something goes wrong */
error_on_this_block = 0;
silent_error_on_this_block = 0;
io_error_on_this_block = 0;
/* if all the blocks at this address are synced */
/* if not, parity is not even checked */
block_is_unsynced = 0;
/* get block specific info */
info = info_get(&state->infoarr, blockcur);
/* if we have to use the old hash */
rehash = info_get_rehash(info);
/* for each disk, process the block */
for (j = 0; j < diskmax; ++j) {
struct snapraid_task* task;
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
int file_is_unsynced;
struct snapraid_disk* disk;
struct snapraid_file* file;
block_off_t file_pos;
unsigned diskcur;
/* if the file on this disk is synced */
/* if not, silent errors are assumed as expected error */
file_is_unsynced = 0;
/* until now is misc */
state_usage_misc(state);
/* get the next task */
task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
/* until now is disk */
state_usage_disk(state, handle, waiting_map, waiting_mac);
/* get the task results */
disk = task->disk;
block = task->block;
file = task->file;
file_pos = task->file_pos;
read_size = task->read_size;
/* by default no rehash in case of "continue" */
rehandle[diskcur].block = 0;
/* if the disk position is not used */
if (!disk)
continue;
/* if the block is unsynced, errors are expected */
if (block_has_invalid_parity(block)) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* if the block is not used */
if (!block_has_file(block))
continue;
/* if the block is unsynced, errors are expected */
if (task->is_timestamp_different) {
/* report that the block and the file are not synced */
block_is_unsynced = 1;
file_is_unsynced = 1;
/* follow */
}
/* handle error conditions */
if (task->state == TASK_STATE_IOERROR) {
++io_error;
goto bail;
}
if (task->state == TASK_STATE_ERROR) {
++error;
goto bail;
}
if (task->state == TASK_STATE_ERROR_CONTINUE) {
++error;
error_on_this_block = 1;
continue;
}
if (task->state == TASK_STATE_IOERROR_CONTINUE) {
++io_error;
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, task->path);
log_fatal("Stopping at block %u\n", blockcur);
goto bail;
/* LCOV_EXCL_STOP */
}
/* otherwise continue */
io_error_on_this_block = 1;
continue;
}
if (task->state != TASK_STATE_DONE) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in task state\n");
os_abort();
/* LCOV_EXCL_STOP */
}
countsize += read_size;
/* now compute the hash */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
/* compute the new hash, and store it */
rehandle[diskcur].block = block;
memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
}
/* until now is hash */
state_usage_hash(state);
if (block_has_updated_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
unsigned diff = memdiff(hash, block->hash, HASH_SIZE);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", blockcur, disk->name, esc(file->sub, esc_buffer), file_pos, diff);
/* it's a silent error only if we are dealing with synced files */
if (file_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
log_error("Data error in file '%s' at position '%u', diff bits %u\n", task->path, file_pos, diff);
++silent_error;
silent_error_on_this_block = 1;
}
continue;
}
}
}
/* buffers for parity read and not computed */
for (l = 0; l < state->level; ++l)
buffer_recov[l] = buffer[diskmax + state->level + l];
for (; l < LEV_MAX; ++l)
buffer_recov[l] = 0;
/* until now is misc */
state_usage_misc(state);
/* read the parity */
for (l = 0; l < state->level; ++l) {
struct snapraid_task* task;
unsigned levcur;
task = io_parity_read(&io, &levcur, waiting_map, &waiting_mac);
/* until now is parity */
state_usage_parity(state, waiting_map, waiting_mac);
/* handle error conditions */
if (task->state == TASK_STATE_IOERROR) {
++io_error;
goto bail;
}
if (task->state == TASK_STATE_ERROR) {
++error;
goto bail;
}
if (task->state == TASK_STATE_ERROR_CONTINUE) {
++error;
error_on_this_block = 1;
/* if continuing on error, clear the missing buffer */
buffer_recov[levcur] = 0;
continue;
}
if (task->state == TASK_STATE_IOERROR_CONTINUE) {
++io_error;
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in the %s disk, it isn't possible to scrub.\n", lev_name(levcur));
log_fatal("Ensure that disk '%s' is sane and can be read.\n", lev_config_name(levcur));
log_fatal("Stopping at block %u\n", blockcur);
goto bail;
/* LCOV_EXCL_STOP */
}
/* otherwise continue */
io_error_on_this_block = 1;
/* if continuing on error, clear the missing buffer */
buffer_recov[levcur] = 0;
continue;
}
if (task->state != TASK_STATE_DONE) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in task state\n");
os_abort();
/* LCOV_EXCL_STOP */
}
}
/* if we have read all the data required and it's correct, proceed with the parity check */
if (!error_on_this_block && !silent_error_on_this_block && !io_error_on_this_block) {
/* compute the parity */
raid_gen(diskmax, state->level, state->block_size, buffer);
/* compare the parity */
for (l = 0; l < state->level; ++l) {
if (buffer_recov[l] && memcmp(buffer[diskmax + l], buffer_recov[l], state->block_size) != 0) {
unsigned diff = memdiff(buffer[diskmax + l], buffer_recov[l], state->block_size);
log_tag("parity_error:%u:%s: Data error, diff bits %u\n", blockcur, lev_config_name(l), diff);
/* it's a silent error only if we are dealing with synced blocks */
if (block_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
log_fatal("Data error in parity '%s' at position '%u', diff bits %u\n", lev_config_name(l), blockcur, diff);
++silent_error;
silent_error_on_this_block = 1;
}
}
}
/* until now is raid */
state_usage_raid(state);
}
if (silent_error_on_this_block || io_error_on_this_block) {
/* set the error status keeping other info */
info_set(&state->infoarr, blockcur, info_set_bad(info));
} else if (error_on_this_block) {
/* do nothing, as this is a generic error */
/* likely caused by a not synced array */
} else {
/* if rehash is needed */
if (rehash) {
/* store all the new hash already computed */
for (j = 0; j < diskmax; ++j) {
if (rehandle[j].block)
memcpy(rehandle[j].block->hash, rehandle[j].hash, HASH_SIZE);
}
}
/* update the time info of the block */
/* and clear any other flag */
info_set(&state->infoarr, blockcur, info_make(now, 0, 0, 0));
}
/* mark the state as needing write */
state->need_write = 1;
/* count the number of processed block */
++countpos;
/* progress */
if (state_progress(state, &io, blockcur, countpos, countmax, countsize)) {
/* LCOV_EXCL_START */
break;
/* LCOV_EXCL_STOP */
}
/* autosave */
if (state->autosave != 0
&& autosavedone >= autosavelimit /* if we have reached the limit */
&& autosavemissing >= autosavelimit /* if we have at least a full step to do */
) {
autosavedone = 0; /* restart the counter */
/* until now is misc */
state_usage_misc(state);
state_progress_stop(state);
msg_progress("Autosaving...\n");
state_write(state);
state_progress_restart(state);
/* drop until now */
state_usage_waste(state);
}
}
state_progress_end(state, countpos, countmax, countsize);
state_usage_print(state);
if (error || silent_error || io_error) {
msg_status("\n");
msg_status("%8u file errors\n", error);
msg_status("%8u io errors\n", io_error);
msg_status("%8u data errors\n", silent_error);
} else {
/* print the result only if processed something */
if (countpos != 0)
msg_status("Everything OK\n");
}
if (error)
log_fatal("WARNING! Unexpected file errors!\n");
if (io_error)
log_fatal("DANGER! Unexpected input/output errors! The failing blocks are now marked as bad!\n");
if (silent_error)
log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
if (io_error || silent_error) {
log_fatal("Use 'snapraid status' to list the bad blocks.\n");
log_fatal("Use 'snapraid -e fix' to recover.\n");
}
log_tag("summary:error_file:%u\n", error);
log_tag("summary:error_io:%u\n", io_error);
log_tag("summary:error_data:%u\n", silent_error);
if (error + silent_error + io_error == 0)
log_tag("summary:exit:ok\n");
else
log_tag("summary:exit:error\n");
log_flush();
bail:
/* stop all the worker threads */
io_stop(&io);
for (j = 0; j < diskmax; ++j) {
struct snapraid_file* file = handle[j].file;
struct snapraid_disk* disk = handle[j].disk;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Close error. %s\n", blockcur, disk->name, esc(file->sub, esc_buffer), strerror(errno));
log_fatal("DANGER! Unexpected close error in a data disk.\n");
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
free(handle);
free(rehandle_alloc);
free(waiting_map);
io_done(&io);
if (state->opt.expect_recoverable) {
if (error + silent_error + io_error == 0)
return -1;
} else {
if (error + silent_error + io_error != 0)
return -1;
}
return 0;
}
struct snapraid_filter* filter_alloc_file(int direction, const char* pattern)
{
struct snapraid_filter* filter;
char* i;
char* first;
char* last;
int token_is_valid;
int token_is_filled;
filter = malloc_nofail(sizeof(struct snapraid_filter));
pathimport(filter->pattern, sizeof(filter->pattern), pattern);
filter->direction = direction;
/* find first and last slash */
first = 0;
last = 0;
/* reject invalid tokens, like "<empty>", ".", ".." and more dots */
token_is_valid = 0;
token_is_filled = 0;
for (i = filter->pattern; *i; ++i) {
if (*i == '/') {
/* reject invalid tokens, but accept an empty one as first */
if (!token_is_valid && (first != 0 || token_is_filled)) {
free(filter);
return 0;
}
token_is_valid = 0;
token_is_filled = 0;
/* update slash position */
if (!first)
first = i;
last = i;
} else if (*i != '.') {
token_is_valid = 1;
token_is_filled = 1;
} else {
token_is_filled = 1;
}
}
/* reject invalid tokens, but accept an empty one as last, but not if it's the only one */
if (!token_is_valid && (first == 0 || token_is_filled)) {
free(filter);
return 0;
}
/* it's a file filter */
filter->is_disk = 0;
if (first == 0) {
/* no slash */
filter->is_path = 0;
filter->is_dir = 0;
} else if (first == last && last[1] == 0) {
/* one slash at the end */
filter->is_path = 0;
filter->is_dir = 1;
last[0] = 0;
} else {
/* at least a slash not at the end */
filter->is_path = 1;
if (last[1] == 0) {
filter->is_dir = 1;
last[0] = 0;
} else {
filter->is_dir = 0;
}
/* a slash must be the first char, as we don't support PATH/FILE and PATH/DIR/ */
if (filter->pattern[0] != '/') {
free(filter);
return 0;
}
}
return filter;
}
int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
{
block_off_t blockmax;
block_off_t countlimit;
block_off_t i;
time_t timelimit;
time_t recentlimit;
unsigned count;
int ret;
struct snapraid_parity parity[LEV_MAX];
struct snapraid_parity* parity_ptr[LEV_MAX];
snapraid_info* infomap;
unsigned error;
time_t now;
unsigned l;
/* get the present time */
now = time(0);
printf("Initializing...\n");
blockmax = parity_size(state);
if (state->opt.force_scrub_even) {
/* no limit */
countlimit = blockmax;
recentlimit = now;
} else if (state->opt.force_scrub) {
/* scrub the specified amount of blocks */
countlimit = state->opt.force_scrub;
recentlimit = now;
} else {
/* by default scrub 1/12 of the array */
countlimit = md(blockmax, 1, 12);
if (percentage != -1)
countlimit = md(blockmax, percentage, 100);
/* by default use a 10 day time limit */
recentlimit = now - 10 * 24 * 3600;
if (olderthan != -1)
recentlimit = now - olderthan * 24 * 3600;
}
/* identify the time limit */
/* we sort all the block times, and we identify the time limit for which we reach the quota */
/* this allow to process first the oldest blocks */
infomap = malloc_nofail(blockmax * sizeof(snapraid_info));
/* copy the info in the temp vector */
count = 0;
for(i=0;i<blockmax;++i) {
snapraid_info info = info_get(&state->infoarr, i);
/* skip unused blocks */
if (info == 0)
continue;
infomap[count++] = info;
}
if (!count) {
/* LCOV_EXCL_START */
fprintf(stderr, "The array appears to be empty.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* sort it */
qsort(infomap, count, sizeof(snapraid_info), info_time_compare);
/* don't check more block than the available ones */
if (countlimit > count)
countlimit = count;
if (countlimit > 0) {
/* get the time limit */
timelimit = info_get_time(infomap[countlimit - 1]);
/* don't scrub too recent blocks */
if (timelimit > recentlimit) {
timelimit = recentlimit;
}
} else {
/* if we select a 0 percentage, disable also the time limit */
timelimit = 0;
}
/* free the temp vector */
free(infomap);
/* open the file for reading */
for(l=0;l<state->level;++l) {
parity_ptr[l] = &parity[l];
ret = parity_open(parity_ptr[l], state->parity_path[l], state->opt.skip_sequential);
if (ret == -1) {
/* LCOV_EXCL_START */
fprintf(stderr, "WARNING! Without an accessible %s file, it isn't possible to scrub.\n", lev_name(l));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
printf("Scrubbing...\n");
error = 0;
ret = state_scrub_process(state, parity_ptr, 0, blockmax, timelimit, countlimit, now);
if (ret == -1) {
++error;
/* continue, as we are already exiting */
}
for(l=0;l<state->level;++l) {
ret = parity_close(parity_ptr[l]);
if (ret == -1) {
/* LCOV_EXCL_START */
fprintf(stderr, "DANGER! Unexpected close error in %s disk.\n", lev_name(l));
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
/* abort if required */
if (error != 0)
return -1;
return 0;
}
int state_status(struct snapraid_state* state)
{
block_off_t blockmax;
block_off_t i;
snapraid_info* infomap;
time_t now;
block_off_t bad;
block_off_t rehash;
block_off_t count;
unsigned dayoldest, daymedian, daynewest;
unsigned bar[GRAPH_COLUMN];
unsigned barpos;
unsigned barmax;
time_t oldest, newest, median;
unsigned x, y;
tommy_node* node_disk;
unsigned file_count;
unsigned file_fragmented;
unsigned extra_fragment;
uint64_t file_size;
unsigned unsynced_blocks;
/* get the present time */
now = time(0);
blockmax = parity_size(state);
/* count fragments */
file_count = 0;
file_size = 0;
file_fragmented = 0;
extra_fragment = 0;
for(node_disk=state->disklist;node_disk!=0;node_disk=node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
tommy_node* node;
unsigned disk_file_count = 0;
unsigned disk_file_fragmented = 0;
unsigned disk_extra_fragment = 0;
uint64_t disk_file_size = 0;
/* for each file in the disk */
node = disk->filelist;
while (node) {
struct snapraid_file* file;
file = node->data;
node = node->next; /* next node */
/* check fragmentation */
if (file->size) {
block_off_t prev_pos;
int fragmented;
fragmented = 0;
prev_pos = file->blockvec[0].parity_pos;
for(i=1;i<file->blockmax;++i) {
block_off_t parity_pos = file->blockvec[i].parity_pos;
if (prev_pos + 1 != parity_pos) {
fragmented = 1;
++extra_fragment;
++disk_extra_fragment;
}
prev_pos = parity_pos;
}
if (fragmented) {
++file_fragmented;
++disk_file_fragmented;
}
}
/* count files */
++file_count;
++disk_file_count;
file_size += file->size;
disk_file_size += file->size;
}
fprintf(stdlog, "summary:disk_file_count:%s:%u\n", disk->name, disk_file_count);
fprintf(stdlog, "summary:disk_fragmented_file_count:%s:%u\n", disk->name, disk_file_fragmented);
fprintf(stdlog, "summary:disk_excess_fragment_count:%s:%u\n", disk->name, disk_extra_fragment);
fprintf(stdlog, "summary:disk_file_size:%s:%"PRIu64"\n", disk->name, disk_file_size);
}
printf("\n");
printf("Files: %u\n", file_count);
printf("Fragmented files: %u\n", file_fragmented);
printf("Excess fragments: %u\n", extra_fragment);
printf("Files size: %"PRIu64" GiB\n", file_size / (1024*1024*1024) );
printf("Parity size: %"PRIu64" GiB\n", blockmax * (uint64_t)state->block_size / (1024*1024*1024) );
printf("\n");
fprintf(stdlog, "summary:file_count:%u\n", file_count);
fprintf(stdlog, "summary:fragmented_file_count:%u\n", file_fragmented);
fprintf(stdlog, "summary:excess_fragment_count:%u\n", extra_fragment);
fprintf(stdlog, "summary:file_size:%"PRIu64"\n", file_size);
fprintf(stdlog, "summary:parity_size:%"PRIu64"\n", blockmax * (uint64_t)state->block_size);
fprintf(stdlog, "summary:hash:%s\n", hash_config_name(state->hash));
fprintf(stdlog, "summary:prev_hash:%s\n", hash_config_name(state->prevhash));
fprintf(stdlog, "summary:best_hash:%s\n", hash_config_name(state->besthash));
fprintf(stdlog, "summary:block_count:%u\n", blockmax);
fflush(stdlog);
/* copy the info a temp vector, and count bad/rehash/unsynced blocks */
infomap = malloc_nofail(blockmax * sizeof(snapraid_info));
bad = 0;
count = 0;
rehash = 0;
unsynced_blocks = 0;
for(i=0;i<blockmax;++i) {
int one_invalid;
int one_valid;
snapraid_info info = info_get(&state->infoarr, i);
/* for each disk */
one_invalid = 0;
one_valid = 0;
for(node_disk=state->disklist;node_disk!=0;node_disk=node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = disk_block_get(disk, i);
if (block_has_file(block))
one_valid = 1;
if (block_has_invalid_parity(block))
one_invalid = 1;
}
/* if both valid and invalid, we need to update */
if (one_invalid && one_valid) {
++unsynced_blocks;
}
/* skip unused blocks */
if (info != 0) {
if (info_get_bad(info))
++bad;
if (info_get_rehash(info))
++rehash;
infomap[count++] = info;
}
if (state->opt.gui) {
if (info != 0)
fprintf(stdlog, "block:%u:%"PRIu64":%s:%s:%s:%s\n", i, (uint64_t)info_get_time(info), one_valid ? "used": "", one_invalid ? "unsynced" : "", info_get_bad(info) ? "bad" : "", info_get_rehash(info) ? "rehash" : "");
else
fprintf(stdlog, "block_noinfo:%u:%s:%s\n", i, one_valid ? "used": "", one_invalid ? "unsynced" : "");
}
}
fprintf(stdlog, "summary:has_unsynced:%u\n", unsynced_blocks);
fprintf(stdlog, "summary:has_rehash:%u\n", rehash);
fprintf(stdlog, "summary:has_bad:%u\n", bad);
fprintf(stdlog, "summary:time_count:%u\n", count);
fflush(stdlog);
if (!count) {
fprintf(stderr, "The array is empty.\n");
free(infomap);
return 0;
}
/* sort the info to get the time info */
qsort(infomap, count, sizeof(snapraid_info), info_time_compare);
/* info for making the graph */
i = 0;
while (i < count) {
unsigned j = i + 1;
while (j < count && info_get_time(infomap[i]) == info_get_time(infomap[j]))
++j;
fprintf(stdlog, "time:%"PRIu64":%u\n", (uint64_t)info_get_time(infomap[i]), j - i);
i = j;
}
fflush(stdlog);
oldest = info_get_time(infomap[0]);
median = info_get_time(infomap[count / 2]);
newest = info_get_time(infomap[count - 1]);
dayoldest = day_ago(oldest, now);
daymedian = day_ago(median, now);
daynewest = day_ago(newest, now);
/* compute graph limits */
barpos = 0;
barmax = 0;
for(i=0;i<GRAPH_COLUMN;++i) {
time_t limit;
unsigned step;
limit = oldest + (newest - oldest) * (i+1) / GRAPH_COLUMN;
step = 0;
while (barpos < count && info_get_time(infomap[barpos]) <= limit) {
++barpos;
++step;
}
if (step > barmax)
barmax = step;
bar[i] = step;
}
/* print the graph */
for(y=0;y<GRAPH_ROW;++y) {
if (y == 0)
printf("%3u%% ", barmax * 100 / count);
else if (y == GRAPH_ROW - 1)
printf(" 0%% ");
else if (y == GRAPH_ROW/2)
printf("%3u%% ", barmax * 50 / count);
else
printf(" ");
for(x=0;x<GRAPH_COLUMN;++x) {
unsigned pivot = barmax * (GRAPH_ROW-y) / GRAPH_ROW;
/* if it's the baseline */
if (y == GRAPH_ROW-1) {
if (bar[x] >= pivot) {
printf("*");
} else if (bar[x] > 0) {
printf("+");
} else {
printf("_");
}
} else {
unsigned halfpivot = pivot - barmax / GRAPH_ROW / 2;
if (bar[x] >= pivot) {
printf("*");
} else if (bar[x] >= halfpivot) {
printf("+");
} else {
printf(" ");
}
}
}
printf("\n");
}
printf(" %3u days ago of the last scrub %3u\n", dayoldest, daynewest);
printf("\n");
printf("The oldest block was scrubbed %u days ago, the median %u, the newest %u.\n", dayoldest, daymedian, daynewest);
printf("\n");
if (unsynced_blocks) {
printf("WARNING! The array is NOT fully synced.\n");
printf("You have a sync in progress at %u%%.\n", (blockmax - unsynced_blocks) * 100 / blockmax);
} else {
printf("No sync is in progress.\n");
}
if (rehash) {
printf("You have a rehash in progress at %u%%.\n", (count - rehash) * 100 / count);
} else {
if (state->besthash != state->hash) {
printf("No rehash is in progress, but for optimal performance one is recommended.\n");
} else {
printf("No rehash is in progress or needed.\n");
}
}
if (bad) {
printf("DANGER! In the array there are %u silent errors!\n\n", bad);
printf("They are at blocks:");
/* print all the errors */
for(i=0;i<blockmax;++i) {
snapraid_info info = info_get(&state->infoarr, i);
/* skip unused blocks */
if (info == 0)
continue;
if (info_get_bad(info))
printf(" %u", i);
}
printf("\n\n");
printf("To fix them use the command 'snapraid -e fix'.\n");
printf("The errors will disapper from the 'status' at the next 'scrub' command.\n");
} else {
printf("No silent error detected.\n");
}
/* free the temp vector */
free(infomap);
return 0;
}
static int state_dry_process(struct snapraid_state* state, struct snapraid_parity_handle* parity_handle, block_off_t blockstart, block_off_t blockmax)
{
struct snapraid_io io;
struct snapraid_handle* handle;
unsigned diskmax;
block_off_t blockcur;
unsigned j;
unsigned buffermax;
int ret;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
unsigned error;
unsigned io_error;
unsigned l;
unsigned* waiting_map;
unsigned waiting_mac;
char esc_buffer[ESC_MAX];
handle = handle_mapping(state, &diskmax);
/* we need 1 * data + 2 * parity */
buffermax = diskmax + 2 * state->level;
/* initialize the io threads */
io_init(&io, state, state->opt.io_cache, buffermax, dry_data_reader, handle, diskmax, dry_parity_reader, 0, parity_handle, state->level);
/* possibly waiting disks */
waiting_mac = diskmax > RAID_PARITY_MAX ? diskmax : RAID_PARITY_MAX;
waiting_map = malloc_nofail(waiting_mac * sizeof(unsigned));
error = 0;
io_error = 0;
/* drop until now */
state_usage_waste(state);
countmax = blockmax - blockstart;
countsize = 0;
countpos = 0;
/* start all the worker threads */
io_start(&io, blockstart, blockmax, &block_is_enabled, 0);
state_progress_begin(state, blockstart, blockmax, countmax);
while (1) {
void** buffer;
/* go to the next block */
blockcur = io_read_next(&io, &buffer);
if (blockcur >= blockmax)
break;
/* until now is scheduling */
state_usage_sched(state);
/* for each disk, process the block */
for (j = 0; j < diskmax; ++j) {
struct snapraid_task* task;
int read_size;
struct snapraid_block* block;
struct snapraid_disk* disk;
unsigned diskcur;
/* until now is misc */
state_usage_misc(state);
/* get the next task */
task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
/* until now is disk */
state_usage_disk(state, handle, waiting_map, waiting_mac);
/* get the task results */
disk = task->disk;
block = task->block;
read_size = task->read_size;
/* if the disk position is not used */
if (!disk)
continue;
/* if the block is not used */
if (!block_has_file(block))
continue;
/* handle error conditions */
if (task->state == TASK_STATE_IOERROR) {
++io_error;
goto bail;
}
if (task->state == TASK_STATE_ERROR) {
++error;
goto bail;
}
if (task->state == TASK_STATE_ERROR_CONTINUE) {
++error;
continue;
}
if (task->state == TASK_STATE_IOERROR_CONTINUE) {
++io_error;
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, task->path);
log_fatal("Stopping at block %u\n", blockcur);
goto bail;
/* LCOV_EXCL_STOP */
}
/* otherwise continue */
continue;
}
if (task->state != TASK_STATE_DONE) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in task state\n");
os_abort();
/* LCOV_EXCL_STOP */
}
countsize += read_size;
}
/* until now is misc */
state_usage_misc(state);
/* read the parity */
for (l = 0; l < state->level; ++l) {
struct snapraid_task* task;
unsigned levcur;
task = io_parity_read(&io, &levcur, waiting_map, &waiting_mac);
/* until now is parity */
state_usage_parity(state, waiting_map, waiting_mac);
/* handle error conditions */
if (task->state == TASK_STATE_IOERROR) {
++io_error;
goto bail;
}
if (task->state == TASK_STATE_ERROR) {
++error;
goto bail;
}
if (task->state == TASK_STATE_ERROR_CONTINUE) {
++error;
continue;
}
if (task->state == TASK_STATE_IOERROR_CONTINUE) {
++io_error;
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in the %s disk, it isn't possible to scrub.\n", lev_name(levcur));
log_fatal("Ensure that disk '%s' is sane and can be read.\n", lev_config_name(levcur));
log_fatal("Stopping at block %u\n", blockcur);
goto bail;
/* LCOV_EXCL_STOP */
}
continue;
}
if (task->state != TASK_STATE_DONE) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in task state\n");
os_abort();
/* LCOV_EXCL_STOP */
}
}
/* count the number of processed block */
++countpos;
/* progress */
if (state_progress(state, &io, blockcur, countpos, countmax, countsize)) {
/* LCOV_EXCL_START */
break;
/* LCOV_EXCL_STOP */
}
}
state_progress_end(state, countpos, countmax, countsize);
state_usage_print(state);
bail:
/* stop all the worker threads */
io_stop(&io);
for (j = 0; j < diskmax; ++j) {
struct snapraid_file* file = handle[j].file;
struct snapraid_disk* disk = handle[j].disk;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc(file->sub, esc_buffer), strerror(errno));
log_fatal("DANGER! Unexpected close error in a data disk.\n");
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
if (error || io_error) {
msg_status("\n");
msg_status("%8u file errors\n", error);
msg_status("%8u io errors\n", io_error);
} else {
msg_status("Everything OK\n");
}
if (error)
log_fatal("DANGER! Unexpected errors!\n");
if (io_error)
log_fatal("DANGER! Unexpected input/output errors!\n");
free(handle);
free(waiting_map);
io_done(&io);
if (error + io_error != 0)
return -1;
return 0;
}
