int add_sort_list(char *path, int priority, int source, char *source_path[])
{
int i, n;
char filename[4096];
struct stat buf;
TRACE("add_sort_list: filename %s, priority %d\n", path, priority);
if(strlen(path) > 1 && strcmp(path + strlen(path) - 2, "/*") == 0)
path[strlen(path) - 2] = '\0';
TRACE("add_sort_list: filename %s, priority %d\n", path, priority);
re_read:
if(path[0] == '/' || strncmp(path, "./", 2) == 0 || strncmp(path, "../", 3) == 0 || mkisofs_style == 1) {
if(lstat(path, &buf) == -1)
goto error;
TRACE("adding filename %s, priority %d, st_dev %llx, st_ino %llx\n", path, priority, buf.st_dev, buf.st_ino);
ADD_ENTRY(buf, priority);
return TRUE;
}
for(i = 0, n = 0; i < source; i++) {
strcat(strcat(strcpy(filename, source_path[i]), "/"), path);
if(lstat(filename, &buf) == -1) {
if(!(errno == ENOENT || errno == ENOTDIR))
goto error;
continue;
}
ADD_ENTRY(buf, priority);
n ++;
}
if(n == 0 && mkisofs_style == -1 && lstat(path, &buf) != -1) {
ERROR("WARNING: Mkisofs style sortlist detected! This is supported but please\n");
ERROR("convert to mksquashfs style sortlist! A sortlist entry ");
ERROR("should be\neither absolute (starting with ");
ERROR("'/') start with './' or '../' (taken to be\nrelative to $PWD), otherwise it ");
ERROR("is assumed the entry is relative to one\nof the source directories, i.e. with ");
ERROR("\"mksquashfs test test.sqsh\",\nthe sortlist ");
ERROR("entry \"file\" is assumed to be inside the directory test.\n\n");
mkisofs_style = 1;
goto re_read;
}
mkisofs_style = 0;
if(n == 1)
return TRUE;
if(n > 1)
BAD_ERROR(" Ambiguous sortlist entry \"%s\"\n\nIt maps to more than one source entry! Please use an absolute path.\n", path);
error:
fprintf(stderr, "Cannot stat sortlist entry \"%s\"\n", path);
fprintf(stderr, "This is probably because you're using the wrong file\n");
fprintf(stderr, "path relative to the source directories\n");
return FALSE;
}
void add_pseudo_file(struct pseudo_dev *dev)
{
pseudo_file = realloc(pseudo_file, (pseudo_count + 1) *
sizeof(struct pseudo_dev *));
if(pseudo_file == NULL)
BAD_ERROR("Failed to realloc pseudo_file\n");
dev->pseudo_id = pseudo_count;
pseudo_file[pseudo_count ++] = dev;
}
void *info_thrd(void *arg)
{
sigset_t sigmask;
struct timespec timespec = { .tv_sec = 1, .tv_nsec = 0 };
int sig, waiting = 0;
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGQUIT);
sigaddset(&sigmask, SIGHUP);
while(1) {
if(waiting)
sig = sigtimedwait(&sigmask, NULL, ×pec);
else
sig = sigwaitinfo(&sigmask, NULL);
if(sig == -1) {
switch(errno) {
case EAGAIN:
/* interval timed out */
waiting = 0;
/* FALLTHROUGH */
case EINTR:
/* if waiting, the wait will be longer, but
that's OK */
continue;
default:
BAD_ERROR("sigtimedwait/sigwaitinfo failed "
"because %s\n", strerror(errno));
}
}
if(sig == SIGQUIT && !waiting) {
if(pathname)
INFO("%s\n", pathname);
/* set one second interval period, if ^\ received
within then, dump queue and cache status */
waiting = 1;
} else
dump_state();
}
}
void init_info()
{
pthread_create(&info_thread, NULL, info_thrd, NULL);
}
void *progress_thrd(void *arg)
{
struct timespec requested_time, remaining;
struct itimerval itimerval;
struct winsize winsize;
if(ioctl(1, TIOCGWINSZ, &winsize) == -1) {
if(isatty(STDOUT_FILENO))
printf("TIOCGWINSZ ioctl failed, defaulting to 80 "
"columns\n");
columns = 80;
} else
columns = winsize.ws_col;
signal(SIGWINCH, sigwinch_handler);
signal(SIGALRM, sigalrm_handler);
itimerval.it_value.tv_sec = 0;
itimerval.it_value.tv_usec = 250000;
itimerval.it_interval.tv_sec = 0;
itimerval.it_interval.tv_usec = 250000;
setitimer(ITIMER_REAL, &itimerval, NULL);
requested_time.tv_sec = 0;
requested_time.tv_nsec = 250000000;
while(1) {
int res = nanosleep(&requested_time, &remaining);
if(res == -1 && errno != EINTR)
BAD_ERROR("nanosleep failed in progress thread\n");
if(progress_enabled) {
pthread_mutex_lock(&progress_mutex);
progress_bar(cur_uncompressed, estimated_uncompressed,
columns);
pthread_mutex_unlock(&progress_mutex);
}
}
}
struct queue *queue_init(int size)
{
struct queue *queue = malloc(sizeof(struct queue));
if(queue == NULL)
MEM_ERROR();
if(add_overflow(size, 1) ||
multiply_overflow(size + 1, sizeof(void *)))
BAD_ERROR("Size too large in queue_init\n");
queue->data = malloc(sizeof(void *) * (size + 1));
if(queue->data == NULL)
MEM_ERROR();
queue->size = size + 1;
queue->readp = queue->writep = 0;
pthread_mutex_init(&queue->mutex, NULL);
pthread_cond_init(&queue->empty, NULL);
pthread_cond_init(&queue->full, NULL);
return queue;
}
static void dump_pseudo(struct pseudo *pseudo, char *string)
{
int i, res;
char *path;
for(i = 0; i < pseudo->names; i++) {
struct pseudo_entry *entry = &pseudo->name[i];
if(string) {
res = asprintf(&path, "%s/%s", string, entry->name);
if(res == -1)
BAD_ERROR("asprintf failed in dump_pseudo\n");
} else
path = entry->name;
if(entry->dev)
ERROR("%s %c 0%o %d %d %d %d\n", path, entry->dev->type,
entry->dev->mode & ~S_IFMT, entry->dev->uid,
entry->dev->gid, entry->dev->major,
entry->dev->minor);
if(entry->pseudo)
dump_pseudo(entry->pseudo, path);
if(string)
free(path);
}
}
int read_pseudo_def(struct pseudo **pseudo, char *def)
{
int n, bytes;
unsigned int major = 0, minor = 0, mode;
char filename[2048], type, suid[100], sgid[100], *ptr;
long long uid, gid;
struct pseudo_dev *dev;
n = sscanf(def, "%s %c %o %s %s %n", filename, &type, &mode, suid,
sgid, &bytes);
if(n < 5) {
ERROR("Not enough or invalid arguments in pseudo file "
"definition\n");
goto error;
}
switch(type) {
case 'b':
case 'c':
n = sscanf(def + bytes, "%u %u", &major, &minor);
if(n < 2) {
ERROR("Not enough or invalid arguments in pseudo file "
"definition\n");
goto error;
}
if(major > 0xfff) {
ERROR("Major %d out of range\n", major);
goto error;
}
if(minor > 0xfffff) {
ERROR("Minor %d out of range\n", minor);
goto error;
}
case 'f':
if(def[bytes] == '\0') {
ERROR("Not enough arguments in pseudo file "
"definition\n");
goto error;
}
break;
case 'd':
case 'm':
break;
default:
ERROR("Unsupported type %c\n", type);
goto error;
}
if(mode > 07777) {
ERROR("Mode %o out of range\n", mode);
goto error;
}
uid = strtoll(suid, &ptr, 10);
if(*ptr == '\0') {
if(uid < 0 || uid > ((1LL << 32) - 1)) {
ERROR("Uid %s out of range\n", suid);
goto error;
}
} else {
struct passwd *pwuid = getpwnam(suid);
if(pwuid)
uid = pwuid->pw_uid;
else {
ERROR("Uid %s invalid uid or unknown user\n", suid);
goto error;
}
}
gid = strtoll(sgid, &ptr, 10);
if(*ptr == '\0') {
if(gid < 0 || gid > ((1LL << 32) - 1)) {
ERROR("Gid %s out of range\n", sgid);
goto error;
}
} else {
struct group *grgid = getgrnam(sgid);
if(grgid)
gid = grgid->gr_gid;
else {
ERROR("Gid %s invalid uid or unknown user\n", sgid);
goto error;
}
}
switch(type) {
case 'b':
mode |= S_IFBLK;
break;
case 'c':
mode |= S_IFCHR;
break;
case 'd':
mode |= S_IFDIR;
break;
case 'f':
mode |= S_IFREG;
break;
}
dev = malloc(sizeof(struct pseudo_dev));
if(dev == NULL)
BAD_ERROR("Failed to create pseudo_dev\n");
dev->type = type;
dev->mode = mode;
dev->uid = uid;
dev->gid = gid;
dev->major = major;
dev->minor = minor;
if(type == 'f') {
int res;
printf("Executing dynamic pseudo file\n");
printf("\t\"%s\"\n", def);
res = exec_file(def + bytes, dev);
if(res == -1) {
ERROR("Failed to execute dynamic pseudo file definition"
" \"%s\"\n", def);
return FALSE;
}
add_pseudo_file(dev);
}
*pseudo = add_pseudo(*pseudo, dev, filename, filename);
return TRUE;
error:
ERROR("Bad pseudo file definition \"%s\"\n", def);
return FALSE;
}
/*
* Add pseudo device target to the set of pseudo devices. Pseudo_dev
* describes the pseudo device attributes.
*/
struct pseudo *add_pseudo(struct pseudo *pseudo, struct pseudo_dev *pseudo_dev,
char *target, char *alltarget)
{
char targname[1024];
int i;
target = get_component(target, targname);
if(pseudo == NULL) {
if((pseudo = malloc(sizeof(struct pseudo))) == NULL)
BAD_ERROR("failed to allocate pseudo file\n");
pseudo->names = 0;
pseudo->count = 0;
pseudo->name = NULL;
}
for(i = 0; i < pseudo->names; i++)
if(strcmp(pseudo->name[i].name, targname) == 0)
break;
if(i == pseudo->names) {
/* allocate new name entry */
pseudo->names ++;
pseudo->name = realloc(pseudo->name, (i + 1) *
sizeof(struct pseudo_entry));
if(pseudo->name == NULL)
BAD_ERROR("failed to allocate pseudo file\n");
pseudo->name[i].name = strdup(targname);
if(target[0] == '\0') {
/* at leaf pathname component */
pseudo->name[i].pseudo = NULL;
pseudo->name[i].pathname = strdup(alltarget);
pseudo->name[i].dev = pseudo_dev;
} else {
/* recurse adding child components */
pseudo->name[i].dev = NULL;
pseudo->name[i].pseudo = add_pseudo(NULL, pseudo_dev,
target, alltarget);
}
} else {
/* existing matching entry */
if(pseudo->name[i].pseudo == NULL) {
/* No sub-directory which means this is the leaf
* component of a pre-existing pseudo file.
*/
if(target[0] != '\0') {
/* entry must exist as a 'd' type pseudo file */
if(pseudo->name[i].dev->type == 'd')
/* recurse adding child components */
pseudo->name[i].pseudo =
add_pseudo(NULL, pseudo_dev,
target, alltarget);
else
ERROR("%s already exists as a non "
"directory. Ignoring %s!\n",
targname, alltarget);
} else if(memcmp(pseudo_dev, pseudo->name[i].dev,
sizeof(struct pseudo_dev)) != 0)
ERROR("%s already exists as a different pseudo "
"definition. Ignoring!\n", alltarget);
else ERROR("%s already exists as an identical "
"pseudo definition!\n", alltarget);
} else {
/* sub-directory exists which means this can only be a
* 'd' type pseudo file */
if(target[0] == '\0') {
if(pseudo->name[i].dev == NULL &&
pseudo_dev->type == 'd') {
pseudo->name[i].pathname =
strdup(alltarget);
pseudo->name[i].dev = pseudo_dev;
} else
ERROR("%s already exists as a "
"directory. Ignoring %s!\n",
targname, alltarget);
} else
/* recurse adding child components */
add_pseudo(pseudo->name[i].pseudo, pseudo_dev,
target, alltarget);
}
}
return pseudo;
}
static struct file_buffer *get_fragment(struct fragment *fragment, char *data_buffer, int fd) {
struct squashfs_fragment_entry *disk_fragment;
struct file_buffer *buffer, *compressed_buffer;
long long start_block;
int res, size, index = fragment->index;
char locked;
/*
* Lookup fragment block in cache.
* If the fragment block doesn't exist, then get the compressed version
* from the writer cache or off disk, and decompress it.
*
* This routine has two things which complicate the code:
*
* 1. Multiple threads can simultaneously lookup/create the
* same buffer. This means a buffer needs to be "locked"
* when it is being filled in, to prevent other threads from
* using it when it is not ready. This is because we now do
* fragment duplicate checking in parallel.
* 2. We have two caches which need to be checked for the
* presence of fragment blocks: the normal fragment cache
* and a "reserve" cache. The reserve cache is used to
* prevent an unnecessary pipeline stall when the fragment cache
* is full of fragments waiting to be compressed.
*/
pthread_cleanup_push((void *)pthread_mutex_unlock, &dup_mutex);
pthread_mutex_lock(&dup_mutex);
again:
buffer = cache_lookup_nowait(fragment_buffer, index, &locked);
if (buffer) {
pthread_mutex_unlock(&dup_mutex);
if (locked)
/* got a buffer being filled in. Wait for it */
cache_wait_unlock(buffer);
goto finished;
}
/* not in fragment cache, is it in the reserve cache? */
buffer = cache_lookup_nowait(reserve_cache, index, &locked);
if (buffer) {
pthread_mutex_unlock(&dup_mutex);
if (locked)
/* got a buffer being filled in. Wait for it */
cache_wait_unlock(buffer);
goto finished;
}
/* in neither cache, try to get it from the fragment cache */
buffer = cache_get_nowait(fragment_buffer, index);
if (!buffer) {
/*
* no room, get it from the reserve cache, this is
* dimensioned so it will always have space (no more than
* processors + 1 can have an outstanding reserve buffer)
*/
buffer = cache_get_nowait(reserve_cache, index);
if (!buffer) {
/* failsafe */
ERROR("no space in reserve cache\n");
goto again;
}
}
pthread_mutex_unlock(&dup_mutex);
compressed_buffer = cache_lookup(fwriter_buffer, index);
pthread_cleanup_push((void *)pthread_mutex_unlock, &fragment_mutex);
pthread_mutex_lock(&fragment_mutex);
disk_fragment = &fragment_table[index];
size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size);
start_block = disk_fragment->start_block;
pthread_cleanup_pop(1);
if (SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) {
int error;
char *data;
if (compressed_buffer)
data = compressed_buffer->data;
else {
res = read_filesystem(fd, start_block, size, data_buffer);
if (res == 0) {
ERROR("Failed to read fragment from output" " filesystem\n");
BAD_ERROR("Output filesystem corrupted?\n");
}
data = data_buffer;
}
res = compressor_uncompress(comp, buffer->data, data, size, block_size, &error);
if (res == -1)
BAD_ERROR("%s uncompress failed with error code %d\n", comp->name, error);
} else if (compressed_buffer)
memcpy(buffer->data, compressed_buffer->data, size);
else {
res = read_filesystem(fd, start_block, size, buffer->data);
if (res == 0) {
ERROR("Failed to read fragment from output " "filesystem\n");
BAD_ERROR("Output filesystem corrupted?\n");
}
}
cache_unlock(buffer);
cache_block_put(compressed_buffer);
finished:
pthread_cleanup_pop(0);
return buffer;
}
void *frag_thrd(void *destination_file) {
sigset_t sigmask, old_mask;
char *data_buffer;
int fd;
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGINT);
sigaddset(&sigmask, SIGTERM);
sigaddset(&sigmask, SIGUSR1);
pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask);
fd = open(destination_file, O_RDONLY);
if (fd == -1)
BAD_ERROR("frag_thrd: can't open destination for reading\n");
data_buffer = malloc(SQUASHFS_FILE_MAX_SIZE);
if (data_buffer == NULL)
MEM_ERROR();
pthread_cleanup_push((void *)pthread_mutex_unlock, &dup_mutex);
while (1) {
struct file_buffer *file_buffer = queue_get(to_process_frag);
struct file_buffer *buffer;
int sparse = checksum_sparse(file_buffer);
struct file_info *dupl_ptr;
long long file_size;
unsigned short checksum;
char flag;
int res;
if (sparse_files && sparse) {
file_buffer->c_byte = 0;
file_buffer->fragment = FALSE;
} else
file_buffer->c_byte = file_buffer->size;
/*
* Specutively pull into the fragment cache any fragment blocks
* which contain fragments which *this* fragment may be
* be a duplicate.
*
* By ensuring the fragment block is in cache ahead of time
* should eliminate the parallelisation stall when the
* main thread needs to read the fragment block to do a
* duplicate check on it.
*
* If this is a fragment belonging to a larger file
* (with additional blocks) then ignore it. Here we're
* interested in the "low hanging fruit" of files which
* consist of only a fragment
*/
if (file_buffer->file_size != file_buffer->size) {
seq_queue_put(to_main, file_buffer);
continue;
}
file_size = file_buffer->file_size;
pthread_mutex_lock(&dup_mutex);
dupl_ptr = dupl[DUP_HASH(file_size)];
pthread_mutex_unlock(&dup_mutex);
file_buffer->dupl_start = dupl_ptr;
file_buffer->duplicate = FALSE;
for (; dupl_ptr; dupl_ptr = dupl_ptr->next) {
if (file_size != dupl_ptr->file_size || file_size != dupl_ptr->fragment->size)
continue;
pthread_mutex_lock(&dup_mutex);
flag = dupl_ptr->have_frag_checksum;
checksum = dupl_ptr->fragment_checksum;
pthread_mutex_unlock(&dup_mutex);
/*
* If we have the checksum and it matches then
* read in the fragment block.
*
* If we *don't* have the checksum, then we are
* appending, and the fragment block is on the
* "old" filesystem. Read it in and checksum
* the entire fragment buffer
*/
if (!flag) {
buffer = get_fragment_cksum(dupl_ptr, data_buffer, fd, &checksum);
if (checksum != file_buffer->checksum) {
cache_block_put(buffer);
continue;
}
} else if (checksum == file_buffer->checksum)
buffer = get_fragment(dupl_ptr->fragment, data_buffer, fd);
else
continue;
res = memcmp(file_buffer->data, buffer->data + dupl_ptr->fragment->offset, file_size);
cache_block_put(buffer);
if (res == 0) {
struct file_buffer *dup = malloc(sizeof(*dup));
if (dup == NULL)
MEM_ERROR();
memcpy(dup, file_buffer, sizeof(*dup));
cache_block_put(file_buffer);
dup->dupl_start = dupl_ptr;
dup->duplicate = TRUE;
file_buffer = dup;
break;
}
}
seq_queue_put(to_main, file_buffer);
}
pthread_cleanup_pop(0);
}
int add_sort_list(char *path, int priority, int source, char *source_path[])
{
int i, n;
struct stat buf;
TRACE("add_sort_list: filename %s, priority %d\n", path, priority);
if(strlen(path) > 1 && strcmp(path + strlen(path) - 2, "/*") == 0)
path[strlen(path) - 2] = '\0';
TRACE("add_sort_list: filename %s, priority %d\n", path, priority);
re_read:
if(path[0] == '/' || strncmp(path, "./", 2) == 0 ||
strncmp(path, "../", 3) == 0 || mkisofs_style == 1) {
if(lstat(path, &buf) == -1)
goto error;
TRACE("adding filename %s, priority %d, st_dev %d, st_ino "
"%lld\n", path, priority, (int) buf.st_dev,
(long long) buf.st_ino);
ADD_ENTRY(buf, priority);
return TRUE;
}
for(i = 0, n = 0; i < source; i++) {
char *filename;
int res = asprintf(&filename, "%s/%s", source_path[i], path);
if(res == -1)
BAD_ERROR("asprintf failed in add_sort_list\n");
res = lstat(filename, &buf);
free(filename);
if(res == -1) {
if(!(errno == ENOENT || errno == ENOTDIR))
goto error;
continue;
}
ADD_ENTRY(buf, priority);
n ++;
}
if(n == 0 && mkisofs_style == -1 && lstat(path, &buf) != -1) {
ERROR("WARNING: Mkisofs style sortlist detected! This is "
"supported but please\n");
ERROR("convert to mksquashfs style sortlist! A sortlist entry");
ERROR(" should be\neither absolute (starting with ");
ERROR("'/') start with './' or '../' (taken to be\nrelative to "
"$PWD), otherwise it ");
ERROR("is assumed the entry is relative to one\nof the source "
"directories, i.e. with ");
ERROR("\"mksquashfs test test.sqsh\",\nthe sortlist ");
ERROR("entry \"file\" is assumed to be inside the directory "
"test.\n\n");
mkisofs_style = 1;
goto re_read;
}
mkisofs_style = 0;
if(n == 1)
return TRUE;
if(n > 1) {
ERROR(" Ambiguous sortlist entry \"%s\"\n\nIt maps to more "
"than one source entry! Please use an absolute path."
"\n", path);
return FALSE;
}
error:
ERROR_START("Cannot stat sortlist entry \"%s\"\n", path);
ERROR("This is probably because you're using the wrong file\n");
ERROR("path relative to the source directories.");
ERROR_EXIT(" Ignoring");
/*
* Historical note
* Failure to stat a sortlist entry is deliberately ignored, even
* though it is an error. Squashfs release 2.2 changed the behaviour
* to treat it as a fatal error, but it was changed back to
* the original behaviour to ignore it in release 2.2-r2 following
* feedback from users at the time.
*/
return TRUE;
}
