bool rm_offsets_match(char *path1, char *path2) {
bool result = FALSE;
int fd1 = rm_sys_open(path1, O_RDONLY);
if(fd1 == -1) {
rm_log_info_line("Error opening %s in rm_offsets_match", path1);
return FALSE;
}
int fd2 = rm_sys_open(path2, O_RDONLY);
if(fd2 == -1) {
rm_log_info_line("Error opening %s in rm_offsets_match", path2);
rm_sys_close(fd1);
return FALSE;
}
RmOff file1_offset_next = 0;
RmOff file2_offset_next = 0;
RmOff file_offset_current = 0;
while(!result &&
(rm_offset_get_from_fd(fd1, file_offset_current, &file1_offset_next) ==
rm_offset_get_from_fd(fd2, file_offset_current, &file2_offset_next)) &&
file1_offset_next != 0 && file1_offset_next == file2_offset_next) {
if(file1_offset_next == file_offset_current) {
/* phew, we got to the end */
result = TRUE;
break;
}
file_offset_current = file1_offset_next;
}
rm_sys_close(fd2);
rm_sys_close(fd1);
return result;
}
RmOffsetTable rm_offset_create_table(const char *path) {
int fd = rm_sys_open(path, O_RDONLY);
if(fd == -1) {
rm_log_info("Error opening %s in setup_fiemap_extents\n", path);
return NULL;
}
/* struct fiemap does not allocate any extents by default,
* so we choose ourself how many of them we allocate.
* */
const int n_extents = 256;
struct fiemap *fiemap = g_malloc0(sizeof(struct fiemap) + n_extents * sizeof(struct fiemap_extent));
struct fiemap_extent *fm_ext = fiemap->fm_extents;
/* data structure we save our offsets in */
GSequence *self = g_sequence_new((GFreeFunc)rm_offset_free_func);
bool last = false;
while(!last) {
fiemap->fm_flags = 0;
fiemap->fm_extent_count = n_extents;
fiemap->fm_length = FIEMAP_MAX_OFFSET;
if(ioctl(fd, FS_IOC_FIEMAP, (unsigned long) fiemap) == -1) {
break;
}
/* This might happen on empty files - those have no
* extents, but they have an offset on the disk.
*/
if(fiemap->fm_mapped_extents <= 0) {
break;
}
/* used for detecting contiguous extents, which we ignore */
unsigned long expected = 0;
/* Remember all non contiguous extents */
for(unsigned i = 0; i < fiemap->fm_mapped_extents && !last; i++) {
if (i == 0 || fm_ext[i].fe_physical != expected) {
RmOffsetEntry *offset_entry = g_slice_new(RmOffsetEntry);
offset_entry->logical = fm_ext[i].fe_logical;
offset_entry->physical = fm_ext[i].fe_physical;
g_sequence_append(self, offset_entry);
}
expected = fm_ext[i].fe_physical + fm_ext[i].fe_length;
fiemap->fm_start = fm_ext[i].fe_logical + fm_ext[i].fe_length;
last = fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST;
}
}
rm_sys_close(fd);
g_free(fiemap);
g_sequence_sort(self, (GCompareDataFunc)rm_offset_sort_logical, NULL);
return self;
}
RmOff rm_offset_get_from_path(const char *path, RmOff file_offset,
RmOff *file_offset_next) {
int fd = rm_sys_open(path, O_RDONLY);
if(fd == -1) {
rm_log_info("Error opening %s in rm_offset_get_from_path\n", path);
return 0;
}
RmOff result = rm_offset_get_from_fd(fd, file_offset, file_offset_next);
rm_sys_close(fd);
return result;
}
static gint64 rm_hasher_unbuffered_read(RmHasher *hasher, GThreadPool *hashpipe,
RmDigest *digest, char *path, gint64 start_offset,
gint64 bytes_to_read) {
gint32 bytes_read = 0;
gint64 total_bytes_read = 0;
guint64 file_offset = start_offset;
if(bytes_to_read == 0) {
RmStat stat_buf;
if(rm_sys_stat(path, &stat_buf) != -1) {
bytes_to_read = MAX(stat_buf.st_size - start_offset, 0);
}
}
/* how many buffers to read? */
const gint16 N_BUFFERS = MIN(4, DIVIDE_CEIL(bytes_to_read, hasher->buf_size));
struct iovec readvec[N_BUFFERS + 1];
int fd = 0;
fd = rm_sys_open(path, O_RDONLY);
if(fd == -1) {
rm_log_info("open(2) failed for %s: %s\n", path, g_strerror(errno));
goto finish;
}
/* preadv() is beneficial for large files since it can cut the
* number of syscall heavily. I suggest N_BUFFERS=4 as good
* compromise between memory and cpu.
*
* With 16 buffers: 43% cpu 33,871 total
* With 8 buffers: 43% cpu 32,098 total
* With 4 buffers: 42% cpu 32,091 total
* With 2 buffers: 44% cpu 32,245 total
* With 1 buffers: 45% cpu 34,491 total
*/
/* Give the kernel scheduler some hints */
rm_hasher_request_readahead(fd, start_offset, bytes_to_read);
/* Initialize the buffers to begin with.
* After a buffer is full, a new one is retrieved.
*/
RmBuffer **buffers;
buffers = g_slice_alloc(sizeof(*buffers) * N_BUFFERS);
memset(readvec, 0, sizeof(readvec));
for(int i = 0; i < N_BUFFERS; ++i) {
/* buffer is one contignous memory block */
buffers[i] = rm_buffer_get(hasher->mem_pool);
readvec[i].iov_base = buffers[i]->data;
readvec[i].iov_len = hasher->buf_size;
}
while((bytes_to_read == 0 || total_bytes_read < bytes_to_read) &&
(bytes_read = rm_sys_preadv(fd, readvec, N_BUFFERS, file_offset)) > 0) {
bytes_read =
MIN(bytes_read, bytes_to_read - total_bytes_read); /* ignore over-reads */
int blocks = DIVIDE_CEIL(bytes_read, hasher->buf_size);
rm_assert_gentle(blocks <= N_BUFFERS);
total_bytes_read += bytes_read;
file_offset += bytes_read;
for(int i = 0; i < blocks; ++i) {
/* Get the RmBuffer from the datapointer */
RmBuffer *buffer = buffers[i];
buffer->len = MIN(hasher->buf_size, bytes_read - i * hasher->buf_size);
buffer->digest = digest;
buffer->user_data = NULL;
/* Send it to the hasher */
rm_util_thread_pool_push(hashpipe, buffer);
/* Allocate a new buffer - hasher will release the old buffer */
buffers[i] = rm_buffer_get(hasher->mem_pool);
readvec[i].iov_base = buffers[i]->data;
readvec[i].iov_len = hasher->buf_size;
}
}
if(bytes_read == -1) {
rm_log_perror("preadv failed");
} else if(total_bytes_read != bytes_to_read) {
rm_log_error_line(_("Something went wrong reading %s; expected %li bytes, "
"got %li; ignoring"),
path, (long int)bytes_to_read, (long int)total_bytes_read);
}
/* Release the rest of the buffers */
for(int i = 0; i < N_BUFFERS; ++i) {
rm_buffer_release(buffers[i]);
}
g_slice_free1(sizeof(*buffers) * N_BUFFERS, buffers);
finish:
if(fd > 0) {
rm_sys_close(fd);
}
return total_bytes_read;
}
/* Method to test if a file is non stripped binary. Uses libelf*/
bool rm_util_is_nonstripped(_U const char *path, _U RmStat *statp) {
bool is_ns = false;
#if HAVE_LIBELF
g_return_val_if_fail(path, false);
if(statp && (statp->st_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) == 0) {
return false;
}
/* inspired by "jschmier"'s answer at http://stackoverflow.com/a/5159890 */
int fd;
/* ELF handle */
Elf *elf;
/* section descriptor pointer */
Elf_Scn *scn;
/* section header */
GElf_Shdr shdr;
/* Open ELF file to obtain file descriptor */
if((fd = rm_sys_open(path, O_RDONLY)) == -1) {
rm_log_warning_line(_("cannot open file '%s' for nonstripped test: "), path);
rm_log_perror("");
return 0;
}
/* Protect program from using an older library */
if(elf_version(EV_CURRENT) == EV_NONE) {
rm_log_error_line(_("ELF Library is out of date!"));
return false;
}
/* Initialize elf pointer for examining contents of file */
elf = elf_begin(fd, ELF_C_READ, NULL);
/* Initialize section descriptor pointer so that elf_nextscn()
* returns a pointer to the section descriptor at index 1.
* */
scn = NULL;
/* Iterate through ELF sections */
while((scn = elf_nextscn(elf, scn)) != NULL) {
/* Retrieve section header */
gelf_getshdr(scn, &shdr);
/* If a section header holding a symbol table (.symtab)
* is found, this ELF file has not been stripped. */
if(shdr.sh_type == SHT_SYMTAB) {
is_ns = true;
break;
}
}
elf_end(elf);
rm_sys_close(fd);
#endif
return is_ns;
}
