int
_kvm_minidump_initvtop(kvm_t *kd)
{
struct vmstate *vmst;
off_t off;
vmst = _kvm_malloc(kd, sizeof(*vmst));
if (vmst == 0) {
_kvm_err(kd, kd->program, "cannot allocate vm");
return (-1);
}
kd->vmst = vmst;
vmst->minidump = 1;
off = lseek(kd->pmfd, 0, SEEK_CUR);
if (pread(kd->pmfd, &vmst->hdr,
sizeof(vmst->hdr), 0) != sizeof(vmst->hdr)) {
_kvm_err(kd, kd->program, "cannot read dump header");
return (-1);
}
if (strncmp(MINIDUMP_MAGIC, vmst->hdr.magic,
sizeof(vmst->hdr.magic)) != 0) {
_kvm_err(kd, kd->program, "not a minidump for this platform");
return (-1);
}
if (vmst->hdr.version != MINIDUMP_VERSION) {
_kvm_err(kd, kd->program, "wrong minidump version. "
"Expected %d got %d", MINIDUMP_VERSION, vmst->hdr.version);
return (-1);
}
/* Skip header and msgbuf */
off = PAGE_SIZE + round_page(vmst->hdr.msgbufsize);
vmst->bitmap = _kvm_malloc(kd, vmst->hdr.bitmapsize);
if (vmst->bitmap == NULL) {
_kvm_err(kd, kd->program, "cannot allocate %d bytes for "
"bitmap", vmst->hdr.bitmapsize);
return (-1);
}
if (pread(kd->pmfd, vmst->bitmap, vmst->hdr.bitmapsize, off) !=
(ssize_t)vmst->hdr.bitmapsize) {
_kvm_err(kd, kd->program, "cannot read %d bytes for page bitmap",
vmst->hdr.bitmapsize);
return (-1);
}
off += round_page(vmst->hdr.bitmapsize);
vmst->ptemap = _kvm_malloc(kd, vmst->hdr.ptesize);
if (vmst->ptemap == NULL) {
_kvm_err(kd, kd->program, "cannot allocate %d bytes for "
"ptemap", vmst->hdr.ptesize);
return (-1);
}
if (pread(kd->pmfd, vmst->ptemap, vmst->hdr.ptesize, off) !=
(ssize_t)vmst->hdr.ptesize) {
_kvm_err(kd, kd->program, "cannot read %d bytes for ptemap",
vmst->hdr.ptesize);
return (-1);
}
off += vmst->hdr.ptesize;
/* Build physical address hash table for sparse pages */
inithash(kd, vmst->bitmap, vmst->hdr.bitmapsize, off);
return (0);
}
static int
_mips_minidump_initvtop(kvm_t *kd)
{
struct vmstate *vmst;
uint32_t *bitmap;
off_t off;
vmst = _kvm_malloc(kd, sizeof(*vmst));
if (vmst == NULL) {
_kvm_err(kd, kd->program, "cannot allocate vm");
return (-1);
}
kd->vmst = vmst;
if (kd->nlehdr.e_ident[EI_CLASS] == ELFCLASS64 ||
kd->nlehdr.e_flags & EF_MIPS_ABI2)
vmst->pte_size = 64;
else
vmst->pte_size = 32;
if (pread(kd->pmfd, &vmst->hdr,
sizeof(vmst->hdr), 0) != sizeof(vmst->hdr)) {
_kvm_err(kd, kd->program, "cannot read dump header");
return (-1);
}
if (strncmp(MINIDUMP_MAGIC, vmst->hdr.magic,
sizeof(vmst->hdr.magic)) != 0) {
_kvm_err(kd, kd->program, "not a minidump for this platform");
return (-1);
}
vmst->hdr.version = _kvm32toh(kd, vmst->hdr.version);
if (vmst->hdr.version != MINIDUMP_VERSION) {
_kvm_err(kd, kd->program, "wrong minidump version. "
"Expected %d got %d", MINIDUMP_VERSION, vmst->hdr.version);
return (-1);
}
vmst->hdr.msgbufsize = _kvm32toh(kd, vmst->hdr.msgbufsize);
vmst->hdr.bitmapsize = _kvm32toh(kd, vmst->hdr.bitmapsize);
vmst->hdr.ptesize = _kvm32toh(kd, vmst->hdr.ptesize);
vmst->hdr.kernbase = _kvm64toh(kd, vmst->hdr.kernbase);
vmst->hdr.dmapbase = _kvm64toh(kd, vmst->hdr.dmapbase);
vmst->hdr.dmapend = _kvm64toh(kd, vmst->hdr.dmapend);
/* Skip header and msgbuf */
off = MIPS_PAGE_SIZE + mips_round_page(vmst->hdr.msgbufsize);
bitmap = _kvm_malloc(kd, vmst->hdr.bitmapsize);
if (bitmap == NULL) {
_kvm_err(kd, kd->program, "cannot allocate %d bytes for "
"bitmap", vmst->hdr.bitmapsize);
return (-1);
}
if (pread(kd->pmfd, bitmap, vmst->hdr.bitmapsize, off) !=
(ssize_t)vmst->hdr.bitmapsize) {
_kvm_err(kd, kd->program, "cannot read %d bytes for page bitmap",
vmst->hdr.bitmapsize);
free(bitmap);
return (-1);
}
off += mips_round_page(vmst->hdr.bitmapsize);
vmst->ptemap = _kvm_malloc(kd, vmst->hdr.ptesize);
if (vmst->ptemap == NULL) {
_kvm_err(kd, kd->program, "cannot allocate %d bytes for "
"ptemap", vmst->hdr.ptesize);
free(bitmap);
return (-1);
}
if (pread(kd->pmfd, vmst->ptemap, vmst->hdr.ptesize, off) !=
(ssize_t)vmst->hdr.ptesize) {
_kvm_err(kd, kd->program, "cannot read %d bytes for ptemap",
vmst->hdr.ptesize);
free(bitmap);
return (-1);
}
off += vmst->hdr.ptesize;
/* Build physical address hash table for sparse pages */
_kvm_hpt_init(kd, &vmst->hpt, bitmap, vmst->hdr.bitmapsize, off,
MIPS_PAGE_SIZE, sizeof(*bitmap));
free(bitmap);
return (0);
}
/*
* Main program
*/
int main(int argc, char **argv)
{
unsigned long ofs, end_addr = 0;
unsigned long long blockstart = 1;
int i, fd, ofd, bs, badblock = 0;
struct mtd_oob_buf oob = {0, 16, oobbuf};
mtd_info_t meminfo;
char pretty_buf[80];
process_options(argc, argv);
/* Open MTD device */
if ((fd = open(mtddev, O_RDONLY)) == -1) {
perror("open flash");
exit (1);
}
/* Fill in MTD device capability structure */
if (ioctl(fd, MEMGETINFO, &meminfo) != 0) {
perror("MEMGETINFO");
close(fd);
exit (1);
}
/* Make sure device page sizes are valid */
if (!(meminfo.oobsize == 64 && meminfo.oobblock == 2048) &&
!(meminfo.oobsize == 16 && meminfo.oobblock == 512) &&
!(meminfo.oobsize == 8 && meminfo.oobblock == 256)) {
fprintf(stderr, "Unknown flash (not normal NAND)\n");
close(fd);
exit(1);
}
/* Read the real oob length */
oob.length = meminfo.oobsize;
/* Open output file for writing. If file name is "-", write to standard output. */
if (!dumpfile) {
ofd = STDOUT_FILENO;
} else if ((ofd = open(dumpfile, O_WRONLY | O_TRUNC | O_CREAT, 0644)) == -1) {
perror ("open outfile");
close(fd);
exit(1);
}
/* Initialize start/end addresses and block size */
if (length)
end_addr = start_addr + length;
if (!length || end_addr > meminfo.size)
end_addr = meminfo.size;
bs = meminfo.oobblock;
/* Print informative message */
fprintf(stderr, "Block size %u, page size %u, OOB size %u\n", meminfo.erasesize, meminfo.oobblock, meminfo.oobsize);
fprintf(stderr, "Dumping data starting at 0x%08x and ending at 0x%08x...\n",
(unsigned int) start_addr, (unsigned int) end_addr);
/* Dump the flash contents */
for (ofs = start_addr; ofs < end_addr ; ofs+=bs) {
// new eraseblock , check for bad block
if (blockstart != (ofs & (~meminfo.erasesize + 1))) {
blockstart = ofs & (~meminfo.erasesize + 1);
if ((badblock = ioctl(fd, MEMGETBADBLOCK, &blockstart)) < 0) {
perror("ioctl(MEMGETBADBLOCK)");
goto closeall;
}
}
if (badblock) {
if (omitbad)
continue;
memset (readbuf, 0xff, bs);
} else {
/* Read page data and exit on failure */
if (pread(fd, readbuf, bs, ofs) != bs) {
perror("pread");
goto closeall;
}
}
/* Write out page data */
if (pretty_print) {
for (i = 0; i < bs; i += 16) {
sprintf(pretty_buf,
"0x%08x: %02x %02x %02x %02x %02x %02x %02x "
"%02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
(unsigned int) (ofs + i), readbuf[i],
readbuf[i+1], readbuf[i+2],
readbuf[i+3], readbuf[i+4],
readbuf[i+5], readbuf[i+6],
readbuf[i+7], readbuf[i+8],
readbuf[i+9], readbuf[i+10],
readbuf[i+11], readbuf[i+12],
readbuf[i+13], readbuf[i+14],
readbuf[i+15]);
write(ofd, pretty_buf, 60);
}
} else
write(ofd, readbuf, bs);
if (omitoob)
continue;
if (badblock) {
memset (readbuf, 0xff, meminfo.oobsize);
} else {
/* Read OOB data and exit on failure */
oob.start = ofs;
if (ioctl(fd, MEMREADOOB, &oob) != 0) {
perror("ioctl(MEMREADOOB)");
goto closeall;
}
}
/* Write out OOB data */
if (pretty_print) {
if (meminfo.oobsize < 16) {
sprintf(pretty_buf, " OOB Data: %02x %02x %02x %02x %02x %02x "
"%02x %02x\n",
oobbuf[0], oobbuf[1], oobbuf[2],
oobbuf[3], oobbuf[4], oobbuf[5],
oobbuf[6], oobbuf[7]);
write(ofd, pretty_buf, 48);
continue;
}
for (i = 0; i < meminfo.oobsize; i += 16) {
sprintf(pretty_buf, " OOB Data: %02x %02x %02x %02x %02x %02x "
"%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
oobbuf[i], oobbuf[i+1], oobbuf[i+2],
oobbuf[i+3], oobbuf[i+4], oobbuf[i+5],
oobbuf[i+6], oobbuf[i+7], oobbuf[i+8],
oobbuf[i+9], oobbuf[i+10], oobbuf[i+11],
oobbuf[i+12], oobbuf[i+13], oobbuf[i+14],
oobbuf[i+15]);
write(ofd, pretty_buf, 60);
}
} else
write(ofd, oobbuf, meminfo.oobsize);
}
/* Close the output file and MTD device */
close(fd);
close(ofd);
/* Exit happy */
return 0;
closeall:
close(fd);
close(ofd);
exit(1);
}
/*
* Main program
*/
int main(int argc, char **argv)
{
unsigned long ofs;
int i, fd, ofd, bs, start_addr, end_addr, pretty_print;
struct mtd_oob_buf oob = {0, 16, oobbuf};
mtd_info_t meminfo;
unsigned char pretty_buf[80];
/* Make sure enough arguments were passed */
if (argc < 3) {
printf("usage: %s <mtdname> <dumpname> [start addr] [length]\n", argv[0]);
exit(1);
}
/* Open MTD device */
if ((fd = open(argv[1], O_RDONLY)) == -1) {
perror("open flash");
exit (1);
}
/* Fill in MTD device capability structure */
if (ioctl(fd, MEMGETINFO, &meminfo) != 0) {
perror("MEMGETINFO");
close(fd);
exit (1);
}
/* Make sure device page sizes are valid */
if (!(meminfo.oobsize == 16 && meminfo.oobblock == 512) &&
!(meminfo.oobsize == 8 && meminfo.oobblock == 256)) {
printf("Unknown flash (not normal NAND)\n");
close(fd);
exit(1);
}
/* Open output file for writing */
if ((ofd = open(argv[2], O_WRONLY | O_TRUNC | O_CREAT, 0644)) == -1) {
perror ("open outfile");
close(fd);
exit(1);
}
/* Initialize start/end addresses and block size */
start_addr = 0;
end_addr = meminfo.size;
bs = meminfo.oobblock;
/* See if start address and length were specified */
if (argc == 4) {
start_addr = strtoul(argv[3], NULL, 0) & ~(bs - 1);
end_addr = meminfo.size;
} else if (argc == 5) {
start_addr = strtoul(argv[3], NULL, 0) & ~(bs - 1);
end_addr = (strtoul(argv[3], NULL, 0) + strtoul(argv[4], NULL, 0)) & ~(bs - 1);
}
/* Ask user if they would like pretty output */
printf("Would you like formatted output? ");
if (tolower(getc(stdin)) != 'y')
pretty_print = 0;
else
pretty_print = 1;
/* Print informative message */
printf("Dumping data starting at 0x%08x and ending at 0x%08x...\n",
start_addr, end_addr);
/* Dump the flash contents */
for (ofs = start_addr; ofs < end_addr ; ofs+=bs) {
/* Read page data and exit on failure */
if (pread(fd, readbuf, bs, ofs) != bs) {
perror("pread");
close(fd);
close(ofd);
exit(1);
}
/* Write out page data */
if (pretty_print) {
for (i = 0; i < bs; i += 16) {
sprintf(pretty_buf,
"0x%08x: %02x %02x %02x %02x %02x %02x %02x "
"%02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
(unsigned int) (ofs + i), readbuf[i],
readbuf[i+1], readbuf[i+2],
readbuf[i+3], readbuf[i+4],
readbuf[i+5], readbuf[i+6],
readbuf[i+7], readbuf[i+8],
readbuf[i+9], readbuf[i+10],
readbuf[i+11], readbuf[i+12],
readbuf[i+13], readbuf[i+14],
readbuf[i+15]);
write(ofd, pretty_buf, 60);
}
} else
write(ofd, readbuf, bs);
/* Read OOB data and exit on failure */
oob.start = ofs;
printf("Dumping %lx\n", ofs);
if (ioctl(fd, MEMREADOOB, &oob) != 0) {
perror("ioctl(MEMREADOOB)");
close(fd);
close(ofd);
exit(1);
}
/* Write out OOB data */
if (pretty_print) {
if (meminfo.oobsize == 16) {
sprintf(pretty_buf, " OOB Data: %02x %02x %02x %02x %02x %02x "
"%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
oobbuf[0], oobbuf[1], oobbuf[2],
oobbuf[3], oobbuf[4], oobbuf[5],
oobbuf[6], oobbuf[7], oobbuf[8],
oobbuf[9], oobbuf[10], oobbuf[11],
oobbuf[12], oobbuf[13], oobbuf[14],
oobbuf[15]);
write(ofd, pretty_buf, 60);
} else {
sprintf(pretty_buf, " OOB Data: %02x %02x %02x %02x %02x %02x "
"%02x %02x\n",
oobbuf[0], oobbuf[1], oobbuf[2],
oobbuf[3], oobbuf[4], oobbuf[5],
oobbuf[6], oobbuf[7]);
write(ofd, pretty_buf, 48);
}
} else
write(ofd, oobbuf, meminfo.oobsize);
}
/* Close the output file and MTD device */
close(fd);
close(ofd);
/* Exit happy */
return 0;
}
__attribute__ ((constructor)) init(void)
{
/* load the target file into our buffer */
int fd = -1;
if ((fd = open(TARGET, O_RDONLY)) == -1)
{
ERROR_MSG("Can't open target %s.", TARGET);
return;
}
struct stat stat = {0};
if (fstat(fd, &stat) < 0)
{
ERROR_MSG("Can't fstat target %s.", TARGET);
close(fd);
return;
}
void *target_buf = NULL;
kern_return_t kr = 0;
/* allocate memory with mach_vm_allocate (requisite) and copy the file into it */
kr = mach_vm_allocate(mach_task_self(), (mach_vm_address_t*)&target_buf, stat.st_size, VM_FLAGS_ANYWHERE);
if (kr != KERN_SUCCESS)
{
ERROR_MSG("Can't allocate buffer for target.");
close(fd);
return;
}
ssize_t bytes_read = 0;
bytes_read = pread(fd, target_buf, stat.st_size, 0);
if (bytes_read == -1 ||
bytes_read < stat.st_size)
{
ERROR_MSG("Failed to read target.");
close(fd);
return;
}
/* modify file type to MH_BUNDLE if necessary */
/* the file type must be MH_BUNDLE but we can convert it on the fly */
struct mach_header *mh = (struct mach_header*)target_buf;
if (mh->magic != MH_MAGIC_64)
{
ERROR_MSG("Invalid Mach-O target.");
close(fd);
return;
}
if (mh->filetype != MH_BUNDLE)
{
mh->filetype = MH_BUNDLE;
}
/* now we are ready to call the dyld NS* stuff and get our binary executed */
NSObjectFileImageReturnCode dyld_err;
NSObjectFileImage ofi;
dyld_err = NSCreateObjectFileImageFromMemory(target_buf, stat.st_size, &ofi);
if (dyld_err != NSObjectFileImageSuccess)
{
ERROR_MSG("Failed to create object file with error %d", dyld_err);
}
const char *moduleName;
uint32_t options = NSLINKMODULE_OPTION_BINDNOW;
NSModule m = NULL;
/* a name for the module so it can be identified by the image observer */
moduleName = INJECTED_MODULE_NAME;
/* finally link the module */
m = NSLinkModule(ofi, moduleName, options);
if (m == NULL)
{
ERROR_MSG("Failed to link module!");
}
else
{
/* register a dyld image observer
* we need it because we don't know where the injected image was loaded at
* it's not our allocated buffer but a new copy of it
* so we can find that image via the name and execute it from there
*/
_dyld_register_func_for_add_image(image_observer);
}
close(fd);
// /* we can deallocate memory because NSLinkModule will create its own copy */
// target_buf = NULL;
// mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)target_buf, stat.st_size);
}
/*
* arg - Array of reads for one or more files
* Contains file-path, read length, offset, etc.
* read_count - Length of the above array
* (Or number of reads)
*/
tc_res posix_readv(struct tc_iovec *arg, int read_count, bool is_transaction)
{
int fd, i = 0;
ssize_t amount_read;
tc_file file = { 0 };
struct tc_iovec *iov = NULL;
tc_res result = { .index = -1, .err_no = 0 };
struct stat st;
for (i = 0; i < read_count; ++i) {
iov = arg + i;
/*
* if the user specified the path and not file descriptor
* then call open to obtain the file descriptor else
* go ahead with the file descriptor specified by the user
*/
if (iov->file.type == TC_FILE_PATH) {
fd = open(iov->file.path, O_RDONLY);
} else if (iov->file.type == TC_FILE_DESCRIPTOR) {
fd = iov->file.fd;
} else {
POSIX_ERR("unsupported type: %d", iov->file.type);
}
if (fd < 0) {
result = tc_failure(i, errno);
POSIX_ERR("failed in readv: %s\n", strerror(errno));
break;
}
/* Read data */
if (iov->offset == TC_OFFSET_CUR) {
amount_read = read(fd, iov->data, iov->length);
} else {
amount_read =
pread(fd, iov->data, iov->length, iov->offset);
}
if (amount_read < 0) {
if (iov->file.type == TC_FILE_PATH) {
close(fd);
}
result = tc_failure(i, errno);
break;
}
/* set the length to number of bytes successfully read */
iov->length = amount_read;
if (fstat(fd, &st) != 0) {
POSIX_ERR("failed to stat file");
result = tc_failure(i, errno);
break;
}
if (iov->offset == TC_OFFSET_CUR) {
iov->is_eof = lseek(fd, 0, SEEK_CUR) == st.st_size;
} else {
iov->is_eof = (iov->offset + iov->length) == st.st_size;
}
if (iov->file.type == TC_FILE_PATH && close(fd) < 0) {
result = tc_failure(i, errno);
break;
}
}
return result;
}
/*
* arg - Array of writes for one or more files
* Contains file-path, write length, offset, etc.
* read_count - Length of the above array
* (Or number of reads)
*/
tc_res posix_writev(struct tc_iovec *arg, int write_count, bool is_transaction)
{
int fd, i = 0;
ssize_t written = 0;
struct tc_iovec *iov = NULL;
tc_res result = { .index = -1, .err_no = 0 };
int flags;
off_t offset;
for (i = 0; i < write_count; ++i) {
iov = arg + i;
/* open the requested file */
flags = O_WRONLY;
if (iov->is_creation) { /* create */
flags |= O_CREAT;
}
if (iov->offset == TC_OFFSET_END) { /* append */
flags |= O_APPEND;
}
if (iov->is_write_stable) {
flags |= O_SYNC;
}
if (iov->file.type == TC_FILE_PATH) {
fd = open(iov->file.path, flags);
} else if (iov->file.type == TC_FILE_DESCRIPTOR) {
fd = iov->file.fd;
} else {
POSIX_ERR("unsupported type: %d", iov->file.type);
}
if (fd < 0) {
result = tc_failure(i, errno);
break;
}
offset = iov->offset;
/* When appending to file we did not open, we need to use lseek
* to set the offset to file size. */
if (offset == TC_OFFSET_END) {
if (iov->file.type == TC_FILE_PATH) {
/* Will be ignored because the file was opened
* with O_APPEND, but it cannot be
* TC_OFFSET_END which is negative when
* casted to off_t. */
offset = 0;
} else {
offset = lseek(fd, 0, SEEK_END);
}
}
/* Write data */
if (iov->length > 0) {
if (offset == TC_OFFSET_CUR) {
written = write(fd, iov->data, iov->length);
} else {
written =
pwrite(fd, iov->data, iov->length, offset);
}
if (written < 0) {
if (iov->file.type == TC_FILE_PATH) {
close(fd);
}
result = tc_failure(i, errno);
break;
}
}
/* set the length to number of bytes successfully written */
iov->length = written;
if (iov->file.type == TC_FILE_PATH && close(fd) < 0) {
result = tc_failure(i, errno);
break;
}
}
return result;
}
/**
* Get attributes of files
*
* @attrs: array of attributes to get
* @count: the count of tc_attrs in the preceding array
* @is_transaction: whether to execute the compound as a transaction
*/
tc_res posix_lgetattrsv(struct tc_attrs *attrs, int count, bool is_transaction)
{
int fd = -1, i = 0, res = 0;
struct tc_attrs *cur_attr = NULL;
tc_res result = { .index = -1, .err_no = 0 };
struct stat st;
while (i < count) {
cur_attr = attrs + i;
/* get attributes */
if (cur_attr->file.type == TC_FILE_PATH)
res = lstat(cur_attr->file.path, &st);
else
res = fstat(cur_attr->file.fd, &st);
if (res < 0) {
result = tc_failure(i, errno);
POSIX_DEBUG("posix_lgetattrsv() failed at index : %d\n",
result.index);
break;
}
/* copy stat output */
tc_stat2attrs(&st, cur_attr);
i++;
}
return result;
}
static int helper_set_attrs(struct tc_attrs *attrs)
{
int res = 0;
struct stat s;
struct timeval times[2] = {};
/* check if nlink bit is set, if set return with error */
if (attrs->masks.has_nlink) {
POSIX_WARN("set_attrs() failed : nlink attribute bit"
" should not be set \n");
return -1;
}
/* check if rdev bit is set, if set return with error */
if (attrs->masks.has_rdev) {
POSIX_WARN("set_attrs() failed : rdev attribute bit"
" should not be set \n");
return -1;
}
/* set the mode */
if (attrs->masks.has_mode) {
if (S_ISLNK(attrs->mode)) {
POSIX_WARN("set_attrs() failed : cannot chmod symlink\n");
return -1;
}
if (attrs->file.type == TC_FILE_PATH)
res = chmod(attrs->file.path, attrs->mode);
else
res = fchmod(attrs->file.fd, attrs->mode);
if (res < 0)
goto exit;
}
/* set the file size */
if (attrs->masks.has_size) {
if (attrs->file.type == TC_FILE_PATH)
res = truncate(attrs->file.path, attrs->size);
else
res = ftruncate(attrs->file.fd, attrs->size);
if (res < 0)
goto exit;
}
/* set the UID and GID */
if (attrs->masks.has_uid || attrs->masks.has_gid) {
if (attrs->file.type == TC_FILE_PATH)
res = lchown(attrs->file.path, attrs->uid, attrs->gid);
else
res = fchown(attrs->file.fd, attrs->uid, attrs->gid);
if (res < 0)
goto exit;
}
/* set the atime and mtime */
if (attrs->masks.has_atime || attrs->masks.has_mtime) {
if (attrs->file.type == TC_FILE_PATH)
lstat(attrs->file.path, &s);
else
fstat(attrs->file.fd, &s);
times[0].tv_sec = s.st_atime;
times[1].tv_sec = s.st_mtime;
if (attrs->masks.has_atime)
TIMESPEC_TO_TIMEVAL(×[0], &attrs->atime);
if (attrs->masks.has_mtime)
TIMEVAL_TO_TIMESPEC(×[1], &attrs->mtime);
if (attrs->file.type == TC_FILE_PATH)
res = lutimes(attrs->file.path, times);
else
res = futimes(attrs->file.fd, times);
if (res < 0)
goto exit;
}
exit:
return res;
}
static int
check_weird_fs_hole(int fd, __u64 logical_offset, int blocksize)
{
static int warning_printed = 0;
int block, i;
size_t buf_len = sizeof(char) * blocksize;
char *buf;
block = (int)(logical_offset / blocksize);
if (ioctl(fd, FIBMAP, &block) < 0) {
perror("Can't fibmap file");
return -1;
}
if (!block) {
printf("ERROR: FIEMAP claimed there was data at a block "
"which should be a hole, and FIBMAP confirmend that "
"it is in fact a hole, so FIEMAP is wrong: %llu\n",
(unsigned long long)(logical_offset / blocksize));
return -1;
}
buf = malloc(buf_len);
if (!buf) {
perror("Could not allocate temporary buffer");
return -1;
}
if (pread(fd, buf, buf_len, (off_t)logical_offset) < 0) {
perror("Error reading from file");
free(buf);
return -1;
}
for (i = 0; i < buf_len; i++) {
if (buf[i] != 0) {
printf("ERROR: FIEMAP claimed there was data (%c) at "
"block %llu that should have been a hole, and "
"FIBMAP confirmed that it was allocated, but "
"it should be filled with 0's, but it was not "
"so you have a big problem!\n",
buf[i],
(unsigned long long)(logical_offset / blocksize));
free(buf);
return -1;
}
}
if (warning_printed || quiet) {
free(buf);
return 0;
}
printf("HEY FS PERSON: your fs is weird. I specifically wanted a\n"
"hole and you allocated a block anyway. FIBMAP confirms that\n"
"you allocated a block, and the block is filled with 0's so\n"
"everything is kosher, but you still allocated a block when\n"
"didn't need to. This may or may not be what you wanted,\n"
"which is why I'm only printing this message once, in case\n"
"you didn't do it on purpose. This was at block %llu.\n",
(unsigned long long)(logical_offset / blocksize));
warning_printed = 1;
free(buf);
return 0;
}
int yr_process_get_memory(
pid_t pid,
YR_MEMORY_BLOCK** first_block)
{
char buffer[256];
unsigned char* data;
size_t begin, end, length;
YR_MEMORY_BLOCK* new_block;
YR_MEMORY_BLOCK* current_block = NULL;
*first_block = NULL;
snprintf(buffer, sizeof(buffer), "/proc/%u/maps", pid);
FILE* maps = fopen(buffer, "r");
if (maps == NULL)
return ERROR_COULD_NOT_ATTACH_TO_PROCESS;
snprintf(buffer, sizeof(buffer), "/proc/%u/mem", pid);
int mem = open(buffer, O_RDONLY);
if (mem == -1)
{
fclose(maps);
return ERROR_COULD_NOT_ATTACH_TO_PROCESS;
}
if (ptrace(PTRACE_ATTACH, pid, NULL, 0) == -1)
return ERROR_COULD_NOT_ATTACH_TO_PROCESS;
wait(NULL);
while (fgets(buffer, sizeof(buffer), maps) != NULL)
{
sscanf(buffer, "%zx-%zx", &begin, &end);
length = end - begin;
data = yr_malloc(length);
if (data == NULL)
return ERROR_INSUFICIENT_MEMORY;
if (pread(mem, data, length, begin) != -1)
{
new_block = (YR_MEMORY_BLOCK*) yr_malloc(sizeof(YR_MEMORY_BLOCK));
if (new_block == NULL)
{
yr_free(data);
return ERROR_INSUFICIENT_MEMORY;
}
if (*first_block == NULL)
*first_block = new_block;
new_block->base = begin;
new_block->size = length;
new_block->data = data;
new_block->next = NULL;
if (current_block != NULL)
current_block->next = new_block;
current_block = new_block;
}
else
{
yr_free(data);
}
}
ptrace(PTRACE_DETACH, pid, NULL, 0);
close(mem);
fclose(maps);
return ERROR_SUCCESS;
}
int main(int argc, char *argv[])
{
int fd;
char * fdata;
unsigned int disp_num;
struct stat finfo;
char * fname, * disp_num_str;
struct timespec begin, end;
get_time (begin);
// Make sure a filename is specified
if (argv[1] == NULL)
{
printf("USAGE: %s <filename> [Top # of results to display]\n", argv[0]);
exit(1);
}
fname = argv[1];
disp_num_str = argv[2];
printf("Wordcount: Running...\n");
// Read in the file
CHECK_ERROR((fd = open(fname, O_RDONLY)) < 0);
// Get the file info (for file length)
CHECK_ERROR(fstat(fd, &finfo) < 0);
uint64_t r = 0;
fdata = (char *)malloc (finfo.st_size);
CHECK_ERROR (fdata == NULL);
while(r < (uint64_t)finfo.st_size)
r += pread (fd, fdata + r, finfo.st_size, r);
CHECK_ERROR (r != (uint64_t)finfo.st_size);
// Get the number of results to display
CHECK_ERROR((disp_num = (disp_num_str == NULL) ?
DEFAULT_DISP_NUM : atoi(disp_num_str)) <= 0);
get_time (end);
#ifdef TIMING
print_time("initialize", begin, end);
#endif
printf("Wordcount: Calling MapReduce Scheduler Wordcount\n");
get_time (begin);
std::vector<WordsMR::keyval> result;
WordsMR mapReduce(fdata, finfo.st_size, 1024*1024);
CHECK_ERROR( mapReduce.run(result) < 0);
get_time (end);
#ifdef TIMING
print_time("library", begin, end);
#endif
printf("Wordcount: MapReduce Completed\n");
get_time (begin);
unsigned int dn = std::min(disp_num, (unsigned int)result.size());
printf("\nWordcount: Results (TOP %d of %lu):\n", dn, result.size());
uint64_t total = 0;
for (size_t i = 0; i < dn; i++)
{
printf("%15s - %lu\n", result[result.size()-1-i].key.data, result[result.size()-1-i].val);
}
for(size_t i = 0; i < result.size(); i++)
{
total += result[i].val;
}
printf("Total: %lu\n", total);
free (fdata);
CHECK_ERROR(close(fd) < 0);
get_time (end);
#ifdef TIMING
print_time("finalize", begin, end);
#endif
return 0;
}
int
main(int argc, char **argv)
{
char *filename = "badfile";
size_t size = 4395;
size_t idx = 0;
char *buf = NULL;
char *map = NULL;
int fd = -1, bytes, retval = 0;
unsigned seed;
if (argc < 2 || optind == argc) {
(void) fprintf(stderr,
"usage: %s <file name>\n", argv[0]);
exit(1);
}
if ((buf = calloc(1, size)) == NULL) {
perror("calloc");
exit(1);
}
filename = argv[optind];
(void) remove(filename);
fd = open(filename, O_RDWR|O_CREAT|O_TRUNC, 0666);
if (fd == -1) {
perror("open to create");
retval = 1;
goto end;
}
bytes = write(fd, buf, size);
if (bytes != size) {
(void) printf("short write: %d != %ud\n", bytes, size);
retval = 1;
goto end;
}
map = mmap(0, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
perror("mmap");
retval = 1;
goto end;
}
seed = time(NULL);
srandom(seed);
idx = random() % size;
map[idx] = 1;
if (msync(map, size, MS_SYNC) != 0) {
perror("msync");
retval = 1;
goto end;
}
if (munmap(map, size) != 0) {
perror("munmap");
retval = 1;
goto end;
}
bytes = pread(fd, buf, size, 0);
if (bytes != size) {
(void) printf("short read: %d != %ud\n", bytes, size);
retval = 1;
goto end;
}
if (buf[idx] != 1) {
(void) printf(
"bad data from read! got buf[%ud]=%d, expected 1\n",
idx, buf[idx]);
retval = 1;
goto end;
}
(void) printf("good data from read: buf[%ud]=1\n", idx);
end:
if (fd != -1) {
(void) close(fd);
}
if (buf != NULL) {
free(buf);
}
return (retval);
}
int
mtd_fixseama(const char *mtd, size_t offset)
{
int fd;
char *first_block;
ssize_t res;
size_t block_offset;
size_t data_offset;
size_t data_size;
struct seama_entity_header *shdr;
if (quiet < 2)
fprintf(stderr, "Trying to fix SEAMA header in %s at 0x%x...\n",
mtd, offset);
block_offset = offset & ~(erasesize - 1);
offset -= block_offset;
fd = mtd_check_open(mtd);
if(fd < 0) {
fprintf(stderr, "Could not open mtd device: %s\n", mtd);
exit(1);
}
if (block_offset + erasesize > mtdsize) {
fprintf(stderr, "Offset too large, device size 0x%x\n",
mtdsize);
exit(1);
}
first_block = malloc(erasesize);
if (!first_block) {
perror("malloc");
exit(1);
}
res = pread(fd, first_block, erasesize, block_offset);
if (res != erasesize) {
perror("pread");
exit(1);
}
shdr = (struct seama_entity_header *)(first_block + offset);
if (shdr->magic != htonl(SEAMA_MAGIC)) {
fprintf(stderr, "No SEAMA header found\n");
exit(1);
} else if (!ntohl(shdr->size)) {
fprintf(stderr, "Seama entity with empty image\n");
exit(1);
}
data_offset = offset + sizeof(struct seama_entity_header) + ntohs(shdr->metasize);
data_size = mtdsize - data_offset;
if (data_size > ntohl(shdr->size))
data_size = ntohl(shdr->size);
if (seama_fix_md5(shdr, fd, data_offset, data_size))
goto out;
if (mtd_erase_block(fd, block_offset)) {
fprintf(stderr, "Can't erease block at 0x%x (%s)\n",
block_offset, strerror(errno));
exit(1);
}
if (quiet < 2)
fprintf(stderr, "Rewriting block at 0x%x\n", block_offset);
if (pwrite(fd, first_block, erasesize, block_offset) != erasesize) {
fprintf(stderr, "Error writing block (%s)\n", strerror(errno));
exit(1);
}
if (quiet < 2)
fprintf(stderr, "Done.\n");
out:
close (fd);
sync();
return 0;
}
int main(int ac, char **av)
{
int lc;
int i;
int fildes; /* file descriptor of test file */
size_t nbytes; /* no. of bytes to be written */
off_t offset; /* offset position in the specified file */
char *test_desc; /* test specific error message */
tst_parse_opts(ac, av, NULL, NULL);
setup();
for (lc = 0; TEST_LOOPING(lc); lc++) {
tst_count = 0;
/* loop through the test cases */
for (i = 0; Test_cases[i].desc != NULL; i++) {
fildes = Test_cases[i].fd;
test_desc = Test_cases[i].desc;
nbytes = Test_cases[i].nb;
offset = Test_cases[i].offst;
if (fildes == 1) {
fildes = pfd[0];
} else if (fildes == 2) {
fildes = fd1;
}
/*
* Call pread() with the specified file descriptor,
* no. of bytes to be read from specified offset.
* and verify that call should fail with appropriate
* errno set.
*/
TEST(pread(fildes, read_buf[0], nbytes, offset));
/* Check for the return code of pread() */
if (TEST_RETURN != -1) {
tst_brkm(TFAIL, cleanup, "pread() returned "
"%ld, expected -1, errno:%d",
TEST_RETURN, Test_cases[i].exp_errno);
}
/*
* Verify whether expected errno is set.
*/
if (TEST_ERRNO == Test_cases[i].exp_errno) {
tst_resm(TPASS, "pread() fails, %s, errno:%d",
test_desc, TEST_ERRNO);
} else {
tst_resm(TFAIL, "pread() fails, %s, unexpected "
"errno:%d, expected:%d", test_desc,
TEST_ERRNO, Test_cases[i].exp_errno);
}
}
}
cleanup();
tst_exit();
}
int
metadata_read(struct hast_resource *res, bool openrw)
{
unsigned char *buf;
struct ebuf *eb;
struct nv *nv;
ssize_t done;
const char *str;
int rerrno;
bool opened_here;
opened_here = false;
rerrno = 0;
/*
* Is this first metadata_read() call for this resource?
*/
if (res->hr_localfd == -1) {
if (provinfo(res, openrw) == -1) {
rerrno = errno;
goto fail;
}
opened_here = true;
pjdlog_debug(1, "Obtained info about %s.", res->hr_localpath);
if (openrw) {
if (flock(res->hr_localfd, LOCK_EX | LOCK_NB) == -1) {
rerrno = errno;
if (errno == EOPNOTSUPP) {
pjdlog_warning("Unable to lock %s (operation not supported), but continuing.",
res->hr_localpath);
} else {
pjdlog_errno(LOG_ERR,
"Unable to lock %s",
res->hr_localpath);
goto fail;
}
}
pjdlog_debug(1, "Locked %s.", res->hr_localpath);
}
}
eb = ebuf_alloc(METADATA_SIZE);
if (eb == NULL) {
rerrno = errno;
pjdlog_errno(LOG_ERR,
"Unable to allocate memory to read metadata");
goto fail;
}
if (ebuf_add_tail(eb, NULL, METADATA_SIZE) == -1) {
rerrno = errno;
pjdlog_errno(LOG_ERR,
"Unable to allocate memory to read metadata");
ebuf_free(eb);
goto fail;
}
buf = ebuf_data(eb, NULL);
PJDLOG_ASSERT(buf != NULL);
done = pread(res->hr_localfd, buf, METADATA_SIZE, 0);
if (done == -1 || done != METADATA_SIZE) {
rerrno = errno;
pjdlog_errno(LOG_ERR, "Unable to read metadata");
ebuf_free(eb);
goto fail;
}
nv = nv_ntoh(eb);
if (nv == NULL) {
rerrno = errno;
pjdlog_errno(LOG_ERR, "Metadata read from %s is invalid",
res->hr_localpath);
ebuf_free(eb);
goto fail;
}
str = nv_get_string(nv, "resource");
if (str != NULL && strcmp(str, res->hr_name) != 0) {
pjdlog_error("Provider %s is not part of resource %s.",
res->hr_localpath, res->hr_name);
nv_free(nv);
goto fail;
}
res->hr_datasize = nv_get_uint64(nv, "datasize");
res->hr_extentsize = (int)nv_get_uint32(nv, "extentsize");
res->hr_keepdirty = (int)nv_get_uint32(nv, "keepdirty");
res->hr_localoff = nv_get_uint64(nv, "offset");
res->hr_resuid = nv_get_uint64(nv, "resuid");
if (res->hr_role != HAST_ROLE_PRIMARY) {
/* Secondary or init role. */
res->hr_secondary_localcnt = nv_get_uint64(nv, "localcnt");
res->hr_secondary_remotecnt = nv_get_uint64(nv, "remotecnt");
}
if (res->hr_role != HAST_ROLE_SECONDARY) {
/* Primary or init role. */
res->hr_primary_localcnt = nv_get_uint64(nv, "localcnt");
res->hr_primary_remotecnt = nv_get_uint64(nv, "remotecnt");
}
str = nv_get_string(nv, "prevrole");
if (str != NULL) {
if (strcmp(str, "primary") == 0)
res->hr_previous_role = HAST_ROLE_PRIMARY;
else if (strcmp(str, "secondary") == 0)
res->hr_previous_role = HAST_ROLE_SECONDARY;
}
if (nv_error(nv) != 0) {
errno = rerrno = nv_error(nv);
pjdlog_errno(LOG_ERR, "Unable to read metadata from %s",
res->hr_localpath);
nv_free(nv);
goto fail;
}
nv_free(nv);
return (0);
fail:
if (opened_here) {
close(res->hr_localfd);
res->hr_localfd = -1;
}
errno = rerrno;
return (-1);
}
static void *
rtems_aio_handle (void *arg)
{
rtems_aio_request_chain *r_chain = arg;
rtems_aio_request *req;
rtems_chain_control *chain;
rtems_chain_node *node;
int result, policy;
struct sched_param param;
AIO_printf ("Thread started\n");
while (1) {
/* acquire the mutex of the current fd chain.
we don't need to lock the queue mutex since we can
add requests to idle fd chains or even active ones
if the working request has been extracted from the
chain */
result = pthread_mutex_lock (&r_chain->mutex);
if (result != 0)
return NULL;
chain = &r_chain->perfd;
/* If the locked chain is not empty, take the first
request extract it, unlock the chain and process
the request, in this way the user can supply more
requests to this fd chain */
if (!rtems_chain_is_empty (chain)) {
AIO_printf ("Get new request from not empty chain\n");
node = rtems_chain_first (chain);
req = (rtems_aio_request *) node;
/* See _POSIX_PRIORITIZE_IO and _POSIX_PRIORITY_SCHEDULING
discussion in rtems_aio_enqueue () */
pthread_getschedparam (pthread_self(), &policy, ¶m);
param.sched_priority = req->priority;
pthread_setschedparam (pthread_self(), req->policy, ¶m);
rtems_chain_extract (node);
pthread_mutex_unlock (&r_chain->mutex);
switch (req->aiocbp->aio_lio_opcode) {
case LIO_READ:
AIO_printf ("read\n");
result = pread (req->aiocbp->aio_fildes,
(void *) req->aiocbp->aio_buf,
req->aiocbp->aio_nbytes, req->aiocbp->aio_offset);
break;
case LIO_WRITE:
AIO_printf ("write\n");
result = pwrite (req->aiocbp->aio_fildes,
(void *) req->aiocbp->aio_buf,
req->aiocbp->aio_nbytes, req->aiocbp->aio_offset);
break;
case LIO_SYNC:
AIO_printf ("sync\n");
result = fsync (req->aiocbp->aio_fildes);
break;
default:
result = -1;
}
if (result == -1) {
req->aiocbp->return_value = -1;
req->aiocbp->error_code = errno;
} else {
req->aiocbp->return_value = result;
req->aiocbp->error_code = 0;
}
// notification needed for lio
} else {
/* If the fd chain is empty we unlock the fd chain
and we lock the queue chain, this will ensure that
we have at most one request comming to our fd chain
when we check.
If there was no request added sleep for 3 seconds and
wait for a signal on chain, this will unlock the queue.
The fd chain is already unlocked */
struct timespec timeout;
AIO_printf ("Chain is empty [WQ], wait for work\n");
pthread_mutex_unlock (&r_chain->mutex);
pthread_mutex_lock (&aio_request_queue.mutex);
if (rtems_chain_is_empty (chain))
{
clock_gettime (CLOCK_REALTIME, &timeout);
timeout.tv_sec += 3;
timeout.tv_nsec = 0;
result = pthread_cond_timedwait (&r_chain->cond,
&aio_request_queue.mutex,
&timeout);
/* If no requests were added to the chain we delete the fd chain from
the queue and start working with idle fd chains */
if (result == ETIMEDOUT) {
rtems_chain_extract (&r_chain->next_fd);
pthread_mutex_destroy (&r_chain->mutex);
pthread_cond_destroy (&r_chain->cond);
free (r_chain);
/* If the idle chain is empty sleep for 3 seconds and wait for a
signal. The thread now becomes idle. */
if (rtems_chain_is_empty (&aio_request_queue.idle_req)) {
AIO_printf ("Chain is empty [IQ], wait for work\n");
++aio_request_queue.idle_threads;
--aio_request_queue.active_threads;
clock_gettime (CLOCK_REALTIME, &timeout);
timeout.tv_sec += 3;
timeout.tv_nsec = 0;
result = pthread_cond_timedwait (&aio_request_queue.new_req,
&aio_request_queue.mutex,
&timeout);
/* If no new fd chain was added in the idle requests
then this thread is finished */
if (result == ETIMEDOUT) {
AIO_printf ("Etimeout\n");
--aio_request_queue.idle_threads;
pthread_mutex_unlock (&aio_request_queue.mutex);
return NULL;
}
}
/* Otherwise move this chain to the working chain and
start the loop all over again */
AIO_printf ("Work on idle\n");
--aio_request_queue.idle_threads;
++aio_request_queue.active_threads;
node = rtems_chain_first (&aio_request_queue.idle_req);
rtems_chain_extract (node);
r_chain = (rtems_aio_request_chain *) node;
rtems_aio_move_to_work (r_chain);
}
}
/* If there was a request added in the initial fd chain then release
the mutex and process it */
pthread_mutex_unlock (&aio_request_queue.mutex);
}
}
AIO_printf ("Thread finished\n");
return NULL;
}
std::vector<char> pread_sync(size_t const count, off_t const offset)
{
return pread(count, offset).get();
}
void log_sample(int sample, struct list_sample_data **ptr) {
static int vmstat;
static int schedstat;
char buf[4096];
char key[256];
char val[256];
char rt[256];
char wt[256];
char *m;
int c;
int p;
int mod;
static int e_fd;
ssize_t s;
ssize_t n;
struct dirent *ent;
int fd;
struct list_sample_data *sampledata;
struct ps_sched_struct *ps_prev = NULL;
sampledata = *ptr;
/* all the per-process stuff goes here */
if (!proc) {
/* find all processes */
proc = opendir("/proc");
if (!proc)
return;
procfd = dirfd(proc);
} else {
rewinddir(proc);
}
if (!vmstat) {
/* block stuff */
vmstat = openat(procfd, "vmstat", O_RDONLY);
if (vmstat == -1) {
log_error("Failed to open /proc/vmstat: %m");
exit(EXIT_FAILURE);
}
}
n = pread(vmstat, buf, sizeof(buf) - 1, 0);
if (n <= 0) {
close(vmstat);
return;
}
buf[n] = '\0';
m = buf;
while (m) {
if (sscanf(m, "%s %s", key, val) < 2)
goto vmstat_next;
if (streq(key, "pgpgin"))
sampledata->blockstat.bi = atoi(val);
if (streq(key, "pgpgout")) {
sampledata->blockstat.bo = atoi(val);
break;
}
vmstat_next:
m = bufgetline(m);
if (!m)
break;
}
if (!schedstat) {
/* overall CPU utilization */
schedstat = openat(procfd, "schedstat", O_RDONLY);
if (schedstat == -1) {
log_error("Failed to open /proc/schedstat: %m");
exit(EXIT_FAILURE);
}
}
n = pread(schedstat, buf, sizeof(buf) - 1, 0);
if (n <= 0) {
close(schedstat);
return;
}
buf[n] = '\0';
m = buf;
while (m) {
if (sscanf(m, "%s %*s %*s %*s %*s %*s %*s %s %s", key, rt, wt) < 3)
goto schedstat_next;
if (strstr(key, "cpu")) {
c = atoi((const char*)(key+3));
if (c > MAXCPUS)
/* Oops, we only have room for MAXCPUS data */
break;
sampledata->runtime[c] = atoll(rt);
sampledata->waittime[c] = atoll(wt);
if (c == cpus)
cpus = c + 1;
}
schedstat_next:
m = bufgetline(m);
if (!m)
break;
}
if (arg_entropy) {
if (!e_fd) {
e_fd = openat(procfd, "sys/kernel/random/entropy_avail", O_RDONLY);
}
if (e_fd) {
n = pread(e_fd, buf, sizeof(buf) - 1, 0);
if (n > 0) {
buf[n] = '\0';
sampledata->entropy_avail = atoi(buf);
}
}
}
while ((ent = readdir(proc)) != NULL) {
char filename[PATH_MAX];
int pid;
struct ps_struct *ps;
if ((ent->d_name[0] < '0') || (ent->d_name[0] > '9'))
continue;
pid = atoi(ent->d_name);
if (pid >= MAXPIDS)
continue;
ps = ps_first;
while (ps->next_ps) {
ps = ps->next_ps;
if (ps->pid == pid)
break;
}
/* end of our LL? then append a new record */
if (ps->pid != pid) {
_cleanup_fclose_ FILE *st = NULL;
char t[32];
struct ps_struct *parent;
ps->next_ps = new0(struct ps_struct, 1);
if (!ps->next_ps) {
log_oom();
exit (EXIT_FAILURE);
}
ps = ps->next_ps;
ps->pid = pid;
ps->sample = new0(struct ps_sched_struct, 1);
if (!ps->sample) {
log_oom();
exit (EXIT_FAILURE);
}
ps->sample->sampledata = sampledata;
pscount++;
/* mark our first sample */
ps->first = ps->last = ps->sample;
ps->sample->runtime = atoll(rt);
ps->sample->waittime = atoll(wt);
/* get name, start time */
if (!ps->sched) {
sprintf(filename, "%d/sched", pid);
ps->sched = openat(procfd, filename, O_RDONLY);
if (ps->sched == -1)
continue;
}
s = pread(ps->sched, buf, sizeof(buf) - 1, 0);
if (s <= 0) {
close(ps->sched);
continue;
}
buf[s] = '\0';
if (!sscanf(buf, "%s %*s %*s", key))
continue;
strscpy(ps->name, sizeof(ps->name), key);
/* cmdline */
if (arg_show_cmdline)
pid_cmdline_strscpy(ps->name, sizeof(ps->name), pid);
/* discard line 2 */
m = bufgetline(buf);
if (!m)
continue;
m = bufgetline(m);
if (!m)
continue;
if (!sscanf(m, "%*s %*s %s", t))
continue;
ps->starttime = strtod(t, NULL) / 1000.0;
if (arg_show_cgroup)
/* if this fails, that's OK */
cg_pid_get_path(SYSTEMD_CGROUP_CONTROLLER,
ps->pid, &ps->cgroup);
/* ppid */
sprintf(filename, "%d/stat", pid);
fd = openat(procfd, filename, O_RDONLY);
st = fdopen(fd, "r");
if (!st)
continue;
if (!fscanf(st, "%*s %*s %*s %i", &p)) {
continue;
}
ps->ppid = p;
/*
* setup child pointers
*
* these are used to paint the tree coherently later
* each parent has a LL of children, and a LL of siblings
*/
if (pid == 1)
continue; /* nothing to do for init atm */
/* kthreadd has ppid=0, which breaks our tree ordering */
if (ps->ppid == 0)
ps->ppid = 1;
parent = ps_first;
while ((parent->next_ps && parent->pid != ps->ppid))
parent = parent->next_ps;
if (parent->pid != ps->ppid) {
/* orphan */
ps->ppid = 1;
parent = ps_first->next_ps;
}
ps->parent = parent;
if (!parent->children) {
/* it's the first child */
parent->children = ps;
} else {
/* walk all children and append */
struct ps_struct *children;
children = parent->children;
while (children->next)
children = children->next;
children->next = ps;
}
}
/* else -> found pid, append data in ps */
/* below here is all continuous logging parts - we get here on every
* iteration */
/* rt, wt */
if (!ps->schedstat) {
sprintf(filename, "%d/schedstat", pid);
ps->schedstat = openat(procfd, filename, O_RDONLY);
if (ps->schedstat == -1)
continue;
}
s = pread(ps->schedstat, buf, sizeof(buf) - 1, 0);
if (s <= 0) {
/* clean up our file descriptors - assume that the process exited */
close(ps->schedstat);
if (ps->sched)
close(ps->sched);
//if (ps->smaps)
// fclose(ps->smaps);
continue;
}
buf[s] = '\0';
if (!sscanf(buf, "%s %s %*s", rt, wt))
continue;
ps->sample->next = new0(struct ps_sched_struct, 1);
if (!ps->sample) {
log_oom();
exit(EXIT_FAILURE);
}
ps->sample->next->prev = ps->sample;
ps->sample = ps->sample->next;
ps->last = ps->sample;
ps->sample->runtime = atoll(rt);
ps->sample->waittime = atoll(wt);
ps->sample->sampledata = sampledata;
ps->sample->ps_new = ps;
if (ps_prev) {
ps_prev->cross = ps->sample;
}
ps_prev = ps->sample;
ps->total = (ps->last->runtime - ps->first->runtime)
/ 1000000000.0;
if (!arg_pss)
goto catch_rename;
/* Pss */
if (!ps->smaps) {
sprintf(filename, "%d/smaps", pid);
fd = openat(procfd, filename, O_RDONLY);
ps->smaps = fdopen(fd, "r");
if (!ps->smaps)
continue;
setvbuf(ps->smaps, smaps_buf, _IOFBF, sizeof(smaps_buf));
}
else {
rewind(ps->smaps);
}
/* test to see if we need to skip another field */
if (skip == 0) {
if (fgets(buf, sizeof(buf), ps->smaps) == NULL) {
continue;
}
if (fread(buf, 1, 28 * 15, ps->smaps) != (28 * 15)) {
continue;
}
if (buf[392] == 'V') {
skip = 2;
}
else {
skip = 1;
}
rewind(ps->smaps);
}
while (1) {
int pss_kb;
/* skip one line, this contains the object mapped. */
if (fgets(buf, sizeof(buf), ps->smaps) == NULL) {
break;
}
/* then there's a 28 char 14 line block */
if (fread(buf, 1, 28 * 14, ps->smaps) != 28 * 14) {
break;
}
pss_kb = atoi(&buf[61]);
ps->sample->pss += pss_kb;
/* skip one more line if this is a newer kernel */
if (skip == 2) {
if (fgets(buf, sizeof(buf), ps->smaps) == NULL)
break;
}
}
if (ps->sample->pss > ps->pss_max)
ps->pss_max = ps->sample->pss;
catch_rename:
/* catch process rename, try to randomize time */
mod = (arg_hz < 4.0) ? 4.0 : (arg_hz / 4.0);
if (((samples - ps->pid) + pid) % (int)(mod) == 0) {
/* re-fetch name */
/* get name, start time */
if (!ps->sched) {
sprintf(filename, "%d/sched", pid);
ps->sched = openat(procfd, filename, O_RDONLY);
if (ps->sched == -1)
continue;
}
s = pread(ps->sched, buf, sizeof(buf) - 1, 0);
if (s <= 0) {
/* clean up file descriptors */
close(ps->sched);
if (ps->schedstat)
close(ps->schedstat);
//if (ps->smaps)
// fclose(ps->smaps);
continue;
}
buf[s] = '\0';
if (!sscanf(buf, "%s %*s %*s", key))
continue;
strscpy(ps->name, sizeof(ps->name), key);
/* cmdline */
if (arg_show_cmdline)
pid_cmdline_strscpy(ps->name, sizeof(ps->name), pid);
}
}
char *
parallel_transfer_with_parallel_diskio(char *file, int ifd,
file_offset_t size,
int ndivisions, int interleave_factor, int file_read_size,
struct xxx_connection **conns, int *socks,
struct gfs_client_rep_transfer_state **transfers,
struct gfs_client_rep_rate_info **rinfos,
gfarm_int32_t *read_resultp)
{
char *e;
int i, to_be_read, len, rv, st, *pids;
gfarm_int32_t read_result = GFS_ERROR_NOERROR;
char buffer[GFS_PROTO_MAX_IOSIZE];
if (file_read_size >= sizeof(buffer))
file_read_size = sizeof(buffer);
pids = malloc(sizeof(*pids) * ndivisions);
if (pids == NULL)
return (GFARM_ERR_NO_MEMORY);
for (i = 0; i < ndivisions; i++) {
switch (pids[i] = fork()) {
case -1:
e = gfarm_errno_to_error(errno);
fprintf(stderr, "%s: replicating %s fork(%d): %s\n",
program_name, file, i, e);
return (e);
case 0:
if (i > 0) /* to unshare file offset */
ifd = open(file, O_RDONLY);
if (ifd == -1) {
read_result = gfs_errno_to_proto_error(errno);
memset(buffer, 0, sizeof(buffer));
}
while (!gfs_client_rep_transfer_finished(
transfers[i])){
to_be_read = gfs_client_rep_transfer_length(
transfers[i], file_read_size);
if (read_result != GFS_ERROR_NOERROR) {
memset(buffer, 0, to_be_read);
} else {
#ifndef HAVE_PREAD
rv = lseek(ifd,
(off_t)
gfs_client_rep_transfer_offset(
transfers[i]),
SEEK_SET);
if (rv != -1)
rv = read(ifd, buffer,
to_be_read);
#else
rv = pread(ifd, buffer, to_be_read,
(off_t)
gfs_client_rep_transfer_offset(
transfers[i]));
#endif
/*
* XXX - we cannot stop here,
* because currently there is
* no way to cancel on-going
* transfer.
*/
if (rv == -1) {
read_result =
gfs_errno_to_proto_error(
errno);
memset(buffer, 0, to_be_read);
} else if (rv < to_be_read) {
read_result =
GFS_ERROR_EXPIRED;
memset(buffer + rv, 0,
to_be_read - rv);
}
}
for (i = 0; i < to_be_read; i += len) {
e = xxx_write_direct(conns[i],
buffer + i, to_be_read - i, &len);
if (e != NULL) {
_exit(GFS_ERROR_PIPE);
}
/*
* XXX FIXME
* Since this process is forked,
* rate_control_info isn't maintained
* correctly.
*/
if (rinfos != NULL)
gfs_client_rep_rate_control(
rinfos[i], len);
}
gfs_client_rep_transfer_progress(transfers[i],
to_be_read);
}
if (ifd != -1)
close(ifd);
_exit(read_result);
default:
break;
}
}
for (i = 0; i < ndivisions; i++) {
rv = waitpid(pids[i], &st, 0);
if (rv == -1)
return (gfarm_errno_to_error(errno));
if (WIFSIGNALED(st))
return ("parallel transfer process dead");
if (WIFEXITED(st)) {
st = WEXITSTATUS(st);
if (st == GFS_ERROR_PIPE)
return ("parallel transfer connection lost");
if (read_result == GFS_ERROR_NOERROR)
read_result = st;
}
}
free(pids);
*read_resultp = read_result;
return (NULL);
}
int main(int argc, char *argv[]) {
char *filename, *ndisp_str, *fdata;
unsigned int fd, ndisp;
struct stat finfo;
if (argv[1] == NULL) {
printf("USAGE: %s <filename> <# results>\n", argv[0]);
exit(1);
}
// Initialize some variables
filename = argv[1];
ndisp_str = argv[2];
CHECK_ERROR((ndisp = ((ndisp_str == NULL) ?
DEFAULT_DISP_NUM : atoi(ndisp_str))) <= 0);
printf("Sequential Sort: Running...\n");
// Open the file
CHECK_ERROR((fd = open(filename, O_RDONLY)) < 0);
CHECK_ERROR(fstat(fd, &finfo) < 0);
#ifndef NO_MMAP
// Memory map the file
printf("Memory mapping the file (with MMAP_POPULATE)\n");
CHECK_ERROR((fdata = (char*) mmap(0, finfo.st_size + 1, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_POPULATE, fd, 0)) == NULL);
#else
uint64_t r = 0;
printf("Mallocing the file\n");
fdata = (char *)malloc (finfo.st_size);
CHECK_ERROR(fdata == NULL);
while(r < (uint64_t)finfo.st_size)
r += pread (fd, fdata + r, finfo.st_size, r);
CHECK_ERROR(r != (uint64_t) finfo.st_size);
#endif
// Prepare splitter arguments
Sorter *sort = new Sorter(fdata, finfo.st_size);
// Divide file on a word border (i.e. a space)
printf("Sequential Sort: Calling word count\n");
sort->splitter();
// Sort the results
sort->sort();
printf("Sequential Sort: Completed\n");
// Print out results
sort->print_results(ndisp);
// Cleanup
delete sort;
#ifndef NO_MMAP
CHECK_ERROR(munmap(fdata, finfo.st_size + 1) < 0);
#else
free (fdata);
#endif
CHECK_ERROR(close(fd));
return 0;
}
/*
* Parse object file located at offset hdr reading data using function
* pread. Save what is useful in info.
*/
int
cle_read_info(struct cle_info *info,
int (*pread)(void *, int, off_t),
off_t hdr)
{
/*
* Save stackspace by using a union!
*
* Beware that the contents of ehdr is gone when shdr is written!!!
*/
union {
struct elf32_ehdr ehdr;
struct elf32_shdr shdr;
} huge;
#define ehdr huge.ehdr
#define shdr huge.shdr
off_t shoff;
cle_off strs;
cle_half shnum; /* number shdrs */
cle_half shentsize; /* sizeof shdr */
cle_word strtabsize = 0;
int i, ret;
memset(info, 0x0, sizeof(*info));
ret = pread(&ehdr, sizeof(ehdr), hdr);
assert(ret > 0);
/* Make sure that we have a correct and compatible ELF header. */
if(memcmp(ehdr.e_ident, ELF_MAGIC_HEADER, ELF_MAGIC_HEADER_SIZE) != 0) {
return CLE_BAD_HEADER;
}
shoff = hdr + ehdr.e_shoff;
shentsize = ehdr.e_shentsize;
shnum = ehdr.e_shnum;
/* The string table section: holds the names of the sections. */
ret = pread(&shdr, sizeof(shdr), shoff + shentsize*ehdr.e_shstrndx);
assert(ret > 0);
/* BEWARE THAT ehdr IS NOW OVERWRITTEN!!! */
/*
* Get a pointer to the actual table of strings. This table holds
* the names of the sections, not the names of other symbols in the
* file (these are in the symtab section).
*/
strs = shdr.sh_offset;
/*
* The ".text" segment holds the actual code from the ELF file, the
* ".data" segment contains initialized data, the ".bss" segment
* holds the size of the unitialized data segment. The ".rela.text"
* and ".rela.data" segments contains relocation information for the
* contents of the ".text" and ".data" segments, respectively. The
* ".symtab" segment contains the symbol table for this file. The
* ".strtab" segment points to the actual string names used by the
* symbol table.
*
* In addition to grabbing pointers to the relevant sections, we
* also save the section index for resolving addresses in the
* relocator code.
*/
for(i = 0; i < shnum; ++i) {
ret = pread(&shdr, sizeof(shdr), shoff);
assert(ret > 0);
/* The name of the section is contained in the strings table. */
ret = pread(info->name, sizeof(info->name), hdr + strs + shdr.sh_name);
assert(ret > 0);
if(strncmp(info->name, ".text", 5) == 0) {
info->textoff = shdr.sh_offset;
info->textsize = shdr.sh_size;
info->text_shndx = i;
} else if(strncmp(info->name, ".rela.text", 10) == 0) {
info->textrelaoff = shdr.sh_offset;
info->textrelasize = shdr.sh_size;
} else if(strncmp(info->name, ".data", 5) == 0) {
info->dataoff = shdr.sh_offset;
info->datasize = shdr.sh_size;
info->data_shndx = i;
} else if(strncmp(info->name, ".rela.data", 10) == 0) {
info->datarelaoff = shdr.sh_offset;
info->datarelasize = shdr.sh_size;
} else if(strncmp(info->name, ".symtab", 7) == 0) {
info->symtaboff = shdr.sh_offset;
info->symtabsize = shdr.sh_size;
} else if(strncmp(info->name, ".strtab", 7) == 0) {
info->strtaboff = shdr.sh_offset;
strtabsize = shdr.sh_size;
} else if(strncmp(info->name, ".bss", 4) == 0) {
info->bsssize = shdr.sh_size;
info->bss_shndx = i;
} else {
info->name[sizeof(info->name) - 1] = 0;
PRINTF("cle: unknown section %.12s\n", info->name);
}
/* Move on to the next section header. */
shoff += shentsize;
}
if(info->symtabsize == 0) {
return CLE_NO_SYMTAB;
}
if(strtabsize == 0) {
return CLE_NO_STRTAB;
}
if(info->textsize == 0) {
return CLE_NO_TEXT;
}
return CLE_OK;
}
void
check_clone(int clonenum)
{
char filename[128];
char imagename[128];
int ret, fd;
struct rbd_ctx cur_ctx = RBD_CTX_INIT;
struct stat file_info;
char *good_buf, *temp_buf;
clone_imagename(imagename, sizeof(imagename), clonenum);
if ((ret = ops->open(imagename, &cur_ctx)) < 0) {
prterrcode("check_clone: ops->open", ret);
exit(167);
}
clone_filename(filename, sizeof(filename), clonenum + 1);
if ((fd = open(filename, O_RDONLY)) < 0) {
simple_err("check_clone: open", -errno);
exit(168);
}
prt("checking clone #%d, image %s against file %s\n",
clonenum, imagename, filename);
if ((ret = fstat(fd, &file_info)) < 0) {
simple_err("check_clone: fstat", -errno);
exit(169);
}
good_buf = NULL;
ret = posix_memalign((void **)&good_buf,
MAX(writebdy, (int)sizeof(void *)),
file_info.st_size);
if (ret > 0) {
prterrcode("check_clone: posix_memalign(good_buf)", -ret);
exit(96);
}
temp_buf = NULL;
ret = posix_memalign((void **)&temp_buf,
MAX(readbdy, (int)sizeof(void *)),
file_info.st_size);
if (ret > 0) {
prterrcode("check_clone: posix_memalign(temp_buf)", -ret);
exit(97);
}
if ((ret = pread(fd, good_buf, file_info.st_size, 0)) < 0) {
simple_err("check_clone: pread", -errno);
exit(170);
}
if ((ret = ops->read(&cur_ctx, 0, file_info.st_size, temp_buf)) < 0) {
prterrcode("check_clone: ops->read", ret);
exit(171);
}
close(fd);
if ((ret = ops->close(&cur_ctx)) < 0) {
prterrcode("check_clone: ops->close", ret);
exit(174);
}
check_buffers(good_buf, temp_buf, 0, file_info.st_size);
unlink(filename);
free(good_buf);
free(temp_buf);
}
int
ioctx_pread (IOCtx ioctx, void *buf, size_t count, off_t offset)
{
return pread (ioctx->fd, buf, count, offset);
}
void
bread(ufs2_daddr_t blkno, char *buf, int size)
{
int secsize, bytes, resid, xfer, base, cnt, i;
static char *tmpbuf;
off_t offset;
loop:
offset = blkno << dev_bshift;
secsize = sblock->fs_fsize;
base = offset % secsize;
resid = size % secsize;
/*
* If the transfer request starts or ends on a non-sector
* boundary, we must read the entire sector and copy out
* just the part that we need.
*/
if (base == 0 && resid == 0) {
cnt = cread(diskfd, buf, size, offset);
if (cnt == size)
return;
} else {
if (tmpbuf == NULL && (tmpbuf = malloc(secsize)) == 0)
quit("buffer malloc failed\n");
xfer = 0;
bytes = size;
if (base != 0) {
cnt = cread(diskfd, tmpbuf, secsize, offset - base);
if (cnt != secsize)
goto bad;
xfer = MIN(secsize - base, size);
offset += xfer;
bytes -= xfer;
resid = bytes % secsize;
memcpy(buf, &tmpbuf[base], xfer);
}
if (bytes >= secsize) {
cnt = cread(diskfd, &buf[xfer], bytes - resid, offset);
if (cnt != bytes - resid)
goto bad;
xfer += cnt;
offset += cnt;
}
if (resid == 0)
return;
cnt = cread(diskfd, tmpbuf, secsize, offset);
if (cnt == secsize) {
memcpy(&buf[xfer], tmpbuf, resid);
return;
}
}
bad:
if (blkno + (size / dev_bsize) > fsbtodb(sblock, sblock->fs_size)) {
/*
* Trying to read the final fragment.
*
* NB - dump only works in TP_BSIZE blocks, hence
* rounds `dev_bsize' fragments up to TP_BSIZE pieces.
* It should be smarter about not actually trying to
* read more than it can get, but for the time being
* we punt and scale back the read only when it gets
* us into trouble. (mkm 9/25/83)
*/
size -= dev_bsize;
goto loop;
}
if (cnt == -1)
msg("read error from %s: %s: [block %jd]: count=%d\n",
disk, strerror(errno), (intmax_t)blkno, size);
else
msg("short read error from %s: [block %jd]: count=%d, got=%d\n",
disk, (intmax_t)blkno, size, cnt);
if (++breaderrors > BREADEMAX) {
msg("More than %d block read errors from %s\n",
BREADEMAX, disk);
broadcast("DUMP IS AILING!\n");
msg("This is an unrecoverable error.\n");
if (!query("Do you want to attempt to continue?")){
dumpabort(0);
/*NOTREACHED*/
} else
breaderrors = 0;
}
/*
* Zero buffer, then try to read each sector of buffer separately,
* and bypass the cache.
*/
memset(buf, 0, size);
for (i = 0; i < size; i += dev_bsize, buf += dev_bsize, blkno++) {
if ((cnt = pread(diskfd, buf, (int)dev_bsize,
((off_t)blkno << dev_bshift))) == dev_bsize)
continue;
if (cnt == -1) {
msg("read error from %s: %s: [sector %jd]: count=%ld\n",
disk, strerror(errno), (intmax_t)blkno, dev_bsize);
continue;
}
msg("short read from %s: [sector %jd]: count=%ld, got=%d\n",
disk, (intmax_t)blkno, dev_bsize, cnt);
}
}
int
mcachefs_read_file(struct mcachefs_file_t *mfile, char *buf, size_t size,
off_t offset)
{
int res = 0;
int fd;
int use_real;
struct mcachefs_metadata_t *mdata;
use_real = mcachefs_read_wait_accessible(mfile, size, offset);
if (use_real && mcachefs_config_get_read_state() == MCACHEFS_STATE_HANDSUP)
{
Err(
"While reading '%s' : mcachefs state set to HANDSUP.\n", mfile->path);
return -EIO;
}
fd = mcachefs_file_getfd(mfile, use_real, O_RDONLY);
Log("reading : fd=%d, use_real=%d\n", fd, use_real);
if (fd < 0)
{
Err(
"Could not get file descriptor for '%s' on %s : err=%d:%s\n", mfile->path, use_real ? "target" : "backing", -fd, strerror (-fd));
return -EIO;
}
res = pread(fd, buf, size, offset);
if (res < 0)
{
res = -errno;
Err(
"Error while reading '%s' on fd=%d, real=%d : %d:%s\n", mfile->path, fd, use_real, errno, strerror (errno));
}
else
{
mfile->sources[use_real].nbrd++;
mfile->sources[use_real].bytesrd += res;
}
mcachefs_file_putfd(mfile, use_real);
if (res > 0)
{
mcachefs_file_update_metadata(mfile, 0, 0);
}
if (res != (int) size)
{
mdata = mcachefs_file_get_metadata(mfile);
if (!mdata)
{
Err("Could not fetch metadata for '%s' !\n", mfile->path);
}
else if ((off_t) size + offset > mdata->st.st_size)
{
Log(
"Read after tail '%s' : size=%lu, offset=%lu, end=%lu, size=%lu\n", mfile->path, (unsigned long) size, (unsigned long) offset, (unsigned long) ((off_t) size + offset), (unsigned long) mdata->st.st_size);
}
else
{
Err(
"Could not fully read '%s' : asked=%lu, had %d, offset=%lu, max=%lu, tail=%lu\n", mfile->path, (unsigned long) size, res, (unsigned long) offset, (unsigned long) (offset + size), (unsigned long) mdata->st.st_size);
}
if (mdata)
mcachefs_metadata_release(mdata);
}
Log("read : res=%d\n", res);
return res;
}
static off_t pread_cb(void *handle, void *buf, size_t count, off_t offset)
{
return pread(*((int*)handle), buf, count, offset);
}
/* Used to test read and write speed. */
static char *
debug_device_speed (const char *subcmd, size_t argc, char *const *const argv)
{
const char *device;
int writing, err;
unsigned secs;
int64_t size, position, copied;
CLEANUP_FREE void *buf = NULL;
struct timeval now, end;
ssize_t r;
int fd;
char *ret;
if (argc != 3) {
bad_args:
reply_with_error ("device_speed <device> <r|w> <secs>");
return NULL;
}
device = argv[0];
if (STREQ (argv[1], "r") || STREQ (argv[1], "read"))
writing = 0;
else if (STREQ (argv[1], "w") || STREQ (argv[1], "write"))
writing = 1;
else
goto bad_args;
if (sscanf (argv[2], "%u", &secs) != 1)
goto bad_args;
/* Find the size of the device. */
size = do_blockdev_getsize64 (device);
if (size == -1)
return NULL;
if (size < BUFSIZ) {
reply_with_error ("%s: device is too small", device);
return NULL;
}
/* Because we're using O_DIRECT, the buffer must be aligned. */
err = posix_memalign (&buf, 4096, BUFSIZ);
if (err != 0) {
reply_with_error_errno (err, "posix_memalign");
return NULL;
}
/* Any non-zero data will do. */
memset (buf, 100, BUFSIZ);
fd = open (device, (writing ? O_WRONLY : O_RDONLY) | O_CLOEXEC | O_DIRECT);
if (fd == -1) {
reply_with_perror ("open: %s", device);
return NULL;
}
/* Now we read or write to the device, wrapping around to the
* beginning when we reach the end, and only stop when <secs>
* seconds has elapsed.
*/
gettimeofday (&end, NULL);
end.tv_sec += secs;
position = copied = 0;
for (;;) {
gettimeofday (&now, NULL);
if (now.tv_sec > end.tv_sec ||
(now.tv_sec == end.tv_sec && now.tv_usec > end.tv_usec))
break;
/* Because of O_DIRECT, only write whole, aligned buffers. */
again:
if (size - position < BUFSIZ) {
position = 0;
goto again;
}
/*
if (verbose) {
fprintf (stderr, "p%s (fd, buf, %d, %" PRIi64 ")\n",
writing ? "write" : "read", BUFSIZ, position);
}
*/
if (writing) {
r = pwrite (fd, buf, BUFSIZ, position);
if (r == -1) {
reply_with_perror ("write: %s", device);
goto error;
}
}
else {
r = pread (fd, buf, BUFSIZ, position);
if (r == -1) {
reply_with_perror ("read: %s", device);
goto error;
}
if (r == 0) {
reply_with_error ("unexpected end of file while reading");
goto error;
}
}
position += BUFSIZ;
copied += r;
}
if (close (fd) == -1) {
reply_with_perror ("close: %s", device);
return NULL;
}
if (asprintf (&ret, "%" PRIi64, copied) == -1) {
reply_with_perror ("asprintf");
return NULL;
}
return ret;
error:
close (fd);
return NULL;
}
static int _cdb_read_records(cdb_t *cdb, cdb_request_t *request, uint64_t *num_recs, cdb_record_t **records) {
int64_t first_requested_logical_record;
int64_t last_requested_logical_record;
uint64_t last_requested_physical_record;
int64_t seek_logical_record;
int64_t seek_physical_record;
cdb_record_t *buffer = NULL;
if (cdb_read_header(cdb) != CDB_SUCCESS) {
return cdb_error();
}
if (request->start != 0 && request->end != 0 && request->end < request->start) {
return CDB_ETMRANGE;
}
if (cdb->header == NULL || cdb->synced == false) {
return CDB_ESANITY;
}
/* bail out if there are no records */
if (cdb->header->num_records <= 0) {
return CDB_ENORECS;
}
/*
get the number of requested records:
-ve indicates n records from the beginning
+ve indicates n records off of the end.
0 or undef means the whole thing.
switch the meaning of -ve/+ve to be more array like
*/
if (request->count != 0) {
request->count = -request->count;
}
#ifdef DEBUG
printf("read_records start: [%ld]\n", request->start);
printf("read_records end: [%ld]\n", request->end);
printf("read_records num_requested: [%"PRIu64"]\n", request->count);
#endif
if (request->count != 0 && request->count < 0 && request->start == 0) {
/* if reading only few records from the end, just set -ve offset to seek to */
first_requested_logical_record = request->count;
} else {
/* compute which record to start reading from the beginning, based on start time specified. */
first_requested_logical_record = _logical_record_for_time(cdb, request->start, 0, 0);
}
/* if end is not defined, read all the records or only read uptill the specified record. */
if (request->end == 0) {
last_requested_logical_record = cdb->header->num_records - 1;
} else {
last_requested_logical_record = _logical_record_for_time(cdb, request->end, 0, 0);
/* this can return something > end, check for that */
if (_time_for_logical_record(cdb, last_requested_logical_record) > request->end) {
last_requested_logical_record -= 1;
}
}
last_requested_physical_record = (last_requested_logical_record + cdb->header->start_record) % cdb->header->num_records;
seek_logical_record = first_requested_logical_record;
seek_physical_record = _seek_to_logical_record(cdb, seek_logical_record);
if (last_requested_physical_record >= seek_physical_record) {
uint64_t nrec = (last_requested_physical_record - seek_physical_record + 1);
uint64_t rlen = RECORD_SIZE * nrec;
if ((buffer = calloc(1, rlen)) == NULL) {
free(buffer);
return CDB_ENOMEM;
}
/* one slurp - XXX TODO - mmap */
if (read(cdb->fd, buffer, rlen) != rlen) {
free(buffer);
return cdb_error();
}
*num_recs = nrec;
} else {
/* We've wrapped around the end of the file */
uint64_t nrec1 = (cdb->header->num_records - seek_physical_record);
uint64_t nrec2 = (last_requested_physical_record + 1);
uint64_t rlen1 = RECORD_SIZE * nrec1;
uint64_t rlen2 = RECORD_SIZE * nrec2;
if ((buffer = calloc(1, rlen1 + rlen2)) == NULL) {
free(buffer);
return CDB_ENOMEM;
}
if (read(cdb->fd, buffer, rlen1) != rlen1) {
free(buffer);
return cdb_error();
}
if (pread(cdb->fd, &buffer[nrec1], rlen2, HEADER_SIZE) != rlen2) {
free(buffer);
return cdb_error();
}
*num_recs = nrec1 + nrec2;
}
/* Deal with cooking the output */
if (request->cooked) {
bool check_min_max = true;
long factor = 0;
uint64_t i = 0;
uint64_t cooked_recs = 0;
double prev_value = 0.0;
cdb_time_t prev_date = 0;
cdb_record_t *crecords;
if ((crecords = calloc(*num_recs, RECORD_SIZE)) == NULL) {
free(crecords);
free(buffer);
return CDB_ENOMEM;
}
if (_compute_scale_factor_and_num_records(cdb, &request->count, &factor)) {
free(crecords);
free(buffer);
return cdb_error();
}
if (cdb->header->min_value == 0 && cdb->header->max_value == 0) {
check_min_max = false;
}
for (i = 0; i < *num_recs; i++) {
cdb_time_t date = buffer[i].time;
double value = buffer[i].value;
if (cdb->header->type == CDB_TYPE_COUNTER) {
double new_value = value;
value = CDB_NAN;
if (!isnan(prev_value) && !isnan(new_value)) {
double val_delta = new_value - prev_value;
if (val_delta >= 0) {
value = val_delta;
}
}
prev_value = new_value;
}
if (factor != 0 && cdb->header->type == CDB_TYPE_COUNTER) {
/* Skip the first entry, since it's absolute and is needed
* to calculate the second */
if (prev_date == 0) {
prev_date = date;
continue;
}
cdb_time_t time_delta = date - prev_date;
if (time_delta > 0 && !isnan(value)) {
value = factor * (value / time_delta);
}
prev_date = date;
}
/* Check for min/max boundaries */
/* Should this be done on write instead of read? */
if (check_min_max && !isnan(value)) {
if (value > cdb->header->max_value || value < cdb->header->min_value) {
value = CDB_NAN;
}
}
/* Copy the munged data to our new array, since we might skip
* elements. Also keep in mind mmap for the future */
crecords[cooked_recs].time = date;
crecords[cooked_recs].value = value;
cooked_recs += 1;
}
/* Now swap our cooked records for the buffer, so we can slice it as needed. */
free(buffer);
buffer = crecords;
*num_recs = cooked_recs;
}
/* If we've been requested to average the records & timestamps */
if (request->step > 1) {
cdb_record_t *arecords;
long step = request->step;
uint64_t step_recs = 0;
uint64_t leftover = (*num_recs % step);
uint64_t walkend = (*num_recs - leftover);
uint64_t i = 0;
if ((arecords = calloc((int)((*num_recs / step) + leftover), RECORD_SIZE)) == NULL) {
free(arecords);
free(buffer);
return CDB_ENOMEM;
}
/* Walk our list of cooked records, jumping ahead by the given step.
For each set of records within that step, we want to get the average
for those records and place them into a new array.
*/
for (i = 0; i < walkend; i += step) {
int j = 0;
double xi[step];
double yi[step];
for (j = 0; j < step; j++) {
/* No NaNs on average - they make bogus graphs. Is there a
* better value than 0 to use here? */
if (isnan(buffer[i+j].value)) {
buffer[i+j].value = 0;
}
xi[j] = (double)buffer[i+j].time;
yi[j] = buffer[i+j].value;
}
arecords[step_recs].time = (cdb_time_t)gsl_stats_mean(xi, 1, step);
arecords[step_recs].value = gsl_stats_mean(yi, 1, step);
step_recs += 1;
}
/* Now collect from the last step point to the end and average. */
if (leftover > 0) {
uint64_t leftover_start = *num_recs - leftover;
int j = 0;
double xi[leftover];
double yi[leftover];
for (i = leftover_start; i < *num_recs; i++) {
/* No NaNs on average - they make bogus graphs. Is there a
* better value than 0 to use here? */
if (isnan(buffer[i].value)) {
buffer[i].value = 0;
}
xi[j] = (double)buffer[i].time;
yi[j] = buffer[i].value;
j++;
}
arecords[step_recs].time = (cdb_time_t)gsl_stats_mean(xi, 1, j);
arecords[step_recs].value = gsl_stats_mean(yi, 1, j);
step_recs += 1;
}
free(buffer);
buffer = arecords;
*num_recs = step_recs;
}
/* now pull out the number of requested records if asked */
if (request->count != 0 && *num_recs >= abs(request->count)) {
uint64_t start_index = 0;
if (request->count <= 0) {
start_index = *num_recs - abs(request->count);
}
*num_recs = abs(request->count);
if ((*records = calloc(*num_recs, RECORD_SIZE)) == NULL) {
free(buffer);
return CDB_ENOMEM;
}
memcpy(*records, &buffer[start_index], RECORD_SIZE * *num_recs);
free(buffer);
} else {
*records = buffer;
}
return CDB_SUCCESS;
}
int storage_read(storage_t * st, uint8_t layout, sid_t * dist_set,
uint8_t spare, fid_t fid, bid_t bid, uint32_t nb_proj,
bin_t * bins, size_t * len_read, uint64_t *file_size) {
int status = -1;
char path[FILENAME_MAX];
int fd = -1;
size_t nb_read = 0;
size_t length_to_read = 0;
off_t bins_file_offset = 0;
uint16_t rozofs_max_psize = 0;
struct stat sb;
// Build the full path of directory that contains the bins file
storage_map_distribution(st, layout, dist_set, spare, path);
// Build the path of bins file
storage_map_projection(fid, path);
// Open bins file
fd = open(path, ROZOFS_ST_BINS_FILE_FLAG, ROZOFS_ST_BINS_FILE_MODE);
if (fd < 0) {
DEBUG("open failed (%s) : %s", path, strerror(errno));
goto out;
}
// Compute the offset and length to read
rozofs_max_psize = rozofs_get_max_psize(layout);
bins_file_offset = ROZOFS_ST_BINS_FILE_HDR_SIZE +
bid * ((off_t) (rozofs_max_psize * sizeof (bin_t)) +
sizeof (rozofs_stor_bins_hdr_t));
length_to_read = nb_proj * (rozofs_max_psize * sizeof (bin_t)
+ sizeof (rozofs_stor_bins_hdr_t));
// Read nb_proj * (projection + header)
nb_read = pread(fd, bins, length_to_read, bins_file_offset);
// Check error
if (nb_read == -1) {
severe("pread failed: %s", strerror(errno));
goto out;
}
// Check the length read
if ((nb_read % (rozofs_max_psize * sizeof (bin_t) +
sizeof (rozofs_stor_bins_hdr_t))) != 0) {
char fid_str[37];
uuid_unparse(fid, fid_str);
severe("storage_read failed (FID: %s): read inconsistent length",
fid_str);
errno = EIO;
goto out;
}
// Update the length read
*len_read = nb_read;
// Stat file for return the size of bins file after the read operation
if (fstat(fd, &sb) == -1) {
severe("fstat failed: %s", strerror(errno));
goto out;
}
*file_size = sb.st_size;
// Read is successful
status = 0;
out:
if (fd != -1) close(fd);
return status;
}
// retrieve file buffer from local storage
// if success == TRUE then "buf" contains "size" bytes of data
void cache_mng_retrieve_file_buf (CacheMng *cmng, fuse_ino_t ino, size_t size, off_t off,
cache_mng_on_retrieve_file_buf_cb on_retrieve_file_buf_cb, void *ctx)
{
struct _CacheContext *context;
struct _CacheEntry *entry;
context = cache_context_create (size, ctx);
context->cb.retrieve_cb = on_retrieve_file_buf_cb;
entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));
if (entry && range_contain (entry->avail_range, off, off + size)) {
int fd;
ssize_t res;
char path[PATH_MAX];
if (ino != entry->ino) {
LOG_err (CMNG_LOG, INO_H"Requested inode doesn't match hashed key!", INO_T (ino));
if (context->cb.retrieve_cb)
context->cb.retrieve_cb (NULL, 0, FALSE, context->user_ctx);
cache_context_destroy (context);
cmng->cache_miss++;
return;
}
cache_mng_file_name (cmng, path, sizeof (path), ino);
fd = open (path, O_RDONLY);
if (fd < 0) {
LOG_err (CMNG_LOG, INO_H"Failed to open file for reading! Path: %s", INO_T (ino), path);
if (context->cb.retrieve_cb)
context->cb.retrieve_cb (NULL, 0, FALSE, context->user_ctx);
cache_context_destroy (context);
cmng->cache_miss++;
return;
}
context->buf = g_malloc (size);
res = pread (fd, context->buf, size, off);
close (fd);
context->success = (res == (ssize_t) size);
LOG_debug (CMNG_LOG, INO_H"Read [%"OFF_FMT":%zu] bytes, result: %s",
INO_T (ino), off, size, context->success ? "OK" : "Failed");
if (!context->success) {
g_free (context->buf);
context->buf = NULL;
cmng->cache_miss++;
} else
cmng->cache_hits++;
// move entry to the front of q_lru
g_queue_unlink (cmng->q_lru, entry->ll_lru);
g_queue_push_head_link (cmng->q_lru, entry->ll_lru);
} else {
LOG_debug (CMNG_LOG, INO_H"Entry isn't found or doesn't contain requested range: [%"OFF_FMT": %"OFF_FMT"]",
INO_T (ino), off, off + size);
cmng->cache_miss++;
}
context->ev = event_new (application_get_evbase (cmng->app), -1, 0,
cache_read_cb, context);
// fire this event at once
event_active (context->ev, 0, 0);
event_add (context->ev, NULL);
}
/*
* Relocate one segment that has been copied to the location pointed
* to by segmem.
*
* Relocation info is read from offset reloff to (reloff + relsize)
* and the start of the object file is at hdr. Data is read using
* function pread.
*/
int
cle_relocate(struct cle_info *info,
int (*pread)(void *, int, off_t),
off_t hdr, /* Offset to start of file. */
void *segmem, /* Where segment is stored in memory. */
cle_off reloff, /* .rela.<segment> start */
cle_word relsize) /* .rela.<segment> size */
{
struct elf32_rela rela;
struct elf32_sym s;
off_t off;
cle_addr addr;
int ret;
for(off = hdr + reloff;
off < hdr + reloff + relsize;
off += sizeof(struct elf32_rela)) {
ret = pread(&rela, sizeof(rela), off);
assert(ret > 0);
ret = pread(&s, sizeof(s),
hdr + info->symtaboff
+ sizeof(struct elf32_sym)*ELF32_R_SYM(rela.r_info));
assert(ret > 0);
if(s.st_shndx == info->bss_shndx) {
addr = (cle_addr)(uintptr_t)info->bss;
} else if(s.st_shndx == info->data_shndx) {
addr = (cle_addr)(uintptr_t)info->data;
} else if(s.st_shndx == info->text_shndx) {
addr = info->text;
} else {
addr = NOLL;
}
if(s.st_name == 0) { /* No name, local symbol? */
if(addr == NOLL) {
return CLE_UNKNOWN_SEGMENT;
}
} else {
ret = pread(info->name, sizeof(info->name),
hdr + info->strtaboff + s.st_name);
assert(ret > 0);
cle_addr sym = (cle_addr)(uintptr_t)sym_function(info->name);
#ifdef __AVR__
if(sym != NOLL)
sym = sym << 1;
#endif
if(sym == NOLL)
sym = (cle_addr)(uintptr_t)sym_object(info->name);
if(addr == NOLL && sym != NOLL) { /* Imported symbol. */
addr = sym;
} else if(addr != NOLL && sym == NOLL) { /* Exported symbol. */
addr = addr + s.st_value;
} else if(addr == NOLL && sym == NOLL) {
PRINTF("cle: undefined reference to %.32s (%d)\n",
info->name, s.st_info);
return CLE_UNDEFINED; /* Or COMMON symbol. */
} else if(addr != NOLL && sym != NOLL) {
PRINTF("cle: multiple definitions of %.32s (%d)\n",
info->name, s.st_info);
return CLE_MULTIPLY_DEFINED;
}
}
addr += rela.r_addend;
ret = cle_write_reloc(segmem + rela.r_offset, &rela, addr, info);
if(ret != CLE_OK) {
return ret;
}
}
return CLE_OK;
}
void LogFile::readAt(unsigned long long offset, void *_buf, size_t _len) {
verify(((size_t)_buf) % g_minOSPageSizeBytes == 0); // aligned
ssize_t rd = pread(_fd, _buf, _len, offset);
verify( rd != -1 );
}
