void vfileLogger(int priority, const char *format, va_list optional_arguments) {
int n = 0;
int error = EXIT_SUCCESS;
const int saved_errno = errno;
int internalError = EXIT_SUCCESS;
const int LogMask = setlogmask(0);
/* Check priority against setlogmask values. */
if ((LOG_MASK (LOG_PRI (priority)) & LogMask) != 0) {
//char logMsg[2048];
//char logFormat[1024];
//char *cursor = logFormat;
char *buf = 0;
size_t bufsize = 1024;
FILE *f = open_memstream(&buf, &bufsize);
if (f != NULL) {
struct tm now_tm;
time_t now;
(void) time(&now);
/*cursor += strftime(cursor,sizeof(logFormat),"%h %e %T ",localtime_r(&now, &now_tm));*/
n = strftime(f->_IO_write_ptr,f->_IO_write_end - f->_IO_write_ptr,"%h %e %T ",localtime_r(&now, &now_tm));
//f->_IO_write_ptr += strftime(f->_IO_write_ptr,f->_IO_write_end - f->_IO_write_ptr,"%h %e %T ",localtime_r(&now, &now_tm));
f->_IO_write_ptr += n;
if (LogTag) {
/*cursor += sprintf (cursor,"%s: ",LogTag);*/
n += fprintf(f,"%s: ",LogTag);
}
if (LogStat & LOG_PID) {
if (LogStat & LOG_TID) {
const pid_t tid = gettid();
/*cursor += sprintf (cursor, "[%d:%d]", (int) getpid (),(int) tid);*/
n += fprintf(f,"[%d:%d]", (int) getpid (),(int) tid);
} else {
/*cursor += sprintf (cursor, "[%d]", (int) getpid ());*/
n += fprintf(f,"[%d]", (int) getpid ());
}
}
if (LogStat & LOG_RDTSC) {
const unsigned long long int t = rdtsc();
/*cursor += sprintf (cursor, "(%llu)",t);*/
n += fprintf(f,"(%llu)",t);
} /* (LogStat & LOG_RDTSC) */
if (LogStat & LOG_CLOCK) {
#if HAVE_CLOCK_GETTIME
struct timespec timeStamp;
if (clock_gettime(CLOCK_MONOTONIC,&timeStamp) == 0) {
/*cursor += sprintf (cursor, "(%lu.%.9d)",timeStamp.tv_sec,timeStamp.tv_nsec);*/
n += fprintf(f,"(%lu.%.9d)",timeStamp.tv_sec,timeStamp.tv_nsec);
} else {
const int error = errno;
ERROR_MSG("clock_gettime CLOCK_MONOTONIC error %d (%m)",error);
}
#else
static unsigned int alreadyPrinted = 0; /* to avoid to print this error msg on each call */
if (unlikely(0 == alreadyPrinted)) {
ERROR_MSG("clock_gettime not available on this system");
alreadyPrinted = 1;
}
#endif
} /* (LogStat & LOG_CLOCK) */
if (LogStat & LOG_LEVEL) {
switch(LOG_PRI(priority)) {
case LOG_EMERG:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Emergency * %s",format);*/
n += fprintf(f, "[EMERG]");
break;
case LOG_ALERT:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Alert * %s",format);*/
n += fprintf(f, "[ALERT]");
break;
case LOG_CRIT:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Critical * %s",format);*/
n += fprintf(f, "[CRIT]");
break;
case LOG_ERR:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Error * %s",format);*/
n += fprintf(f, "[ERROR]");
break;
case LOG_WARNING:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Warning * %s",format);*/
n += fprintf(f, "[WARNING]");
break;
case LOG_NOTICE:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Notice * %s",format); */
n += fprintf(f, "[NOTICE]");
break;
case LOG_INFO:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Info * %s",format);*/
n += fprintf(f, "[INFO]");
break;
case LOG_DEBUG:
/*cursor += snprintf(cursor,sizeof(logFormat) - (cursor - logFormat),"* Debug * %s",format);*/
n += fprintf(f, "[DEBUG]");
break;
default:
/*cursor += sprintf(cursor,"* <%d> * %s",priority,format);*/
n += fprintf(f,"[<%d>]",priority);
} /* switch(priority) */
} /* (LogStat & LOG_LEVEL) */
errno = saved_errno; /* restore errno for %m format */
n += vfprintf(f, format, optional_arguments);
/* Close the memory stream; this will finalize the data
into a malloc'd buffer in BUF. */
fclose(f);
/*
* begin of the critical section.
* Some of the function used below are thread cancellation points
* (according to Advanced Programming in the Unix Environment 2nd ed p411)
* so set the handler to avoid deadlocks and memory leaks
*/
cleanup_args cancelArgs;
cancelArgs.buffer = buf;
pthread_cleanup_push(cancel_handler, &cancelArgs);
error = pthread_mutex_lock(&fileLock);
if (likely(EXIT_SUCCESS == error)) {
if (unlikely(LogStat & LOG_PERROR)) {
/* add the format argument to the header */
const size_t n = strlen(format);
const size_t size = bufsize + n + 1;
char *logFormatBuffer = (char *)malloc(size);
if (logFormatBuffer) {
pthread_cleanup_push(free,logFormatBuffer);
strcpyNcat(logFormatBuffer,buf,format);
vfprintf(stderr, logFormatBuffer, optional_arguments);
pthread_cleanup_pop(1); /* pop the handler and free the allocated memory */
} else {
ERROR_MSG("failed to allocate %u bytes to print the msg on stderr",size);
}
} /* (LogStat & LOG_PERROR) */
/* file management */
if (unlikely(LOG_FILE_DURATION & LogStat)) {
const time_t currentTime = time(NULL);
const time_t elapsedTime = currentTime - startTime;
if (unlikely(elapsedTime >= maxDuration)) {
close(logFile);
logFile = -1;
}
} /* (LOG_FILE_DURATION & LogStat) */
if (unlikely(-1 == logFile)) {
error = createFile();
/* error already logged */
}
if (likely(EXIT_SUCCESS == error)) {
const ssize_t written = write(logFile,buf,n);
if (written > 0) {
if (unlikely(written != n)) {
ERROR_MSG("only %d byte(s) of %d has been written to %s",written,n,fullFileName);
if (LogStat & LOG_CONS) {
int fd = open("/dev/console", O_WRONLY | O_NOCTTY, 0);
if (fd >= 0 ) {
dprintf(fd,"logMsg");
close(fd);
fd = -1;
}
} /* (LogStat & LOG_CONS) */
if (unlikely((LogStat & LOG_FILE_SYNC_ON_ERRORS_ONLY) && (LOG_PRI(priority) <= LOG_ERR))) {
/* flush data if the log priority is "upper" or equal to error */
error = fdatasync(logFile);
if (error != 0) {
error = errno;
ERROR_MSG("fdatasync to %s error %d (%m)",fullFileName,error);
}
error = posix_fadvise(logFile, 0,0,POSIX_FADV_DONTNEED); /* tell the OS that log message bytes could be released from the file system cache */
if (error != 0) {
ERROR_MSG("posix_fadvise to %s error %d (%m)",fullFileName,error);
}
} /* (unlikely((LogStat & LOG_FILE_SYNC_ON_ERRORS_ONLY) && (LOG_PRI(priority) <= LOG_ERR))) */
} /* (unlikely(written != n)) */
fileSize += written;
#ifdef FILESYSTEM_PAGE_CACHE_FLUSH_THRESHOLD
static size_t currentPageCacheMaxSize = 0;
currentPageCacheMaxSize += written;
if (currentPageCacheMaxSize >= FILESYSTEM_PAGE_CACHE_FLUSH_THRESHOLD) {
/* tell the OS that log message bytes could be released from the file system cache */
if (likely(posix_fadvise(logFile, 0,0,POSIX_FADV_DONTNEED) == 0)) {
currentPageCacheMaxSize = 0;
DEBUG_MSG("used file system cache allowed to be flushed (size was %u)",currentPageCacheMaxSize);
} else {
NOTICE_MSG("posix_fadvise to %s error %d (%m), current page cache max size is %u",fullFileName,error,currentPageCacheMaxSize);
}
}
#endif /* FILESYSTEM_PAGE_CACHE_FLUSH_THRESHOLD */
if ((LOG_FILE_ROTATE & LogStat) || (LOG_FILE_HISTO & LogStat)) {
if (fileSize >= maxSize) {
close(logFile);
logFile = -1;
}
}
} else if (0 == written) {
WARNING_MSG("nothing has been written in %s", fullFileName);
} else {
error = errno;
ERROR_MSG("write to %s error %d (%m)", fullFileName, error);
}
} /*(likely(EXIT_SUCCESS == error)) */
/* End of critical section. */
/*pthread_cleanup_pop(0); implementation can't allow to set this instruction here */
/* free allocated ressources */
internalError = pthread_mutex_unlock(&fileLock);
if (internalError != EXIT_SUCCESS) {
ERROR_MSG("pthread_mutex_lock fileLock error %d (%s)", internalError, strerror(internalError));
if (EXIT_SUCCESS == error) {
error = internalError;
}
} /* (internalError != EXIT_SUCCESS) */
} else {
ERROR_MSG("pthread_mutex_lock fileLock error %d (%s)", error, strerror(error));
}
pthread_cleanup_pop(0); /* moved to avoid a build error, use the preprocessor to understand why
* or have a look on man 3 pthread_cleanup_push:
* push & pop MUST BE in the SAME lexical nesting level !!!
*/
free(buf);
cancelArgs.buffer = buf = NULL;
} else { /* open_memstream(&buf, &bufsize) == NULL */
ERROR_MSG("failed to get stream buffer");
/* TMP! display the raw message to the console */
vfprintf(stderr,format,optional_arguments);
/* TODO: write the raw message to the file */
}
} /* ((LOG_MASK (LOG_PRI (priority)) & LogMask) != 0) */
/* message has not to be displayed because of the current LogMask and its priority */
/*return n;*/
}
/*
* __wt_bm_read --
* Map or read address cookie referenced block into a buffer.
*/
int
__wt_bm_read(WT_BM *bm, WT_SESSION_IMPL *session,
WT_ITEM *buf, const uint8_t *addr, size_t addr_size)
{
WT_BLOCK *block;
int mapped;
off_t offset;
uint32_t cksum, size;
WT_UNUSED(addr_size);
block = bm->block;
/* Crack the cookie. */
WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &cksum));
/*
* Map the block if it's possible.
*/
mapped = bm->map != NULL && offset + size <= (off_t)bm->maplen;
if (mapped) {
buf->data = (uint8_t *)bm->map + offset;
buf->size = size;
WT_RET(__wt_mmap_preload(session, buf->data, buf->size));
WT_STAT_FAST_CONN_INCR(session, block_map_read);
WT_STAT_FAST_CONN_INCRV(session, block_byte_map_read, size);
return (0);
}
#ifdef HAVE_DIAGNOSTIC
/*
* In diagnostic mode, verify the block we're about to read isn't on
* the available list, or for live systems, the discard list.
*
* Don't check during the salvage read phase, we might be reading an
* already freed overflow page.
*/
if (!F_ISSET(session, WT_SESSION_SALVAGE_CORRUPT_OK))
WT_RET(__wt_block_misplaced(
session, block, "read", offset, size, bm->is_live));
#endif
/* Read the block. */
WT_RET(__wt_block_read_off(session, block, buf, offset, size, cksum));
#ifdef HAVE_POSIX_FADVISE
/* Optionally discard blocks from the system's buffer cache. */
if (block->os_cache_max != 0 &&
(block->os_cache += size) > block->os_cache_max) {
WT_DECL_RET;
block->os_cache = 0;
/* Ignore EINVAL - some file systems don't support the flag. */
if ((ret = posix_fadvise(block->fh->fd,
(off_t)0, (off_t)0, POSIX_FADV_DONTNEED)) != 0 &&
ret != EINVAL)
WT_RET_MSG(
session, ret, "%s: posix_fadvise", block->name);
}
#endif
return (0);
}
static struct btrfs_fs_info *__open_ctree_fd(int fp, const char *path,
u64 sb_bytenr,
u64 root_tree_bytenr, int writes,
int partial)
{
u32 sectorsize;
u32 nodesize;
u32 leafsize;
u32 blocksize;
u32 stripesize;
u64 generation;
struct btrfs_key key;
struct btrfs_root *tree_root = malloc(sizeof(struct btrfs_root));
struct btrfs_root *extent_root = malloc(sizeof(struct btrfs_root));
struct btrfs_root *chunk_root = malloc(sizeof(struct btrfs_root));
struct btrfs_root *dev_root = malloc(sizeof(struct btrfs_root));
struct btrfs_root *csum_root = malloc(sizeof(struct btrfs_root));
struct btrfs_fs_info *fs_info = malloc(sizeof(*fs_info));
int ret;
struct btrfs_super_block *disk_super;
struct btrfs_fs_devices *fs_devices = NULL;
u64 total_devs;
u64 features;
if (sb_bytenr == 0)
sb_bytenr = BTRFS_SUPER_INFO_OFFSET;
/* try to drop all the caches */
if (posix_fadvise(fp, 0, 0, POSIX_FADV_DONTNEED))
fprintf(stderr, "Warning, could not drop caches\n");
ret = btrfs_scan_one_device(fp, path, &fs_devices,
&total_devs, sb_bytenr);
if (ret) {
fprintf(stderr, "No valid Btrfs found on %s\n", path);
goto out;
}
if (total_devs != 1) {
ret = btrfs_scan_for_fsid(fs_devices, total_devs, 1);
if (ret)
goto out;
}
memset(fs_info, 0, sizeof(*fs_info));
fs_info->super_copy = calloc(1, BTRFS_SUPER_INFO_SIZE);
fs_info->tree_root = tree_root;
fs_info->extent_root = extent_root;
fs_info->chunk_root = chunk_root;
fs_info->dev_root = dev_root;
fs_info->csum_root = csum_root;
if (!writes)
fs_info->readonly = 1;
extent_io_tree_init(&fs_info->extent_cache);
extent_io_tree_init(&fs_info->free_space_cache);
extent_io_tree_init(&fs_info->block_group_cache);
extent_io_tree_init(&fs_info->pinned_extents);
extent_io_tree_init(&fs_info->pending_del);
extent_io_tree_init(&fs_info->extent_ins);
cache_tree_init(&fs_info->fs_root_cache);
cache_tree_init(&fs_info->mapping_tree.cache_tree);
mutex_init(&fs_info->fs_mutex);
fs_info->fs_devices = fs_devices;
INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
INIT_LIST_HEAD(&fs_info->space_info);
__setup_root(4096, 4096, 4096, 4096, tree_root,
fs_info, BTRFS_ROOT_TREE_OBJECTID);
if (writes)
ret = btrfs_open_devices(fs_devices, O_RDWR);
else
ret = btrfs_open_devices(fs_devices, O_RDONLY);
if (ret)
goto out_cleanup;
fs_info->super_bytenr = sb_bytenr;
disk_super = fs_info->super_copy;
ret = btrfs_read_dev_super(fs_devices->latest_bdev,
disk_super, sb_bytenr);
if (ret) {
printk("No valid btrfs found\n");
goto out_devices;
}
memcpy(fs_info->fsid, &disk_super->fsid, BTRFS_FSID_SIZE);
features = btrfs_super_incompat_flags(disk_super) &
~BTRFS_FEATURE_INCOMPAT_SUPP;
if (features) {
printk("couldn't open because of unsupported "
"option features (%Lx).\n",
(unsigned long long)features);
goto out_devices;
}
features = btrfs_super_incompat_flags(disk_super);
if (!(features & BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF)) {
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
btrfs_set_super_incompat_flags(disk_super, features);
}
features = btrfs_super_compat_ro_flags(disk_super) &
~BTRFS_FEATURE_COMPAT_RO_SUPP;
if (writes && features) {
printk("couldn't open RDWR because of unsupported "
"option features (%Lx).\n",
(unsigned long long)features);
goto out_devices;
}
nodesize = btrfs_super_nodesize(disk_super);
leafsize = btrfs_super_leafsize(disk_super);
sectorsize = btrfs_super_sectorsize(disk_super);
stripesize = btrfs_super_stripesize(disk_super);
tree_root->nodesize = nodesize;
tree_root->leafsize = leafsize;
tree_root->sectorsize = sectorsize;
tree_root->stripesize = stripesize;
ret = btrfs_read_sys_array(tree_root);
if (ret)
goto out_devices;
blocksize = btrfs_level_size(tree_root,
btrfs_super_chunk_root_level(disk_super));
generation = btrfs_super_chunk_root_generation(disk_super);
__setup_root(nodesize, leafsize, sectorsize, stripesize,
chunk_root, fs_info, BTRFS_CHUNK_TREE_OBJECTID);
chunk_root->node = read_tree_block(chunk_root,
btrfs_super_chunk_root(disk_super),
blocksize, generation);
if (!extent_buffer_uptodate(chunk_root->node)) {
printk("Couldn't read chunk root\n");
goto out_devices;
}
read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
(unsigned long)btrfs_header_chunk_tree_uuid(chunk_root->node),
BTRFS_UUID_SIZE);
if (!(btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_METADUMP)) {
ret = btrfs_read_chunk_tree(chunk_root);
if (ret) {
printk("Couldn't read chunk tree\n");
goto out_chunk;
}
}
blocksize = btrfs_level_size(tree_root,
btrfs_super_root_level(disk_super));
generation = btrfs_super_generation(disk_super);
if (!root_tree_bytenr)
root_tree_bytenr = btrfs_super_root(disk_super);
tree_root->node = read_tree_block(tree_root,
root_tree_bytenr,
blocksize, generation);
if (!extent_buffer_uptodate(tree_root->node)) {
printk("Couldn't read tree root\n");
goto out_failed;
}
ret = find_and_setup_root(tree_root, fs_info,
BTRFS_EXTENT_TREE_OBJECTID, extent_root);
if (ret) {
printk("Couldn't setup extent tree\n");
goto out_failed;
}
extent_root->track_dirty = 1;
ret = find_and_setup_root(tree_root, fs_info,
BTRFS_DEV_TREE_OBJECTID, dev_root);
if (ret) {
printk("Couldn't setup device tree\n");
goto out_failed;
}
dev_root->track_dirty = 1;
ret = find_and_setup_root(tree_root, fs_info,
BTRFS_CSUM_TREE_OBJECTID, csum_root);
if (ret) {
printk("Couldn't setup csum tree\n");
if (!partial)
goto out_failed;
}
csum_root->track_dirty = 1;
find_and_setup_log_root(tree_root, fs_info, disk_super);
fs_info->generation = generation;
fs_info->last_trans_committed = generation;
btrfs_read_block_groups(fs_info->tree_root);
key.objectid = BTRFS_FS_TREE_OBJECTID;
key.type = BTRFS_ROOT_ITEM_KEY;
key.offset = (u64)-1;
fs_info->fs_root = btrfs_read_fs_root(fs_info, &key);
if (!fs_info->fs_root)
goto out_failed;
fs_info->data_alloc_profile = (u64)-1;
fs_info->metadata_alloc_profile = (u64)-1;
fs_info->system_alloc_profile = fs_info->metadata_alloc_profile;
return fs_info;
out_failed:
if (partial)
return fs_info;
if (fs_info->csum_root)
free_extent_buffer(fs_info->csum_root->node);
if (fs_info->dev_root)
free_extent_buffer(fs_info->dev_root->node);
if (fs_info->extent_root)
free_extent_buffer(fs_info->extent_root->node);
if (fs_info->tree_root)
free_extent_buffer(fs_info->tree_root->node);
out_chunk:
if (fs_info->chunk_root)
free_extent_buffer(fs_info->chunk_root->node);
out_devices:
close_all_devices(fs_info);
out_cleanup:
extent_io_tree_cleanup(&fs_info->extent_cache);
extent_io_tree_cleanup(&fs_info->free_space_cache);
extent_io_tree_cleanup(&fs_info->block_group_cache);
extent_io_tree_cleanup(&fs_info->pinned_extents);
extent_io_tree_cleanup(&fs_info->pending_del);
extent_io_tree_cleanup(&fs_info->extent_ins);
out:
free(tree_root);
free(extent_root);
free(chunk_root);
free(dev_root);
free(csum_root);
free(fs_info);
return NULL;
}
int main(void)
{
posix_fadvise(1, 0, 0, POSIX_FADV_NORMAL);
return 0;
}
void eFilePushThread::thread()
{
setIoPrio(prio_class, prio);
off_t dest_pos = 0, source_pos = 0;
size_t bytes_read = 0;
off_t current_span_offset = 0;
size_t current_span_remaining = 0;
size_t written_since_last_sync = 0;
eDebug("FILEPUSH THREAD START");
/* we set the signal to not restart syscalls, so we can detect our signal. */
struct sigaction act;
act.sa_handler = signal_handler; // no, SIG_IGN doesn't do it. we want to receive the -EINTR
act.sa_flags = 0;
sigaction(SIGUSR1, &act, 0);
hasStarted();
source_pos = m_raw_source.lseek(0, SEEK_CUR);
/* m_stop must be evaluated after each syscall. */
while (!m_stop)
{
/* first try flushing the bufptr */
if (m_buf_start != m_buf_end)
{
/* filterRecordData wants to work on multiples of blocksize.
if it returns a negative result, it means that this many bytes should be skipped
*in front* of the buffer. Then, it will be called again. with the newer, shorter buffer.
if filterRecordData wants to skip more data then currently available, it must do that internally.
Skipped bytes will also not be output.
if it returns a positive result, that means that only these many bytes should be used
in the buffer.
In either case, current_span_remaining is given as a reference and can be modified. (Of course it
doesn't make sense to decrement it to a non-zero value unless you return 0 because that would just
skip some data). This is probably a very special application for fast-forward, where the current
span is to be cancelled after a complete iframe has been output.
we always call filterRecordData with our full buffer (otherwise we couldn't easily strip from the end)
we filter data only once, of course, but it might not get immediately written.
that's what m_filter_end is for - it points to the start of the unfiltered data.
*/
int filter_res;
do
{
filter_res = filterRecordData(m_buffer + m_filter_end, m_buf_end - m_filter_end, current_span_remaining);
if (filter_res < 0)
{
eDebug("[eFilePushThread] filterRecordData re-syncs and skips %d bytes", -filter_res);
m_buf_start = m_filter_end + -filter_res; /* this will also drop unwritten data */
ASSERT(m_buf_start <= m_buf_end); /* otherwise filterRecordData skipped more data than available. */
continue; /* try again */
}
/* adjust end of buffer to strip dropped tail bytes */
m_buf_end = m_filter_end + filter_res;
/* mark data as filtered. */
m_filter_end = m_buf_end;
} while (0);
ASSERT(m_filter_end == m_buf_end);
if (m_buf_start == m_buf_end)
continue;
/* now write out data. it will be 'aligned' (according to filterRecordData).
absolutely forbidden is to return EINTR and consume a non-aligned number of bytes.
*/
int w = write(m_fd_dest, m_buffer + m_buf_start, m_buf_end - m_buf_start);
// fwrite(m_buffer + m_buf_start, 1, m_buf_end - m_buf_start, f);
// eDebug("wrote %d bytes", w);
if (w <= 0)
{
if (errno == EINTR || errno == EAGAIN || errno == EBUSY)
continue;
eDebug("eFilePushThread WRITE ERROR");
sendEvent(evtWriteError);
break;
// ... we would stop the thread
}
written_since_last_sync += w;
if (written_since_last_sync >= 512*1024)
{
int toflush = written_since_last_sync > 2*1024*1024 ?
2*1024*1024 : written_since_last_sync &~ 4095; // write max 2MB at once
dest_pos = lseek(m_fd_dest, 0, SEEK_CUR);
dest_pos -= toflush;
posix_fadvise(m_fd_dest, dest_pos, toflush, POSIX_FADV_DONTNEED);
written_since_last_sync -= toflush;
}
// printf("FILEPUSH: wrote %d bytes\n", w);
m_buf_start += w;
continue;
}
/* now fill our buffer. */
if (m_sg && !current_span_remaining)
{
m_sg->getNextSourceSpan(source_pos, bytes_read, current_span_offset, current_span_remaining);
ASSERT(!(current_span_remaining % m_blocksize));
if (source_pos != current_span_offset)
source_pos = m_raw_source.lseek(current_span_offset, SEEK_SET);
bytes_read = 0;
}
size_t maxread = sizeof(m_buffer);
/* if we have a source span, don't read past the end */
if (m_sg && maxread > current_span_remaining)
maxread = current_span_remaining;
/* align to blocksize */
maxread -= maxread % m_blocksize;
m_buf_start = 0;
m_filter_end = 0;
m_buf_end = 0;
if (maxread)
m_buf_end = m_raw_source.read(m_buffer, maxread);
if (m_buf_end < 0)
{
m_buf_end = 0;
if (errno == EINTR || errno == EBUSY || errno == EAGAIN)
continue;
if (errno == EOVERFLOW)
{
eWarning("OVERFLOW while recording");
continue;
}
eDebug("eFilePushThread *read error* (%m) - not yet handled");
}
/* a read might be mis-aligned in case of a short read. */
int d = m_buf_end % m_blocksize;
if (d)
{
m_raw_source.lseek(-d, SEEK_CUR);
m_buf_end -= d;
}
if (m_buf_end == 0)
{
/* on EOF, try COMMITting once. */
if (m_send_pvr_commit)
{
struct pollfd pfd;
pfd.fd = m_fd_dest;
pfd.events = POLLIN;
switch (poll(&pfd, 1, 250)) // wait for 250ms
{
case 0:
eDebug("wait for driver eof timeout");
continue;
case 1:
eDebug("wait for driver eof ok");
break;
default:
eDebug("wait for driver eof aborted by signal");
continue;
}
}
/* in stream_mode, we are sending EOF events
over and over until somebody responds.
in stream_mode, think of evtEOF as "buffer underrun occured". */
sendEvent(evtEOF);
if (m_stream_mode)
{
eDebug("reached EOF, but we are in stream mode. delaying 1 second.");
sleep(1);
continue;
}
#if 0
eDebug("FILEPUSH: end-of-file! (currently unhandled)");
if (!m_raw_source.lseek(0, SEEK_SET))
{
eDebug("(looping)");
continue;
}
#endif
break;
} else
{
source_pos += m_buf_end;
bytes_read += m_buf_end;
if (m_sg)
current_span_remaining -= m_buf_end;
}
// printf("FILEPUSH: read %d bytes\n", m_buf_end);
}
fdatasync(m_fd_dest);
eDebug("FILEPUSH THREAD STOP");
}
static tr_bool
verifyTorrent( tr_torrent * tor, tr_bool * stopFlag )
{
time_t end;
SHA_CTX sha;
int fd = -1;
int64_t filePos = 0;
tr_bool changed = 0;
tr_bool hadPiece = 0;
time_t lastSleptAt = 0;
uint32_t piecePos = 0;
tr_file_index_t fileIndex = 0;
tr_file_index_t prevFileIndex = !fileIndex;
tr_piece_index_t pieceIndex = 0;
const time_t begin = tr_time( );
const size_t buflen = 1024 * 128; /* 128 KiB buffer */
uint8_t * buffer = tr_valloc( buflen );
SHA1_Init( &sha );
tr_tordbg( tor, "%s", "verifying torrent..." );
tr_torrentSetChecked( tor, 0 );
while( !*stopFlag && ( pieceIndex < tor->info.pieceCount ) )
{
uint32_t leftInPiece;
uint32_t bytesThisPass;
uint64_t leftInFile;
const tr_file * file = &tor->info.files[fileIndex];
/* if we're starting a new piece... */
if( piecePos == 0 )
hadPiece = tr_cpPieceIsComplete( &tor->completion, pieceIndex );
/* if we're starting a new file... */
if( !filePos && (fd<0) && (fileIndex!=prevFileIndex) )
{
char * filename = tr_torrentFindFile( tor, fileIndex );
fd = filename == NULL ? -1 : tr_open_file_for_scanning( filename );
tr_free( filename );
prevFileIndex = fileIndex;
}
/* figure out how much we can read this pass */
leftInPiece = tr_torPieceCountBytes( tor, pieceIndex ) - piecePos;
leftInFile = file->length - filePos;
bytesThisPass = MIN( leftInFile, leftInPiece );
bytesThisPass = MIN( bytesThisPass, buflen );
/* read a bit */
if( fd >= 0 ) {
const ssize_t numRead = tr_pread( fd, buffer, bytesThisPass, filePos );
if( numRead > 0 ) {
bytesThisPass = (uint32_t)numRead;
SHA1_Update( &sha, buffer, bytesThisPass );
#if defined HAVE_POSIX_FADVISE && defined POSIX_FADV_DONTNEED
posix_fadvise( fd, filePos, bytesThisPass, POSIX_FADV_DONTNEED );
#endif
}
}
/* move our offsets */
leftInPiece -= bytesThisPass;
leftInFile -= bytesThisPass;
piecePos += bytesThisPass;
filePos += bytesThisPass;
/* if we're finishing a piece... */
if( leftInPiece == 0 )
{
time_t now;
tr_bool hasPiece;
uint8_t hash[SHA_DIGEST_LENGTH];
SHA1_Final( hash, &sha );
hasPiece = !memcmp( hash, tor->info.pieces[pieceIndex].hash, SHA_DIGEST_LENGTH );
if( hasPiece || hadPiece ) {
tr_torrentSetHasPiece( tor, pieceIndex, hasPiece );
changed |= hasPiece != hadPiece;
}
tr_torrentSetPieceChecked( tor, pieceIndex );
now = tr_time( );
tor->anyDate = now;
/* sleeping even just a few msec per second goes a long
* way towards reducing IO load... */
if( lastSleptAt != now ) {
lastSleptAt = now;
tr_wait_msec( MSEC_TO_SLEEP_PER_SECOND_DURING_VERIFY );
}
SHA1_Init( &sha );
++pieceIndex;
piecePos = 0;
}
/* if we're finishing a file... */
if( leftInFile == 0 )
{
if( fd >= 0 ) { tr_close_file( fd ); fd = -1; }
++fileIndex;
filePos = 0;
}
}
/* cleanup */
if( fd >= 0 )
tr_close_file( fd );
free( buffer );
/* stopwatch */
end = tr_time( );
tr_tordbg( tor, "Verification is done. It took %d seconds to verify %"PRIu64" bytes (%"PRIu64" bytes per second)",
(int)(end-begin), tor->info.totalSize,
(uint64_t)(tor->info.totalSize/(1+(end-begin))) );
return changed;
}
/*
* parse (pseudo) makefile
*
* it may have only the following form:
*
* TARGETS = xxx ...
* INTERACTIVE = yyy ...
* aaa:
* bbb: xxx ddd ...
*
* other lines are ignored.
*/
void parse_makefile(const char *path)
{
FILE *fp;
char buf[LINE_MAX]; /* FIXME: is this enough big? */
char *s, *strp, *p;
struct makenode *node;
#if defined _XOPEN_SOURCE && (_XOPEN_SOURCE - 0) >= 600
int fd;
if (getuid() == (uid_t)0)
o_flags |= O_NOATIME;
if ((fd = open(path, o_flags)) < 0) {
fprintf(stderr, "Can't open %s: %s\n", path, strerror(errno));
exit(1);
}
(void)posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED);
(void)posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
(void)posix_fadvise(fd, 0, 0, POSIX_FADV_NOREUSE);
if ((fp = fdopen(fd, "r")) == NULL)
#else
if ((fp = fopen(path, "r")) == NULL)
#endif
{
fprintf(stderr, "Can't open %s: %s\n", path, strerror(errno));
exit(1);
}
while (fgets(buf, sizeof(buf), fp)) {
for (s = buf; *s && isspace(*s); s++)
;
if (! *s || *s == '#')
continue;
if (! strncmp(s, "TARGETS =", 9)) {
s += 9;
strp = s;
while ((s = strsep(&strp, DELIMITER))) {
if (! *s)
continue;
add_target(s);
}
} else if (! strncmp(s, "INTERACTIVE =", 13)) {
s += 13;
strp = s;
while ((s = strsep(&strp, DELIMITER))) {
if (! *s)
continue;
mark_interactive(s);
}
} else {
p = strchr(s, ':');
if (! p)
continue;
*p = 0;
node = add_target(s);
strp = p + 1;
while ((s = strsep(&strp, DELIMITER))) {
if (! *s)
continue;
add_depend(node, s);
}
}
}
#if defined _XOPEN_SOURCE && (_XOPEN_SOURCE - 0) >= 600
(void)posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
#endif
fclose(fp);
for (node = tree_list; node; node = node->next) {
int importance = 0;
if (! strcmp(node->name, "xdm")
|| ! strncmp(node->name, "gdm", 3)
|| ! strncmp(node->name, "kdm", 3)
|| ! strcmp(node->name, "boot.udev")
|| ! strcmp(node->name, "udev"))
importance = 100;
if (! strcmp(node->name, "sshd"))
importance = 2000;
if (! strncmp(node->name, "early", 5))
importance = 8000;
if (importance)
add_importance(node, importance);
}
}
static void measure(int fd, struct flow *fw)
{
long delay;
fw->written_blocks++;
fw->total_written += fw->block_size;
if (fw->written_blocks < fw->blocks_per_delay)
return;
assert(!fdatasync(fd));
assert(!gettimeofday(&fw->t2, NULL));
/* Help the kernel to help us. */
assert(!posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED));
delay = delay_ms(&fw->t1, &fw->t2);
switch (fw->state) {
case FW_INC:
if (delay > fw->delay_ms) {
move_to_search(fw,
fw->blocks_per_delay - fw->step / 2,
fw->blocks_per_delay);
} else if (delay < fw->delay_ms) {
inc_step(fw);
} else
move_to_steady(fw);
break;
case FW_DEC:
if (delay > fw->delay_ms) {
dec_step(fw);
} else if (delay < fw->delay_ms) {
move_to_search(fw, fw->blocks_per_delay,
fw->blocks_per_delay + fw->step / 2);
} else
move_to_steady(fw);
break;
case FW_SEARCH:
if (fw->bpd2 - fw->bpd1 <= 3) {
move_to_steady(fw);
break;
}
if (delay > fw->delay_ms) {
fw->bpd2 = fw->blocks_per_delay;
fw->blocks_per_delay = (fw->bpd1 + fw->bpd2) / 2;
} else if (delay < fw->delay_ms) {
fw->bpd1 = fw->blocks_per_delay;
fw->blocks_per_delay = (fw->bpd1 + fw->bpd2) / 2;
} else
move_to_steady(fw);
break;
case FW_STEADY:
update_mean(fw);
if (delay <= fw->delay_ms) {
move_to_inc(fw);
}
else if (fw->blocks_per_delay > 1) {
move_to_dec(fw);
}
break;
default:
assert(0);
}
if (fw->progress) {
/* Instantaneous speed. */
double inst_speed =
(double)fw->blocks_per_delay * fw->block_size * 1000 /
fw->delay_ms;
const char *unit = adjust_unit(&inst_speed);
double percent;
/* The following shouldn't be necessary, but sometimes
* the initial free space isn't exactly reported
* by the kernel; this issue has been seen on Macs.
*/
if (fw->total_size < fw->total_written)
fw->total_size = fw->total_written;
percent = (double)fw->total_written * 100 / fw->total_size;
erase(fw->erase);
fw->erase = printf("%.2f%% -- %.2f %s/s",
percent, inst_speed, unit);
assert(fw->erase > 0);
if (fw->measurements > 0)
fw->erase += pr_time(
(fw->total_size - fw->total_written) /
get_avg_speed(fw));
fflush(stdout);
}
start_measurement(fw);
}
int pa_sound_file_load(
pa_mempool *pool,
const char *fname,
pa_sample_spec *ss,
pa_channel_map *map,
pa_memchunk *chunk,
pa_proplist *p) {
SNDFILE *sf = NULL;
SF_INFO sfi;
int ret = -1;
size_t l;
sf_count_t (*readf_function)(SNDFILE *sndfile, void *ptr, sf_count_t frames) = NULL;
void *ptr = NULL;
int fd;
pa_assert(fname);
pa_assert(ss);
pa_assert(chunk);
pa_memchunk_reset(chunk);
if ((fd = pa_open_cloexec(fname, O_RDONLY, 0)) < 0) {
pa_log("Failed to open file %s: %s", fname, pa_cstrerror(errno));
goto finish;
}
#ifdef HAVE_POSIX_FADVISE
if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL) < 0) {
pa_log_warn("POSIX_FADV_SEQUENTIAL failed: %s", pa_cstrerror(errno));
goto finish;
} else
pa_log_debug("POSIX_FADV_SEQUENTIAL succeeded.");
#endif
pa_zero(sfi);
if (!(sf = sf_open_fd(fd, SFM_READ, &sfi, 1))) {
pa_log("Failed to open file %s", fname);
goto finish;
}
fd = -1;
if (pa_sndfile_read_sample_spec(sf, ss) < 0) {
pa_log("Failed to determine file sample format.");
goto finish;
}
if ((map && pa_sndfile_read_channel_map(sf, map) < 0)) {
if (ss->channels > 2)
pa_log("Failed to determine file channel map, synthesizing one.");
pa_channel_map_init_extend(map, ss->channels, PA_CHANNEL_MAP_DEFAULT);
}
if (p)
pa_sndfile_init_proplist(sf, p);
if ((l = pa_frame_size(ss) * (size_t) sfi.frames) > PA_SCACHE_ENTRY_SIZE_MAX) {
pa_log("File too large");
goto finish;
}
chunk->memblock = pa_memblock_new(pool, l);
chunk->index = 0;
chunk->length = l;
readf_function = pa_sndfile_readf_function(ss);
ptr = pa_memblock_acquire(chunk->memblock);
if ((readf_function && readf_function(sf, ptr, sfi.frames) != sfi.frames) ||
(!readf_function && sf_read_raw(sf, ptr, (sf_count_t) l) != (sf_count_t) l)) {
pa_log("Premature file end");
goto finish;
}
ret = 0;
finish:
if (sf)
sf_close(sf);
if (ptr)
pa_memblock_release(chunk->memblock);
if (ret != 0 && chunk->memblock)
pa_memblock_unref(chunk->memblock);
if (fd >= 0)
pa_close(fd);
return ret;
}
/*
* Map a cache file into memory
*/
static FcCache *
FcDirCacheMapFd (FcConfig *config, int fd, struct stat *fd_stat, struct stat *dir_stat)
{
FcCache *cache;
FcBool allocated = FcFalse;
if (fd_stat->st_size < (int) sizeof (FcCache))
return NULL;
cache = FcCacheFindByStat (fd_stat);
if (cache)
{
if (FcCacheTimeValid (config, cache, dir_stat) &&
FcCacheDirsValid (config, cache))
return cache;
FcDirCacheUnload (cache);
cache = NULL;
}
/*
* Large cache files are mmap'ed, smaller cache files are read. This
* balances the system cost of mmap against per-process memory usage.
*/
if (FcCacheIsMmapSafe (fd) && fd_stat->st_size >= FC_CACHE_MIN_MMAP)
{
#if defined(HAVE_MMAP) || defined(__CYGWIN__)
cache = mmap (0, fd_stat->st_size, PROT_READ, MAP_SHARED, fd, 0);
#if (HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
posix_fadvise (fd, 0, fd_stat->st_size, POSIX_FADV_WILLNEED);
#endif
if (cache == MAP_FAILED)
cache = NULL;
#elif defined(_WIN32)
{
HANDLE hFileMap;
cache = NULL;
hFileMap = CreateFileMapping((HANDLE) _get_osfhandle(fd), NULL,
PAGE_READONLY, 0, 0, NULL);
if (hFileMap != NULL)
{
cache = MapViewOfFile (hFileMap, FILE_MAP_READ, 0, 0,
fd_stat->st_size);
CloseHandle (hFileMap);
}
}
#endif
}
if (!cache)
{
cache = malloc (fd_stat->st_size);
if (!cache)
return NULL;
if (read (fd, cache, fd_stat->st_size) != fd_stat->st_size)
{
free (cache);
return NULL;
}
allocated = FcTrue;
}
if (cache->magic != FC_CACHE_MAGIC_MMAP ||
cache->version < FC_CACHE_VERSION_NUMBER ||
cache->size != (intptr_t) fd_stat->st_size ||
!FcCacheTimeValid (config, cache, dir_stat) ||
!FcCacheDirsValid (config, cache) ||
!FcCacheInsert (cache, fd_stat))
{
if (allocated)
free (cache);
else
{
#if defined(HAVE_MMAP) || defined(__CYGWIN__)
munmap (cache, fd_stat->st_size);
#elif defined(_WIN32)
UnmapViewOfFile (cache);
#endif
}
return NULL;
}
/* Mark allocated caches so they're freed rather than unmapped */
if (allocated)
cache->magic = FC_CACHE_MAGIC_ALLOC;
return cache;
}
/*
* __wt_block_write_off --
* Write a buffer into a block, returning the block's addr/size and
* checksum.
*/
int
__wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
WT_ITEM *buf, off_t *offsetp, uint32_t *sizep, uint32_t *cksump,
int data_cksum, int locked)
{
WT_BLOCK_HEADER *blk;
WT_DECL_RET;
WT_FH *fh;
off_t offset;
uint32_t align_size;
blk = WT_BLOCK_HEADER_REF(buf->mem);
fh = block->fh;
/* Buffers should be aligned for writing. */
if (!F_ISSET(buf, WT_ITEM_ALIGNED)) {
WT_ASSERT(session, F_ISSET(buf, WT_ITEM_ALIGNED));
WT_RET_MSG(session, EINVAL,
"direct I/O check: write buffer incorrectly allocated");
}
/*
* Align the size to an allocation unit.
*
* The buffer must be big enough for us to zero to the next allocsize
* boundary, this is one of the reasons the btree layer must find out
* from the block-manager layer the maximum size of the eventual write.
*/
align_size = (uint32_t)WT_ALIGN(buf->size, block->allocsize);
if (align_size > buf->memsize) {
WT_ASSERT(session, align_size <= buf->memsize);
WT_RET_MSG(session, EINVAL,
"buffer size check: write buffer incorrectly allocated");
}
/* Zero out any unused bytes at the end of the buffer. */
memset((uint8_t *)buf->mem + buf->size, 0, align_size - buf->size);
/*
* Set the disk size so we don't have to incrementally read blocks
* during salvage.
*/
blk->disk_size = align_size;
/*
* Update the block's checksum: if our caller specifies, checksum the
* complete data, otherwise checksum the leading WT_BLOCK_COMPRESS_SKIP
* bytes. The assumption is applications with good compression support
* turn off checksums and assume corrupted blocks won't decompress
* correctly. However, if compression failed to shrink the block, the
* block wasn't compressed, in which case our caller will tell us to
* checksum the data to detect corruption. If compression succeeded,
* we still need to checksum the first WT_BLOCK_COMPRESS_SKIP bytes
* because they're not compressed, both to give salvage a quick test
* of whether a block is useful and to give us a test so we don't lose
* the first WT_BLOCK_COMPRESS_SKIP bytes without noticing.
*/
blk->flags = 0;
if (data_cksum)
F_SET(blk, WT_BLOCK_DATA_CKSUM);
blk->cksum = 0;
blk->cksum = __wt_cksum(
buf->mem, data_cksum ? align_size : WT_BLOCK_COMPRESS_SKIP);
if (!locked)
__wt_spin_lock(session, &block->live_lock);
ret = __wt_block_alloc(session, block, &offset, (off_t)align_size);
if (!locked)
__wt_spin_unlock(session, &block->live_lock);
WT_RET(ret);
#if defined(HAVE_POSIX_FALLOCATE) || defined(HAVE_FTRUNCATE)
/*
* Extend the file in chunks. We aren't holding a lock and we'd prefer
* to limit the number of threads extending the file at the same time,
* so choose the one thread that's crossing the extended boundary. We
* don't extend newly created files, and it's theoretically possible we
* might wait so long our extension of the file is passed by another
* thread writing single blocks, that's why there's a check in case the
* extended file size becomes too small: if the file size catches up,
* every thread will try to extend it.
*/
if (fh->extend_len != 0 &&
(fh->extend_size <= fh->size ||
(offset + fh->extend_len <= fh->extend_size &&
offset + fh->extend_len + align_size >= fh->extend_size))) {
fh->extend_size = offset + fh->extend_len * 2;
#if defined(HAVE_POSIX_FALLOCATE)
if ((ret =
posix_fallocate(fh->fd, offset, fh->extend_len * 2)) != 0)
WT_RET_MSG(
session, ret, "%s: posix_fallocate", fh->name);
#elif defined(HAVE_FTRUNCATE)
if ((ret = ftruncate(fh->fd, fh->extend_size)) != 0)
WT_RET_MSG(session, ret, "%s: ftruncate", fh->name);
#endif
}
#endif
if ((ret =
__wt_write(session, fh, offset, align_size, buf->mem)) != 0) {
if (!locked)
__wt_spin_lock(session, &block->live_lock);
WT_TRET(
__wt_block_off_free(session, block, offset, align_size));
if (!locked)
__wt_spin_unlock(session, &block->live_lock);
WT_RET(ret);
}
#ifdef HAVE_SYNC_FILE_RANGE
/*
* Optionally schedule writes for dirty pages in the system buffer
* cache.
*/
if (block->os_cache_dirty_max != 0 &&
(block->os_cache_dirty += align_size) > block->os_cache_dirty_max) {
block->os_cache_dirty = 0;
if ((ret = sync_file_range(fh->fd,
(off64_t)0, (off64_t)0, SYNC_FILE_RANGE_WRITE)) != 0)
WT_RET_MSG(
session, ret, "%s: sync_file_range", block->name);
}
#endif
#ifdef HAVE_POSIX_FADVISE
/* Optionally discard blocks from the system buffer cache. */
if (block->os_cache_max != 0 &&
(block->os_cache += align_size) > block->os_cache_max) {
block->os_cache = 0;
if ((ret = posix_fadvise(fh->fd,
(off_t)0, (off_t)0, POSIX_FADV_DONTNEED)) != 0)
WT_RET_MSG(
session, ret, "%s: posix_fadvise", block->name);
}
#endif
WT_CSTAT_INCR(session, block_write);
WT_CSTAT_INCRV(session, block_byte_write, align_size);
WT_VERBOSE_RET(session, write,
"off %" PRIuMAX ", size %" PRIu32 ", cksum %" PRIu32,
(uintmax_t)offset, align_size, blk->cksum);
*offsetp = offset;
*sizep = align_size;
*cksump = blk->cksum;
return (ret);
}
static void
help_build_index(help_file *h, int restricted)
{
long bigpos, pos = 0;
bool in_topic;
int i, lineno, ntopics;
size_t n;
char *s, *topic;
char the_topic[TOPIC_NAME_LEN + 1];
char line[LINE_SIZE + 1];
FILE *rfp;
tlist *cur;
/* Quietly ignore null values for the file */
if (!h || !h->file)
return;
if ((rfp = fopen(h->file, FOPEN_READ)) == NULL) {
do_rawlog(LT_ERR, "Can't open %s for reading: %s", h->file,
strerror(errno));
return;
}
if (restricted)
do_rawlog(LT_WIZ, "Indexing file %s (admin topics)", h->file);
else
do_rawlog(LT_WIZ, "Indexing file %s", h->file);
topics = NULL;
num_topics = 0;
top_topics = 0;
bigpos = 0L;
lineno = 0;
ntopics = 0;
in_topic = 0;
#ifdef HAVE_POSIX_FADVISE
posix_fadvise(fileno(rfp), 0, 0, POSIX_FADV_SEQUENTIAL);
#endif
while (fgets(line, LINE_SIZE, rfp) != NULL) {
++lineno;
if (ntopics == 0) {
/* Looking for the first topic, but we'll ignore blank lines */
if (!line[0]) {
/* Someone's feeding us /dev/null? */
do_rawlog(LT_ERR, "Malformed help file %s doesn't start with &",
h->file);
fclose(rfp);
return;
}
if (isspace(line[0]))
continue;
if (line[0] != '&') {
do_rawlog(LT_ERR, "Malformed help file %s doesn't start with &",
h->file);
fclose(rfp);
return;
}
}
n = strlen(line);
if (line[n - 1] != '\n') {
do_rawlog(LT_ERR, "Line %d of %s: line too long", lineno, h->file);
}
if (line[0] == '&') {
++ntopics;
if (!in_topic) {
/* Finish up last entry */
if (ntopics > 1) {
write_topic(pos);
}
in_topic = true;
}
/* parse out the topic */
/* Get the beginning of the topic string */
for (topic = &line[1];
(*topic == ' ' || *topic == '\t') && *topic != '\0'; topic++) ;
/* Get the topic */
strcpy(the_topic, "");
for (i = -1, s = topic; *s != '\n' && *s != '\0'; s++) {
if (i >= TOPIC_NAME_LEN - 1)
break;
if (*s != ' ' || the_topic[i] != ' ')
the_topic[++i] = *s;
}
if ((restricted && the_topic[0] == '&')
|| (!restricted && the_topic[0] != '&')) {
the_topic[++i] = '\0';
cur = (tlist *) malloc(sizeof(tlist));
strcpy(cur->topic, the_topic);
cur->next = top;
top = cur;
}
} else {
if (in_topic) {
pos = bigpos;
}
in_topic = false;
}
bigpos = ftell(rfp);
}
/* Handle last topic */
write_topic(pos);
if (topics)
qsort(topics, num_topics, sizeof(help_indx), topic_cmp);
h->entries = num_topics;
h->indx = topics;
add_check("help_index");
fclose(rfp);
do_rawlog(LT_WIZ, "%d topics indexed.", num_topics);
return;
}
/*
* __posix_open_file --
* Open a file handle.
*/
static int
__posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session,
const char *name, WT_FS_OPEN_FILE_TYPE file_type, uint32_t flags,
WT_FILE_HANDLE **file_handlep)
{
WT_CONNECTION_IMPL *conn;
WT_DECL_RET;
WT_FILE_HANDLE *file_handle;
WT_FILE_HANDLE_POSIX *pfh;
WT_SESSION_IMPL *session;
mode_t mode;
int f;
WT_UNUSED(file_system);
*file_handlep = NULL;
session = (WT_SESSION_IMPL *)wt_session;
conn = S2C(session);
WT_RET(__wt_calloc_one(session, &pfh));
/* Set up error handling. */
pfh->fd = -1;
if (file_type == WT_FS_OPEN_FILE_TYPE_DIRECTORY) {
f = O_RDONLY;
#ifdef O_CLOEXEC
/*
* Security:
* The application may spawn a new process, and we don't want
* another process to have access to our file handles.
*/
f |= O_CLOEXEC;
#endif
WT_SYSCALL_RETRY((
(pfh->fd = open(name, f, 0444)) == -1 ? -1 : 0), ret);
if (ret != 0)
WT_ERR_MSG(session, ret, "%s: handle-open: open", name);
WT_ERR(__posix_open_file_cloexec(session, pfh->fd, name));
goto directory_open;
}
f = LF_ISSET(WT_FS_OPEN_READONLY) ? O_RDONLY : O_RDWR;
if (LF_ISSET(WT_FS_OPEN_CREATE)) {
f |= O_CREAT;
if (LF_ISSET(WT_FS_OPEN_EXCLUSIVE))
f |= O_EXCL;
mode = 0666;
} else
mode = 0;
#ifdef O_BINARY
/* Windows clones: we always want to treat the file as a binary. */
f |= O_BINARY;
#endif
#ifdef O_CLOEXEC
/*
* Security:
* The application may spawn a new process, and we don't want another
* process to have access to our file handles.
*/
f |= O_CLOEXEC;
#endif
#ifdef O_DIRECT
/* Direct I/O. */
if (LF_ISSET(WT_FS_OPEN_DIRECTIO)) {
f |= O_DIRECT;
pfh->direct_io = true;
} else
pfh->direct_io = false;
#endif
#ifdef O_NOATIME
/* Avoid updating metadata for read-only workloads. */
if (file_type == WT_FS_OPEN_FILE_TYPE_DATA)
f |= O_NOATIME;
#endif
if (file_type == WT_FS_OPEN_FILE_TYPE_LOG &&
FLD_ISSET(conn->txn_logsync, WT_LOG_DSYNC)) {
#ifdef O_DSYNC
f |= O_DSYNC;
#elif defined(O_SYNC)
f |= O_SYNC;
#else
WT_ERR_MSG(session, ENOTSUP,
"unsupported log sync mode configured");
#endif
}
/* Create/Open the file. */
WT_SYSCALL_RETRY(((pfh->fd = open(name, f, mode)) == -1 ? -1 : 0), ret);
if (ret != 0)
WT_ERR_MSG(session, ret,
pfh->direct_io ?
"%s: handle-open: open: failed with direct I/O configured, "
"some filesystem types do not support direct I/O" :
"%s: handle-open: open", name);
#ifdef __linux__
/*
* Durability: some filesystems require a directory sync to be confident
* the file will appear.
*/
if (LF_ISSET(WT_FS_OPEN_DURABLE))
WT_ERR(__posix_directory_sync(session, name));
#endif
WT_ERR(__posix_open_file_cloexec(session, pfh->fd, name));
#if defined(HAVE_POSIX_FADVISE)
/*
* Disable read-ahead on trees: it slows down random read workloads.
* Ignore fadvise when doing direct I/O, the kernel cache isn't
* interesting.
*/
if (!pfh->direct_io && file_type == WT_FS_OPEN_FILE_TYPE_DATA) {
WT_SYSCALL(
posix_fadvise(pfh->fd, 0, 0, POSIX_FADV_RANDOM), ret);
if (ret != 0)
WT_ERR_MSG(session, ret,
"%s: handle-open: posix_fadvise", name);
}
#endif
directory_open:
/* Initialize public information. */
file_handle = (WT_FILE_HANDLE *)pfh;
WT_ERR(__wt_strdup(session, name, &file_handle->name));
file_handle->close = __posix_file_close;
#if defined(HAVE_POSIX_FADVISE)
/*
* Ignore fadvise when doing direct I/O, the kernel cache isn't
* interesting.
*/
if (!pfh->direct_io)
file_handle->fh_advise = __posix_file_advise;
#endif
file_handle->fh_allocate = __wt_posix_file_fallocate;
file_handle->fh_lock = __posix_file_lock;
#ifdef WORDS_BIGENDIAN
/*
* The underlying objects are little-endian, mapping objects isn't
* currently supported on big-endian systems.
*/
#else
file_handle->fh_map = __wt_posix_map;
#ifdef HAVE_POSIX_MADVISE
file_handle->fh_map_discard = __wt_posix_map_discard;
file_handle->fh_map_preload = __wt_posix_map_preload;
#endif
file_handle->fh_unmap = __wt_posix_unmap;
#endif
file_handle->fh_read = __posix_file_read;
file_handle->fh_size = __posix_file_size;
file_handle->fh_sync = __posix_file_sync;
#ifdef HAVE_SYNC_FILE_RANGE
file_handle->fh_sync_nowait = __posix_file_sync_nowait;
#endif
file_handle->fh_truncate = __posix_file_truncate;
file_handle->fh_write = __posix_file_write;
*file_handlep = file_handle;
return (0);
err: WT_TRET(__posix_file_close((WT_FILE_HANDLE *)pfh, wt_session));
return (ret);
}
/* given a filename, its meta data, its list of segments, and list of destination containers,
* copy file to container files */
static int scr_flush_file_to_containers(
const char* file,
scr_meta* meta,
scr_hash* segments,
const char* dst_dir)
{
/* check that we got something for a source file */
if (file == NULL || strcmp(file, "") == 0) {
scr_err("Invalid source file @ %s:%d",
__FILE__, __LINE__
);
return SCR_FAILURE;
}
/* check that our other arguments are valid */
if (meta == NULL || segments == NULL) {
scr_err("Invalid metadata or segments @ %s:%d",
__FILE__, __LINE__
);
return SCR_FAILURE;
}
/* open the file for reading */
int fd_src = scr_open(file, O_RDONLY);
if (fd_src < 0) {
scr_err("Opening file to copy: scr_open(%s) errno=%d %s @ %s:%d",
file, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
#if !defined(__APPLE__)
/* TODO:
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL)
that tells the kernel that you don't ever need the pages
from the file again, and it won't bother keeping them in the page cache.
*/
posix_fadvise(fd_src, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL);
#endif
/* get the buffer size we'll use to write to the file */
unsigned long buf_size = scr_file_buf_size;
/* allocate buffer to read in file chunks */
char* buf = (char*) SCR_MALLOC(buf_size);
/* initialize crc value */
uLong crc;
if (scr_crc_on_flush) {
crc = crc32(0L, Z_NULL, 0);
}
int rc = SCR_SUCCESS;
/* write out each segment */
scr_hash_sort_int(segments, SCR_HASH_SORT_ASCENDING);
scr_hash_elem* elem;
for (elem = scr_hash_elem_first(segments);
elem != NULL;
elem = scr_hash_elem_next(elem))
{
/* get the container info for this segment */
scr_hash* hash = scr_hash_elem_hash(elem);
/* get the offset into the container and the length of the segment (both in bytes) */
char* container_name;
unsigned long container_offset, segment_length;
if (scr_container_get_name_offset_length(hash,
&container_name, &container_offset, &segment_length) != SCR_SUCCESS)
{
scr_err("Failed to get segment offset and length @ %s:%d",
__FILE__, __LINE__
);
rc = SCR_FAILURE;
break;
}
/* build full name to destination file */
scr_path* dst_path = scr_path_from_str(dst_dir);
scr_path_append_str(dst_path, container_name);
scr_path_reduce(dst_path);
char* dst_file = scr_path_strdup(dst_path);
/* open container file for writing -- we don't truncate here because more than one
* process may be writing to the same file */
int fd_container = scr_open(dst_file, O_WRONLY);
if (fd_container < 0) {
scr_err("Opening file for writing: scr_open(%s) errno=%d %s @ %s:%d",
dst_file, errno, strerror(errno), __FILE__, __LINE__
);
rc = SCR_FAILURE;
break;
}
#if !defined(__APPLE__)
/* TODO:
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL)
that tells the kernel that you don't ever need the pages
from the file again, and it won't bother keeping them in the page cache.
*/
posix_fadvise(fd_container, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL);
#endif
/* seek to offset within container */
off_t pos = (off_t) container_offset;
if (lseek(fd_container, pos, SEEK_SET) == (off_t)-1) {
/* our seek failed, return an error */
scr_err("Failed to seek to byte %lu in %s @ %s:%d",
pos, dst_file, __FILE__, __LINE__
);
rc = SCR_FAILURE;
break;
}
/* copy data from file into container in chunks */
unsigned long remaining = segment_length;
while (remaining > 0) {
/* read / write up to buf_size bytes at a time from file */
unsigned long count = remaining;
if (count > buf_size) {
count = buf_size;
}
/* attempt to read buf_size bytes from file */
int nread = scr_read_attempt(file, fd_src, buf, count);
/* if we read some bytes, write them out */
if (nread > 0) {
/* optionally compute crc value as we go */
if (scr_crc_on_flush) {
crc = crc32(crc, (const Bytef*) buf, (uInt) nread);
}
/* write our nread bytes out */
int nwrite = scr_write_attempt(dst_file, fd_container, buf, nread);
/* check for a write error or a short write */
if (nwrite != nread) {
/* write had a problem, stop copying and return an error */
rc = SCR_FAILURE;
break;
}
/* subtract the bytes we've processed from the number remaining */
remaining -= (unsigned long) nread;
}
/* assume a short read is an error */
if (nread < count) {
/* read had a problem, stop copying and return an error */
rc = SCR_FAILURE;
break;
}
/* check for a read error, stop copying and return an error */
if (nread < 0) {
/* read had a problem, stop copying and return an error */
rc = SCR_FAILURE;
break;
}
}
/* close container */
if (scr_close(dst_file, fd_container) != SCR_SUCCESS) {
rc = SCR_FAILURE;
}
/* free the container file name and path */
scr_free(&dst_file);
scr_path_delete(&dst_path);
}
/* close the source file */
if (scr_close(file, fd_src) != SCR_SUCCESS) {
rc = SCR_FAILURE;
}
/* free buffer */
scr_free(&buf);
/* verify / set crc value */
if (rc == SCR_SUCCESS) {
uLong crc2;
if (scr_crc_on_flush) {
if (scr_meta_get_crc32(meta, &crc2) == SCR_SUCCESS) {
/* if a crc is already set in the meta data, check that we computed the same value */
if (crc != crc2) {
scr_err("CRC32 mismatch detected when flushing file %s @ %s:%d",
file, __FILE__, __LINE__
);
rc = SCR_FAILURE;
}
} else {
/* if there is no crc set, let's set it now */
scr_meta_set_crc32(meta, crc);
}
}
}
return rc;
}
bool rotate() {
if ( ! _enabled ) {
cout << "LoggingManager not enabled" << endl;
return true;
}
if ( _file ) {
#ifdef POSIX_FADV_DONTNEED
posix_fadvise(fileno(_file), 0, 0, POSIX_FADV_DONTNEED);
#endif
// Rename the (open) existing log file to a timestamped name
stringstream ss;
ss << _path << "." << terseCurrentTime( false );
string s = ss.str();
if (0 != rename(_path.c_str(), s.c_str())) {
error() << "Failed to rename " << _path << " to " << s;
return false;
}
}
FILE* tmp = 0; // The new file using the original logpath name
#if _WIN32
// We rename an open log file (above, on next rotation) and the trick to getting Windows to do that is
// to open the file with FILE_SHARE_DELETE. So, we can't use the freopen() call that non-Windows
// versions use because it would open the file without the FILE_SHARE_DELETE flag we need.
//
HANDLE newFileHandle = CreateFileA(
_path.c_str(),
GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
NULL,
OPEN_ALWAYS,
FILE_ATTRIBUTE_NORMAL,
NULL
);
if ( INVALID_HANDLE_VALUE != newFileHandle ) {
int newFileDescriptor = _open_osfhandle( reinterpret_cast<intptr_t>(newFileHandle), _O_APPEND );
tmp = _fdopen( newFileDescriptor, _append ? "a" : "w" );
}
#else
tmp = freopen(_path.c_str(), _append ? "a" : "w", stdout);
#endif
if ( !tmp ) {
cerr << "can't open: " << _path.c_str() << " for log file" << endl;
return false;
}
// redirect stdout and stderr to log file
dup2( fileno( tmp ), 1 ); // stdout
dup2( fileno( tmp ), 2 ); // stderr
Logstream::setLogFile(tmp); // after this point no thread will be using old file
#if _WIN32
if ( _file )
fclose( _file ); // In Windows, we still have the old file open, close it now
#endif
#if 0 // enable to test redirection
cout << "written to cout" << endl;
cerr << "written to cerr" << endl;
log() << "written to log()" << endl;
#endif
_file = tmp; // Save new file for next rotation
return true;
}
int main(int argc, char ** argv)
{
ssize_t bytes_read = 0, bytes_read_pread = 0;
int fd1 = 0, fd2 = 0;
char data[READSIZE];
char data_pread[READSIZE];
struct aiocb aio;
int i;
struct timespec time1, time2;
off_t offset;
off_t len1, len2;
struct timeval start, end, result, totaltime, tmp;
if ( argc != 3 ) {
printf( "Usage: %s file-1-path file-2-path", argv[0] );
exit(1);
}
fd1 = open(argv[1], O_RDONLY);
if (fd1 == -1) {
perror("open");
return EXIT_FAILURE;
}
fd2 = open(argv[2], O_RDONLY);
if (fd2 == -1) {
perror("open");
return EXIT_FAILURE;
}
/* ************************************************ */
memset(&aio, 0, sizeof(aio));
memset(&data, 0, sizeof(data));
memset(&data_pread, 0, sizeof(data_pread));
aio.aio_fildes = fd1;
aio.aio_buf = data;
aio.aio_nbytes = READSIZE;
len1 = lseek(fd1, 0, SEEK_END);
assert(len1 >= 0);
timerclear(&totaltime);
srand(5);
//clock_gettime(TIMING_METHOD, &time1);
gettimeofday(&start, NULL);
for ( i = 0 ; i < REPTIMES ; i++ ) {
offset = len1 * (rand() / (double)RAND_MAX);
//printf("%ld ", aio.aio_offset);
posix_fadvise( fd1, offset, READSIZE, POSIX_FADV_WILLNEED);
}
gettimeofday(&end, NULL);
timersub( &end, &start, &result );
printf("posix_fadvise time (REPTIMES:%d): %ld.%.6ld\n",
REPTIMES, result.tv_sec, result.tv_usec);
sleep(20);
srand(5);
//clock_gettime(TIMING_METHOD, &time1);
gettimeofday(&start, NULL);
for ( i = 0 ; i < REPTIMES ; i++ ) {
offset = len1 * (rand() / (double)RAND_MAX);
bytes_read_pread += pread( fd1, data_pread, READSIZE, offset );
}
gettimeofday(&end, NULL);
timersub( &end, &start, &result );
printf("pread() with caching time (REPTIMES:%d): %ld.%.6ld\n",
REPTIMES, result.tv_sec, result.tv_usec);
printf("%d bytes read by pread()\n", bytes_read_pread);
close(fd1);
/* ************************************************ */
len2 = lseek(fd2, 0, SEEK_END);
assert(len2 >= 0);
timerclear(&totaltime);
srand(5);
bytes_read = 0;
gettimeofday(&start, NULL);
for ( i = 0 ; i < REPTIMES ; i++ ) {
offset = len2 * (rand() / (double)RAND_MAX);
bytes_read += pread(fd2, data, READSIZE, offset);
}
gettimeofday(&end, NULL);
timersub( &end, &start, &result );
printf("pread() without caching time (REPTIMES:%d): %ld.%.6ld\n",
REPTIMES, result.tv_sec, result.tv_usec);
printf("%d bytes read by pread().\n", bytes_read);
close(fd2);
return EXIT_SUCCESS;
}
void Logstream::flush(Tee *t) {
const size_t MAX_LOG_LINE = 1024 * 10;
// this ensures things are sane
if ( doneSetup == 1717 ) {
string msg = ss.str();
string threadName = getThreadName();
const char * type = logLevelToString(logLevel);
size_t msgLen = msg.size();
if ( msgLen > MAX_LOG_LINE )
msgLen = MAX_LOG_LINE;
const int spaceNeeded = (int)( msgLen + 64 /* for extra info */ + threadName.size());
int bufSize = 128;
while ( bufSize < spaceNeeded )
bufSize += 128;
BufBuilder b(bufSize);
char* dateStr = b.grow(24);
curTimeString(dateStr);
dateStr[23] = ' '; // change null char to space
if (!threadName.empty()) {
b.appendChar( '[' );
b.appendStr( threadName , false );
b.appendChar( ']' );
b.appendChar( ' ' );
}
for ( int i=0; i<indent; i++ )
b.appendChar( '\t' );
if ( type[0] ) {
b.appendStr( type , false );
b.appendStr( ": " , false );
}
if ( msg.size() > MAX_LOG_LINE ) {
stringstream sss;
sss << "warning: log line attempted (" << msg.size() / 1024 << "k) over max size(" << MAX_LOG_LINE / 1024 << "k)";
sss << ", printing beginning and end ... ";
b.appendStr( sss.str(), false );
const char * xx = msg.c_str();
b.appendBuf( xx , MAX_LOG_LINE / 3 );
b.appendStr( " .......... ", false );
b.appendStr( xx + msg.size() - ( MAX_LOG_LINE / 3 ) );
}
else {
b.appendStr( msg );
}
string out( b.buf() , b.len() - 1);
verify( b.len() < spaceNeeded );
scoped_lock lk(mutex);
if( t ) t->write(logLevel,out);
if ( globalTees ) {
for ( unsigned i=0; i<globalTees->size(); i++ )
(*globalTees)[i]->write(logLevel,out);
}
#if defined(_WIN32)
int fd = fileno( logfile );
if ( _isatty( fd ) ) {
fflush( logfile );
writeUtf8ToWindowsConsole( out.data(), out.size() );
}
#else
if ( isSyslog ) {
syslog( logLevelToSysLogLevel(logLevel) , "%s" , out.data() );
}
#endif
else if ( fwrite( out.data(), out.size(), 1, logfile ) ) {
fflush(logfile);
}
else {
int x = errno;
cout << "Failed to write to logfile: " << errnoWithDescription(x) << ": " << out << endl;
}
#ifdef POSIX_FADV_DONTNEED
// This only applies to pages that have already been flushed
RARELY posix_fadvise(fileno(logfile), 0, 0, POSIX_FADV_DONTNEED);
#endif
}
_init();
}
/**
* Segment prefetching routine
* @param ptr Queue that holds either segment or bucket number
*/
void * prefetch(void * ptr) {
struct timeval a,b,c;
gettimeofday(&a,NULL);
Queue * q = (Queue *) ptr;
uint64_t sid, lsid = 0, bid = 0;
uint32_t pos, len;
char buf[128];
while ((sid = (uint64_t) Dequeue(q)) != 0) {
#ifdef PREFETCH_WHOLE_BUCKET
bid = sen[sid].bucket;
sprintf(buf, DATA_DIR "bucket/%08lx", bid);
int fd = open(buf, O_RDONLY);
posix_fadvise(fd, sen[sid].pos, sen[sid].len, POSIX_FADV_WILLNEED);
close(fd);
}
#else
if (lsid == sid) {
continue;
}
lsid = sid;
if (bid == 0) {
bid = sen[sid].bucket;
pos = sen[sid].pos;
len = sen[sid].len;
continue;
}
if (sen[sid].bucket == bid && pos + len == sen[sid].pos) {
len += sen[sid].len;
continue;
}
sprintf(buf, DATA_DIR "bucket/%08lx", bid);
int fd = open(buf, O_RDONLY);
posix_fadvise(fd, pos, len, POSIX_FADV_WILLNEED);
//posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
seg_seeks++;
if (bid != last_bid){
b1_seeks++;
if(bid != llast_bid){
b2_seeks++;
llast_bid = last_bid;
last_bid = bid;
}
//fprintf(stdout,"BID: %d, size: %ld\n",last_bid,tmp_len);
}
/*
if(bcache == NULL){
BucketCache* tmp = (BucketCache*)malloc(sizeof(BucketCache));
tmp->bid = bid;
tmp->next=NULL;
bcache = tmp;
} else {
BucketCache* tmp;
int unique = 1;
for(tmp=bcache;tmp!=NULL;tmp=tmp->next){
if(tmp->bid == bid){
unique=0;
break;
}
}
if(unique){
tmp = (BucketCache*)malloc(sizeof(BucketCache));
tmp->bid = bid;
tmp->next = bcache->next;
bcache->next = tmp;
inf_cache_seeks++;
}
}
*/
close(fd);
bid = sen[sid].bucket;
pos = sen[sid].pos;
len = sen[sid].len;
}
int main (int argc, char **argv)
{
ssize_t ret_size;
struct stat st;
int ret, flags;
long long this_time, time_total;
long long part_min, part_max, time_min, time_max;
double time_sum, time_sum2, time_mdev, time_avg;
double part_sum, part_sum2, part_mdev, part_avg;
parse_options(argc, argv);
if (wsize)
temp_wsize = wsize;
else if (size > temp_wsize)
temp_wsize = size;
if (size <= 0)
errx(1, "request size must be greather than zero");
flags = O_RDONLY;
#if !defined(HAVE_POSIX_FADVICE) && !defined(HAVE_NOCACHE_IO) && \
defined(HAVE_DIRECT_IO)
direct |= !cached;
#endif
if (direct)
#ifdef HAVE_DIRECT_IO
flags |= O_DIRECT;
#else
errx(1, "direct I/O not supportted by this platform");
#endif
if (stat(path, &st))
err(2, "stat \"%s\" failed", path);
if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) {
if (S_ISDIR(st.st_mode))
st.st_size = offset + temp_wsize;
parse_device(st.st_dev);
} else if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)) {
fd = open(path, flags);
if (fd < 0)
err(2, "failed to open \"%s\"", path);
st.st_size = get_device_size(fd, &st);
fstype = "device";
device = malloc(32);
if (!device)
err(2, "no mem");
snprintf(device, 32, "%.1f Gb", (double)st.st_size/(1ll<<30));
} else {
errx(2, "unsupported destination: \"%s\"", path);
}
if (offset + wsize > st.st_size)
errx(2, "target is too small for this");
if (!wsize)
wsize = st.st_size - offset;
if (size > wsize)
errx(2, "request size is too big for this target");
ret = posix_memalign(&buf, sysconf(_SC_PAGE_SIZE), size);
if (ret)
errx(2, "buffer allocation failed");
memset(buf, '*', size);
if (S_ISDIR(st.st_mode)) {
char *tmpl = "/ioping.XXXXXX";
char *temp = malloc(strlen(path) + strlen(tmpl) + 1);
if (!temp)
err(2, NULL);
sprintf(temp, "%s%s", path, tmpl);
fd = mkstemp(temp);
if (fd < 0)
err(2, "failed to create temporary file at \"%s\"", path);
if (unlink(temp))
err(2, "unlink \"%s\" failed", temp);
if (fcntl(fd, F_SETFL, flags)) {
warn("fcntl failed");
if (direct)
errx(2, "please retry without -D");
errx(2, "it is so sad");
}
for (woffset = 0 ; woffset + size <= wsize ; woffset += size) {
if (pwrite(fd, buf, size, offset + woffset) != size)
err(2, "write failed");
}
if (fsync(fd))
err(2, "fsync failed");
free(temp);
} else if (S_ISREG(st.st_mode)) {
fd = open(path, flags);
if (fd < 0)
err(2, "failed to open \"%s\"", path);
}
if (!cached) {
#ifdef HAVE_POSIX_FADVICE
ret = posix_fadvise(fd, offset, wsize, POSIX_FADV_RANDOM);
if (ret)
err(2, "fadvise failed");
#endif
#ifdef HAVE_NOCACHE_IO
ret = fcntl(fd, F_NOCACHE, 1);
if (ret)
err(2, "fcntl nocache failed");
#endif
}
srandom(now());
if (deadline)
deadline += now();
set_signal(SIGINT, sig_exit);
request = 0;
woffset = 0;
part_min = time_min = LLONG_MAX;
part_max = time_max = LLONG_MIN;
part_sum = time_sum = 0;
part_sum2 = time_sum2 = 0;
time_total = now();
while (!exiting) {
request++;
if (randomize)
woffset = random() % (wsize / size) * size;
#ifdef HAVE_POSIX_FADVICE
if (!cached) {
ret = posix_fadvise(fd, offset + woffset, size,
POSIX_FADV_DONTNEED);
if (ret)
err(3, "fadvise failed");
}
#endif
this_time = now();
ret_size = pread(fd, buf, size, offset + woffset);
if (ret_size < 0 && errno != EINTR)
err(3, "read failed");
this_time = now() - this_time;
part_sum += this_time;
part_sum2 += this_time * this_time;
if (this_time < part_min)
part_min = this_time;
if (this_time > part_max)
part_max = this_time;
if (!quiet)
printf("%lld bytes from %s (%s %s): request=%d time=%.1f ms\n",
(long long)ret_size, path, fstype, device,
request, this_time / 1000.);
if (period && request % period == 0) {
part_avg = part_sum / period;
part_mdev = sqrt(part_sum2 / period - part_avg * part_avg);
printf("%lld %.0f %lld %.0f\n",
part_min, part_avg,
part_max, part_mdev);
time_sum += part_sum;
time_sum2 += part_sum2;
if (part_min < time_min)
time_min = part_min;
if (part_max > time_max)
time_max = part_max;
part_min = LLONG_MAX;
part_max = LLONG_MIN;
part_sum = part_sum2 = 0;
}
if (!randomize) {
woffset += size;
if (woffset + size > wsize)
woffset = 0;
}
if (exiting)
break;
if (count && request >= count)
break;
if (deadline && now() + interval >= deadline)
break;
usleep(interval);
}
time_total = now() - time_total;
time_sum += part_sum;
time_sum2 += part_sum2;
if (part_min < time_min)
time_min = part_min;
if (part_max > time_max)
time_max = part_max;
time_avg = time_sum / request;
time_mdev = sqrt(time_sum2 / request - time_avg * time_avg);
if (!quiet || !period) {
printf("\n--- %s (%s %s) ioping statistics ---\n",
path, fstype, device);
printf("%d requests completed in %.1f ms, "
"%.0f iops, %.1f mb/s\n",
request, time_total/1000.,
request * 1000000. / time_sum,
(double)request * size / time_sum /
(1<<20) * 1000000);
printf("min/avg/max/mdev = %.1f/%.1f/%.1f/%.1f ms\n",
time_min/1000., time_avg/1000.,
time_max/1000., time_mdev/1000.);
}
return 0;
}
/* drop cache except for the header and the active pages */
void rrd_dontneed(
rrd_file_t *rrd_file,
rrd_t *rrd)
{
rrd_simple_file_t *rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt;
#if defined USE_MADVISE || defined HAVE_POSIX_FADVISE
size_t dontneed_start;
size_t rra_start;
size_t active_block;
size_t i;
ssize_t _page_size = sysconf(_SC_PAGESIZE);
if (rrd_file == NULL) {
#if defined DEBUG && DEBUG
fprintf (stderr, "rrd_dontneed: Argument 'rrd_file' is NULL.\n");
#endif
return;
}
#if defined DEBUG && DEBUG > 1
mincore_print(rrd_file, "before");
#endif
/* ignoring errors from RRDs that are smaller then the file_len+rounding */
rra_start = rrd_file->header_len;
dontneed_start = PAGE_START(rra_start) + _page_size;
for (i = 0; i < rrd->stat_head->rra_cnt; ++i) {
active_block =
PAGE_START(rra_start
+ rrd->rra_ptr[i].cur_row
* rrd->stat_head->ds_cnt * sizeof(rrd_value_t));
if (active_block > dontneed_start) {
#ifdef USE_MADVISE
madvise(rrd_simple_file->file_start + dontneed_start,
active_block - dontneed_start - 1, MADV_DONTNEED);
#endif
/* in linux at least only fadvise DONTNEED seems to purge pages from cache */
#ifdef HAVE_POSIX_FADVISE
posix_fadvise(rrd_simple_file->fd, dontneed_start,
active_block - dontneed_start - 1,
POSIX_FADV_DONTNEED);
#endif
}
dontneed_start = active_block;
/* do not release 'hot' block if update for this RAA will occur
* within 10 minutes */
if (rrd->stat_head->pdp_step * rrd->rra_def[i].pdp_cnt -
rrd->live_head->last_up % (rrd->stat_head->pdp_step *
rrd->rra_def[i].pdp_cnt) < 10 * 60) {
dontneed_start += _page_size;
}
rra_start +=
rrd->rra_def[i].row_cnt * rrd->stat_head->ds_cnt *
sizeof(rrd_value_t);
}
if (dontneed_start < rrd_file->file_len) {
#ifdef USE_MADVISE
madvise(rrd_simple_file->file_start + dontneed_start,
rrd_file->file_len - dontneed_start, MADV_DONTNEED);
#endif
#ifdef HAVE_POSIX_FADVISE
posix_fadvise(rrd_simple_file->fd, dontneed_start,
rrd_file->file_len - dontneed_start,
POSIX_FADV_DONTNEED);
#endif
}
#if defined DEBUG && DEBUG > 1
mincore_print(rrd_file, "after");
#endif
#endif /* without madvise and posix_fadvise it does not make much sense todo anything */
}
static void filter_files(const char *dir, int prefix, int inverse)
{
char path[64];
int i, ndirs;
static struct dirent **dirlist;
struct makenode *t, *next;
filter_prefix = prefix;
#ifdef SUSE /* SuSE */
snprintf(path, sizeof(path), "/etc/init.d/%s.d", dir);
#else /* Debian */
snprintf(path, sizeof(path), "/etc/%s.d", dir);
#endif
#if defined _XOPEN_SOURCE && (_XOPEN_SOURCE - 0) >= 600
if ((i = open(path, o_flags|O_DIRECTORY|O_LARGEFILE)) >= 0) {
(void)posix_fadvise(i, 0, 0, POSIX_FADV_SEQUENTIAL);
(void)posix_fadvise(i, 0, 0, POSIX_FADV_NOREUSE);
}
#endif
ndirs = scandir(path, &dirlist, dirfilter, alphasort);
#if defined _XOPEN_SOURCE && (_XOPEN_SOURCE - 0) >= 600
if (i >= 0) {
(void)posix_fadvise(i, 0, 0, POSIX_FADV_DONTNEED);
close(i);
}
#endif
/* mark all matching nodes */
if (ndirs >= 0) {
for (i = 0; i < ndirs; i++) {
t = lookup_target(dirlist[i]->d_name + 3);
if (t) {
t->status = 1;
t->filter_prefix = filter_prefix;
if (asprintf(&t->arg0, "%s/%s", path, dirlist[i]->d_name) < 0)
t->arg0 = (char*)0;
}
free(dirlist[i]);
}
free(dirlist);
}
/* deselect non-matching nodes */
for (t = tree_list; t; t = next) {
next = t->next;
if ((! t->status && ! inverse) || (t->status && inverse)) {
/* remove from the list */
struct makelist *x, *nx;
struct makenode *p;
for (x = t->select; x; x = nx) {
nx = x->next;
x->node->num_deps--;
free(x);
}
for (x = t->depend; x; x = nx) {
nx = x->next;
x->node->num_sels--;
free(x);
}
if (t == tree_list)
tree_list = next;
else {
for (p = tree_list; p->next != t; p = p->next)
;
p->next = next;
}
/* don't free the node instance itself - it may be selected
* by others
*/
tree_entries--;
continue;
}
t->status = 0;
}
}
rrd_file_t *rrd_open(
const char *const file_name,
rrd_t *rrd,
unsigned rdwr)
{
unsigned long ui;
int flags = 0;
int version;
#ifdef HAVE_MMAP
char *data = MAP_FAILED;
#endif
off_t offset = 0;
struct stat statb;
rrd_file_t *rrd_file = NULL;
rrd_simple_file_t *rrd_simple_file = NULL;
size_t newfile_size = 0;
size_t header_len, value_cnt, data_len;
/* Are we creating a new file? */
if((rdwr & RRD_CREAT) && (rrd->stat_head != NULL))
{
header_len = rrd_get_header_size(rrd);
value_cnt = 0;
for (ui = 0; ui < rrd->stat_head->rra_cnt; ui++)
value_cnt += rrd->stat_head->ds_cnt * rrd->rra_def[ui].row_cnt;
data_len = sizeof(rrd_value_t) * value_cnt;
newfile_size = header_len + data_len;
}
rrd_file = (rrd_file_t*)malloc(sizeof(rrd_file_t));
if (rrd_file == NULL) {
rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
return NULL;
}
memset(rrd_file, 0, sizeof(rrd_file_t));
rrd_file->rrd = rrd;
rrd_file->pvt = malloc(sizeof(rrd_simple_file_t));
if(rrd_file->pvt == NULL) {
rrd_set_error("allocating rrd_simple_file for '%s'", file_name);
return NULL;
}
memset(rrd_file->pvt, 0, sizeof(rrd_simple_file_t));
rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt;
#ifdef DEBUG
if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
(RRD_READONLY | RRD_READWRITE)) {
/* Both READONLY and READWRITE were given, which is invalid. */
rrd_set_error("in read/write request mask");
exit(-1);
}
#endif
#ifdef HAVE_MMAP
rrd_simple_file->mm_prot = PROT_READ;
rrd_simple_file->mm_flags = 0;
#endif
if (rdwr & RRD_READONLY) {
flags |= O_RDONLY;
#ifdef HAVE_MMAP
# if !defined(AIX)
rrd_simple_file->mm_flags = MAP_PRIVATE;
# endif
# ifdef MAP_NORESERVE
rrd_simple_file->mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
# endif
#endif
} else {
if (rdwr & RRD_READWRITE) {
flags |= O_RDWR;
#ifdef HAVE_MMAP
rrd_simple_file->mm_flags = MAP_SHARED;
rrd_simple_file->mm_prot |= PROT_WRITE;
#endif
}
if (rdwr & RRD_CREAT) {
flags |= (O_CREAT | O_TRUNC);
}
if (rdwr & RRD_EXCL) {
flags |= O_EXCL;
}
}
if (rdwr & RRD_READAHEAD) {
#ifdef MAP_POPULATE
rrd_simple_file->mm_flags |= MAP_POPULATE; /* populate ptes and data */
#endif
#if defined MAP_NONBLOCK
rrd_simple_file->mm_flags |= MAP_NONBLOCK; /* just populate ptes */
#endif
}
#if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)
flags |= O_BINARY;
#endif
if ((rrd_simple_file->fd = open(file_name, flags, 0666)) < 0) {
rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
goto out_free;
}
#ifdef HAVE_MMAP
#ifdef HAVE_BROKEN_MS_ASYNC
if (rdwr & RRD_READWRITE) {
/* some unices, the files mtime does not get updated
on memory mapped files, in order to help them,
we update the the timestamp at this point.
The thing happens pretty 'close' to the open
call so the chances of a race should be minimal.
Maybe ask your vendor to fix your OS ... */
utime(file_name,NULL);
}
#endif
#endif
/* Better try to avoid seeks as much as possible. stat may be heavy but
* many concurrent seeks are even worse. */
if (newfile_size == 0 && ((fstat(rrd_simple_file->fd, &statb)) < 0)) {
rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
goto out_close;
}
if (newfile_size == 0) {
rrd_file->file_len = statb.st_size;
} else {
rrd_file->file_len = newfile_size;
#ifdef HAVE_POSIX_FALLOCATE
if (posix_fallocate(rrd_simple_file->fd, 0, newfile_size) == 0){
/* if all is well we skip the seeking below */
goto no_lseek_necessary;
}
#endif
lseek(rrd_simple_file->fd, newfile_size - 1, SEEK_SET);
if ( write(rrd_simple_file->fd, "\0", 1) == -1){ /* poke */
rrd_set_error("write '%s': %s", file_name, rrd_strerror(errno));
goto out_close;
}
lseek(rrd_simple_file->fd, 0, SEEK_SET);
}
no_lseek_necessary:
#ifdef HAVE_POSIX_FADVISE
/* In general we need no read-ahead when dealing with rrd_files.
When we stop reading, it is highly unlikely that we start up again.
In this manner we actually save time and diskaccess (and buffer cache).
Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
posix_fadvise(rrd_simple_file->fd, 0, 0, POSIX_FADV_RANDOM);
#endif
/*
if (rdwr & RRD_READWRITE)
{
if (setvbuf((rrd_simple_file->fd),NULL,_IONBF,2)) {
rrd_set_error("failed to disable the stream buffer\n");
return (-1);
}
}
*/
#ifdef HAVE_MMAP
#ifndef HAVE_POSIX_FALLOCATE
/* force allocating the file on the underlaying filesystem to prevent any
* future bus error when the filesystem is full and attempting to write
* trough the file mapping. Filling the file using memset on the file
* mapping can also lead some bus error, so we use the old fashioned
* write().
*/
if (rdwr & RRD_CREAT) {
char buf[4096];
unsigned i;
memset(buf, DNAN, sizeof buf);
lseek(rrd_simple_file->fd, offset, SEEK_SET);
for (i = 0; i < (newfile_size - 1) / sizeof buf; ++i)
{
if (write(rrd_simple_file->fd, buf, sizeof buf) == -1)
{
rrd_set_error("write '%s': %s", file_name, rrd_strerror(errno));
goto out_close;
}
}
if (write(rrd_simple_file->fd, buf,
(newfile_size - 1) % sizeof buf) == -1)
{
rrd_set_error("write '%s': %s", file_name, rrd_strerror(errno));
goto out_close;
}
lseek(rrd_simple_file->fd, 0, SEEK_SET);
}
#endif
data = mmap(0, rrd_file->file_len,
rrd_simple_file->mm_prot, rrd_simple_file->mm_flags,
rrd_simple_file->fd, offset);
/* lets see if the first read worked */
if (data == MAP_FAILED) {
rrd_set_error("mmaping file '%s': %s", file_name,
rrd_strerror(errno));
goto out_close;
}
rrd->__mmap_start = data;
rrd->__mmap_size = rrd_file->file_len;
rrd_simple_file->file_start = data;
#endif
if (rdwr & RRD_CREAT)
goto out_done;
#ifdef USE_MADVISE
if (rdwr & RRD_COPY) {
/* We will read everything in a moment (copying) */
madvise(data, rrd_file->file_len, MADV_SEQUENTIAL );
} else {
/* We do not need to read anything in for the moment */
madvise(data, rrd_file->file_len, MADV_RANDOM);
}
#endif
__rrd_read(rrd->stat_head, stat_head_t,
1);
/* lets do some test if we are on track ... */
if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
rrd_set_error("'%s' is not an RRD file", file_name);
goto out_nullify_head;
}
if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
rrd_set_error("This RRD was created on another architecture");
goto out_nullify_head;
}
version = atoi(rrd->stat_head->version);
if (version > atoi(RRD_VERSION)) {
rrd_set_error("can't handle RRD file version %s",
rrd->stat_head->version);
goto out_nullify_head;
}
__rrd_read(rrd->ds_def, ds_def_t,
rrd->stat_head->ds_cnt);
__rrd_read(rrd->rra_def, rra_def_t,
rrd->stat_head->rra_cnt);
/* handle different format for the live_head */
if (version < 3) {
rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
if (rrd->live_head == NULL) {
rrd_set_error("live_head_t malloc");
goto out_close;
}
__rrd_read(rrd->legacy_last_up, time_t,
1);
rrd->live_head->last_up = *rrd->legacy_last_up;
rrd->live_head->last_up_usec = 0;
} else {
__rrd_read(rrd->live_head, live_head_t,
1);
}
__rrd_read(rrd->pdp_prep, pdp_prep_t,
rrd->stat_head->ds_cnt);
__rrd_read(rrd->cdp_prep, cdp_prep_t,
rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
__rrd_read(rrd->rra_ptr, rra_ptr_t,
rrd->stat_head->rra_cnt);
rrd_file->header_len = offset;
rrd_file->pos = offset;
{
unsigned long row_cnt = 0;
for (ui=0; ui<rrd->stat_head->rra_cnt; ui++)
row_cnt += rrd->rra_def[ui].row_cnt;
size_t correct_len = rrd_file->header_len +
sizeof(rrd_value_t) * row_cnt * rrd->stat_head->ds_cnt;
if (correct_len > rrd_file->file_len)
{
rrd_set_error("'%s' is too small (should be %ld bytes)",
file_name, (long long) correct_len);
goto out_nullify_head;
}
if (rdwr & RRD_READVALUES) {
long d_offset = offset;
__rrd_read(rrd->rrd_value, rrd_value_t,
row_cnt * rrd->stat_head->ds_cnt);
rrd_file->header_len = d_offset;
rrd_file->pos = d_offset;
}
}
out_done:
return (rrd_file);
out_nullify_head:
rrd->stat_head = NULL;
out_close:
#ifdef HAVE_MMAP
if (data != MAP_FAILED)
munmap(data, rrd_file->file_len);
rrd->__mmap_start = NULL;
rrd->__mmap_size = 0;
#endif
close(rrd_simple_file->fd);
out_free:
free(rrd_file->pvt);
free(rrd_file);
return NULL;
}
static void validate_file(const char *path, int number,
uint64_t *ptr_ok, uint64_t *ptr_corrupted, uint64_t *ptr_changed,
uint64_t *ptr_overwritten, uint64_t *ptr_size, int *ptr_read_all,
struct timeval *ptr_dt, int progress)
{
char *full_fn;
const char *filename;
const int num_int64 = SECTOR_SIZE >> 3;
uint64_t sector[num_int64];
FILE *f;
int fd;
size_t sectors_read;
uint64_t expected_offset;
int final_errno;
struct timeval t1, t2;
/* Progress time. */
struct timeval pt1 = { .tv_sec = -1000, .tv_usec = 0 };
*ptr_ok = *ptr_corrupted = *ptr_changed = *ptr_overwritten = 0;
full_fn = full_fn_from_number(&filename, path, number);
assert(full_fn);
printf("Validating file %s ... %s", filename, progress ? BLANK : "");
fflush(stdout);
#ifdef __CYGWIN__
/* We don't need write access, but some kernels require that
* the file descriptor passed to fdatasync(2) to be writable.
*/
f = fopen(full_fn, "rb+");
#else
f = fopen(full_fn, "rb");
#endif
if (!f)
err(errno, "Can't open file %s", full_fn);
fd = fileno(f);
assert(fd >= 0);
/* If the kernel follows our advice, f3read won't ever read from cache
* even when testing small memory cards without a remount, and
* we should have a better reading-speed measurement.
*/
assert(!fdatasync(fd));
assert(!posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED));
/* Obtain initial time. */
assert(!gettimeofday(&t1, NULL));
/* Help the kernel to help us. */
assert(!posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL));
sectors_read = fread(sector, SECTOR_SIZE, 1, f);
final_errno = errno;
expected_offset = (uint64_t)number * GIGABYTES;
while (sectors_read > 0) {
uint64_t rn;
int error_count, i;
assert(sectors_read == 1);
rn = sector[0];
error_count = 0;
for (i = 1; error_count <= TOLERANCE && i < num_int64; i++) {
rn = random_number(rn);
if (rn != sector[i])
error_count++;
}
if (expected_offset == sector[0]) {
if (error_count == 0)
(*ptr_ok)++;
else if (error_count <= TOLERANCE)
(*ptr_changed)++;
else
(*ptr_corrupted)++;
} else if (error_count <= TOLERANCE)
(*ptr_overwritten)++;
else
(*ptr_corrupted)++;
sectors_read = fread(sector, SECTOR_SIZE, 1, f);
final_errno = errno;
expected_offset += SECTOR_SIZE;
if (progress) {
struct timeval pt2;
assert(!gettimeofday(&pt2, NULL));
/* Avoid often printouts. */
if (delay_ms(&pt1, &pt2) >= 200) {
PRINT_STATUS(CLEAR);
fflush(stdout);
pt1 = pt2;
}
}
}
assert(!gettimeofday(&t2, NULL));
update_dt(ptr_dt, &t1, &t2);
*ptr_read_all = feof(f);
*ptr_size = ftell(f);
PRINT_STATUS(progress ? CLEAR : "");
if (!*ptr_read_all) {
assert(ferror(f));
printf(" - NOT fully read due to \"%s\"",
strerror(final_errno));
}
printf("\n");
fclose(f);
free(full_fn);
}
static void report(const char *prefix, uint64_t i)
{
double f = (double) (i * SECTOR_SIZE);
const char *unit = adjust_unit(&f);
printf("%s %.2f %s (%" PRIu64 " sectors)\n", prefix, f, unit, i);
}
void RecData::RecordThread()
{
lt_info("%s: begin\n", __func__);
hal_set_threadname("hal:record");
const int readsize = bufsize / 16;
int buf_pos = 0;
int queued = 0;
struct aiocb a;
int val = fcntl(file_fd, F_GETFL);
if (fcntl(file_fd, F_SETFL, val|O_APPEND))
lt_info("%s: O_APPEND? (%m)\n", __func__);
memset(&a, 0, sizeof(a));
a.aio_fildes = file_fd;
a.aio_sigevent.sigev_notify = SIGEV_NONE;
dmx->Start();
int overflow_count = 0;
bool overflow = false;
int r = 0;
while (exit_flag == RECORD_RUNNING)
{
if (buf_pos < bufsize)
{
if (overflow_count) {
lt_info("%s: Overflow cleared after %d iterations\n", __func__, overflow_count);
overflow_count = 0;
}
int toread = bufsize - buf_pos;
if (toread > readsize)
toread = readsize;
ssize_t s = dmx->Read(buf + buf_pos, toread, 50);
lt_debug("%s: buf_pos %6d s %6d / %6d\n", __func__,
buf_pos, (int)s, bufsize - buf_pos);
if (s < 0)
{
if (errno != EAGAIN && (errno != EOVERFLOW || !overflow))
{
lt_info("%s: read failed: %m\n", __func__);
exit_flag = RECORD_FAILED_READ;
state = REC_STATUS_OVERFLOW;
break;
}
}
else
{
overflow = false;
buf_pos += s;
}
}
else
{
if (!overflow)
overflow_count = 0;
overflow = true;
if (!(overflow_count % 10))
lt_info("%s: buffer full! Overflow? (%d)\n", __func__, ++overflow_count);
state = REC_STATUS_SLOW;
}
r = aio_error(&a);
if (r == EINPROGRESS)
{
lt_debug("%s: aio in progress, free: %d\n", __func__, bufsize - buf_pos);
continue;
}
// not calling aio_return causes a memory leak --martii
r = aio_return(&a);
if (r < 0)
{
exit_flag = RECORD_FAILED_FILE;
lt_debug("%s: aio_return = %d (%m)\n", __func__, r);
break;
}
else
lt_debug("%s: aio_return = %d, free: %d\n", __func__, r, bufsize - buf_pos);
if (posix_fadvise(file_fd, 0, 0, POSIX_FADV_DONTNEED))
perror("posix_fadvise");
if (queued)
{
memmove(buf, buf + queued, buf_pos - queued);
buf_pos -= queued;
}
queued = buf_pos;
a.aio_buf = buf;
a.aio_nbytes = queued;
r = aio_write(&a);
if (r)
{
lt_info("%s: aio_write %d (%m)\n", __func__, r);
exit_flag = RECORD_FAILED_FILE;
break;
}
}
dmx->Stop();
while (true) /* write out the unwritten buffer content */
{
lt_debug("%s: run-out write, buf_pos %d\n", __func__, buf_pos);
r = aio_error(&a);
if (r == EINPROGRESS)
{
usleep(50000);
continue;
}
r = aio_return(&a);
if (r < 0)
{
exit_flag = RECORD_FAILED_FILE;
lt_info("%s: aio_result: %d (%m)\n", __func__, r);
break;
}
if (!queued)
break;
memmove(buf, buf + queued, buf_pos - queued);
buf_pos -= queued;
queued = buf_pos;
a.aio_buf = buf;
a.aio_nbytes = queued;
r = aio_write(&a);
}
#if 0
// TODO: do we need to notify neutrino about failing recording?
CEventServer eventServer;
eventServer.registerEvent2(NeutrinoMessages::EVT_RECORDING_ENDED, CEventServer::INITID_NEUTRINO, "/tmp/neutrino.sock");
stream2file_status2_t s;
s.status = exit_flag;
strncpy(s.filename,basename(myfilename),512);
s.filename[511] = '\0';
strncpy(s.dir,dirname(myfilename),100);
s.dir[99] = '\0';
eventServer.sendEvent(NeutrinoMessages::EVT_RECORDING_ENDED, CEventServer::INITID_NEUTRINO, &s, sizeof(s));
printf("[stream2file]: pthreads exit code: %i, dir: '%s', filename: '%s' myfilename: '%s'\n", exit_flag, s.dir, s.filename, myfilename);
#endif
lt_info("%s: end\n", __func__);
pthread_exit(NULL);
}
int main(int argc, char **argv) {
FILE *ifile = stdin;
FILE *ofile = stdout;
FILE *ffile = NULL;
int ret;
float alpha, ilines_rate, ilines_rate_avg;
int64_t raw_lines = -1;
uint64_t report_mask = 0;
uint64_t time_last, time_curr, time_delta;
uint64_t time_start, time_elapsed;
uint64_t ilines_last, ilines_curr, ilines_delta;
uint64_t olines;
int skipping = 0, tty = 0;
char *line = NULL;
size_t line_sz = 0;
int line_read;
int c, spok = 0, aopt = 0, vopt = 0, wopt = 16, Lopt = 0;
int nopt_mod = 0, nopt_rem = 0;
uint64_t kopt = 0;
unsigned char *bopt = NULL, *iopt = NULL, *oopt = NULL;
unsigned char *topt = NULL, *sopt = NULL, *popt = NULL;
unsigned char *mopt = NULL, *fopt = NULL, *ropt = NULL;
while ((c = getopt(argc, argv, "avb:hi:k:f:m:n:o:p:s:r:t:w:L")) != -1) {
switch (c) {
case 'a':
aopt = 1; // open output file in append mode
break;
case 'k':
kopt = strtoull(optarg, NULL, 10); // skip first k lines of input
skipping = 1;
break;
case 'n':
// only try the rem'th of every mod lines (one indexed)
nopt_rem = atoi(optarg) - 1;
optarg = strchr(optarg, '/');
if (optarg != NULL) { nopt_mod = atoi(optarg+1); }
skipping = 1;
break;
case 'w':
if (wopt > 1) wopt = atoi(optarg);
break;
case 'm':
mopt = optarg; // table file
wopt = 1; // auto
break;
case 'v':
vopt = 1; // verbose
break;
case 'b':
bopt = optarg; // bloom filter file
break;
case 'f':
fopt = optarg; // full filter file
break;
case 'i':
iopt = optarg; // input file
break;
case 'o':
oopt = optarg; // output file
break;
case 's':
sopt = optarg; // salt
break;
case 'p':
popt = optarg; // passphrase
break;
case 'r':
ropt = optarg; // rushwallet
break;
case 't':
topt = optarg; // type of input
break;
case 'L':
Lopt = 1; // lookup output
break;
case 'h':
// show help
usage(argv[0]);
return 0;
case '?':
// show error
return 1;
default:
// should never be reached...
printf("got option '%c' (%d)\n", c, c);
return 1;
}
}
if (optind < argc) {
if (optind == 1 && argc == 2) {
// older versions of brainflayer had the bloom filter file as a
// single optional argument, this keeps compatibility with that
bopt = argv[1];
} else {
fprintf(stderr, "Invalid arguments:\n");
while (optind < argc) {
fprintf(stderr, " '%s'\n", argv[optind++]);
}
exit(1);
}
}
if (nopt_rem != 0 || nopt_mod != 0) {
// note that nopt_rem has had one subtracted at option parsing
if (nopt_rem >= nopt_mod) {
bail(1, "Invalid '-n' argument, remainder '%d' must be <= modulus '%d'\n", nopt_rem+1, nopt_mod);
} else if (nopt_rem < 0) {
bail(1, "Invalid '-n' argument, remainder '%d' must be > 0\n", nopt_rem+1);
} else if (nopt_mod < 1) {
bail(1, "Invalid '-n' argument, modulus '%d' must be > 0\n", nopt_mod);
}
}
if (wopt < 1 || wopt > 28) {
bail(1, "Invalid window size '%d' - must be >= 1 and <= 28\n", wopt);
} else {
// very rough sanity check of window size
struct sysinfo info;
sysinfo(&info);
uint64_t sysram = info.mem_unit * info.totalram;
if (3584LLU*(1<<wopt) > sysram) {
bail(1, "Not enough ram for requested window size '%d'\n", wopt);
}
}
if (topt != NULL) {
if (strcmp(topt, "str") == 0) {
input2hash160 = &pass2hash160;
} else if (strcmp(topt, "hex") == 0) {
input2hash160 = &hexpass2hash160;
} else if (strcmp(topt, "priv") == 0) {
input2hash160 = &hexpriv2hash160;
} else if (strcmp(topt, "warp") == 0) {
spok = 1;
input2hash160 = popt ? &warpsalt2hash160 : &warppass2hash160;
} else if (strcmp(topt, "bwio") == 0) {
spok = 1;
input2hash160 = popt ? &bwiosalt2hash160 : &bwiopass2hash160;
} else if (strcmp(topt, "bv2") == 0) {
spok = 1;
input2hash160 = popt ? &brainv2salt2hash160 : &brainv2pass2hash160;
} else if (strcmp(topt, "rush") == 0) {
input2hash160 = &rush2hash160;
} else {
bail(1, "Unknown input type '%s'.\n", topt);
}
} else {
topt = "str";
input2hash160 = &pass2hash160;
}
if (spok) {
if (sopt && popt) {
bail(1, "Cannot specify both a salt and a passphrase\n");
}
if (popt) {
kdfpass = popt;
kdfpass_sz = strlen(popt);
} else {
if (sopt) {
kdfsalt = sopt;
kdfsalt_sz = strlen(kdfsalt);
} else {
kdfsalt = chkmalloc(0);
kdfsalt_sz = 0;
}
}
} else {
if (popt) {
bail(1, "Specifying a passphrase not supported with input type '%s'\n", topt);
} else if (sopt) {
bail(1, "Specifying a salt not supported with this input type '%s'\n", topt);
}
}
if (ropt) {
if (input2hash160 != &rush2hash160) {
bail(1, "Specifying a url fragment only supported with input type 'rush'\n");
}
kdfsalt = ropt;
kdfsalt_sz = strlen(kdfsalt) - sizeof(rushchk)*2;
if (kdfsalt[kdfsalt_sz-1] != '!') {
bail(1, "Invalid rushwallet url fragment '%s'\n", kdfsalt);
}
unhex(kdfsalt+kdfsalt_sz, sizeof(rushchk)*2, rushchk, sizeof(rushchk));
kdfsalt[kdfsalt_sz] = '\0';
} else if (input2hash160 == &rush2hash160) {
bail(1, "The '-r' option is required for rushwallet.\n");
}
if (bopt) {
if (Lopt) {
bail(1, "The '-L' option cannot be used with a bloom filter\n");
}
if ((ret = mmapf(&bloom_mmapf, bopt, BLOOM_SIZE, MMAPF_RNDRD)) != MMAPF_OKAY) {
bail(1, "failed to open bloom filter '%s': %s\n", bopt, mmapf_strerror(ret));
} else if (bloom_mmapf.mem == NULL) {
bail(1, "got NULL pointer trying to set up bloom filter\n");
}
bloom = bloom_mmapf.mem;
}
if (fopt) {
if (!bopt) {
bail(1, "The '-f' option must be used with a bloom filter\n");
}
if ((ffile = fopen(fopt, "r")) == NULL) {
bail(1, "failed to open '%s' for reading: %s\n", fopt, strerror(errno));
}
}
if (iopt) {
if ((ifile = fopen(iopt, "r")) == NULL) {
bail(1, "failed to open '%s' for reading: %s\n", iopt, strerror(errno));
}
// increases readahead window, don't really care if it fails
posix_fadvise(fileno(ifile), 0, 0, POSIX_FADV_SEQUENTIAL);
}
if (oopt && (ofile = fopen(oopt, (aopt ? "a" : "w"))) == NULL) {
bail(1, "failed to open '%s' for writing: %s\n", oopt, strerror(errno));
}
/* line buffer output */
setvbuf(ofile, NULL, _IOLBF, 0);
/* line buffer stderr */
setvbuf(stderr, NULL, _IOLBF, 0);
if (vopt && ofile == stdout && isatty(fileno(stdout))) { tty = 1; }
brainflayer_init_globals();
if (secp256k1_ec_pubkey_precomp_table(wopt, mopt) != 0) {
bail(1, "failed to initialize precomputed table\n");
}
if (vopt) {
/* initialize timing data */
time_start = time_last = getns();
olines = ilines_last = ilines_curr = 0;
ilines_rate_avg = -1;
alpha = 0.500;
} else {
time_start = time_last = 0; // prevent compiler warning about uninitialized use
}
for (;;) {
if ((line_read = getline(&line, &line_sz, ifile)-1) > -1) {
if (skipping) {
++raw_lines;
if (kopt && raw_lines < kopt) { continue; }
if (nopt_mod && raw_lines % nopt_mod != nopt_rem) { continue; }
}
line[line_read] = 0;
if (input2hash160(line, line_read) == 0) {
if (bloom) {
if (bloom_chk_hash160(bloom, hash160_uncmp.ul)) {
if (!fopt || hsearchf(ffile, &hash160_uncmp)) {
if (tty) { fprintf(ofile, "\033[0K"); }
fprintresult(ofile, &hash160_uncmp, 'u', topt, line);
++olines;
}
}
if (bloom_chk_hash160(bloom, hash160_compr.ul)) {
if (!fopt || hsearchf(ffile, &hash160_compr)) {
if (tty) { fprintf(ofile, "\033[0K"); }
fprintresult(ofile, &hash160_compr, 'c', topt, line);
++olines;
}
}
} else if (Lopt) {
fprintlookup(ofile, &hash160_uncmp, &hash160_compr, priv256, topt, line);
} else {
fprintresult(ofile, &hash160_uncmp, 'u', topt, line);
fprintresult(ofile, &hash160_compr, 'c', topt, line);
}
}
} else {
if (!vopt) break;
}
if (vopt) {
++ilines_curr;
if (line_read < 0 || (ilines_curr & report_mask) == 0) {
time_curr = getns();
time_delta = time_curr - time_last;
time_elapsed = time_curr - time_start;
time_last = time_curr;
ilines_delta = ilines_curr - ilines_last;
ilines_last = ilines_curr;
ilines_rate = (ilines_delta * 1.0e9) / (time_delta * 1.0);
if (line_read < 0) {
/* report overall average on last status update */
ilines_rate_avg = (--ilines_curr * 1.0e9) / (time_elapsed * 1.0);
} else if (ilines_rate_avg < 0) {
ilines_rate_avg = ilines_rate;
/* target reporting frequency to about once every five seconds */
} else if (time_delta < 2500000000) {
report_mask = (report_mask << 1) | 1;
ilines_rate_avg = ilines_rate; /* reset EMA */
} else if (time_delta > 10000000000) {
report_mask >>= 1;
ilines_rate_avg = ilines_rate; /* reset EMA */
} else {
/* exponetial moving average */
ilines_rate_avg = alpha * ilines_rate + (1 - alpha) * ilines_rate_avg;
}
fprintf(stderr,
"\033[0G\033[2K"
" rate: %9.2f p/s"
" found: %5zu/%-10zu"
" elapsed: %8.3f s"
"\033[0G",
ilines_rate_avg,
olines,
ilines_curr,
time_elapsed / 1.0e9
);
if (line_read < 0) {
fprintf(stderr, "\n");
break;
} else {
fflush(stderr);
}
}
int MtpFfsCompatHandle::receiveFile(mtp_file_range mfr, bool zero_packet) {
// When receiving files, the incoming length is given in 32 bits.
// A >4G file is given as 0xFFFFFFFF
uint32_t file_length = mfr.length;
uint64_t offset = mfr.offset;
int packet_size = getPacketSize(mBulkOut);
unsigned char *data = mIobuf[0].bufs.data();
unsigned char *data2 = mIobuf[1].bufs.data();
struct aiocb aio;
aio.aio_fildes = mfr.fd;
aio.aio_buf = nullptr;
struct aiocb *aiol[] = {&aio};
int ret = -1;
size_t length;
bool read = false;
bool write = false;
posix_fadvise(mfr.fd, 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
// Break down the file into pieces that fit in buffers
while (file_length > 0 || write) {
if (file_length > 0) {
length = std::min(static_cast<uint32_t>(MAX_FILE_CHUNK_SIZE), file_length);
// Read data from USB, handle errors after waiting for write thread.
ret = readHandle(mBulkOut, data, length);
if (file_length != MAX_MTP_FILE_SIZE && ret < static_cast<int>(length)) {
ret = -1;
errno = EIO;
}
read = true;
}
if (write) {
// get the return status of the last write request
aio_suspend(aiol, 1, nullptr);
int written = aio_return(&aio);
if (written == -1) {
errno = aio_error(&aio);
return -1;
}
if (static_cast<size_t>(written) < aio.aio_nbytes) {
errno = EIO;
return -1;
}
write = false;
}
// If there was an error reading above
if (ret == -1) {
return -1;
}
if (read) {
if (file_length == MAX_MTP_FILE_SIZE) {
// For larger files, receive until a short packet is received.
if (static_cast<size_t>(ret) < length) {
file_length = 0;
}
} else {
file_length -= ret;
}
// Enqueue a new write request
aio_prepare(&aio, data, length, offset);
aio_write(&aio);
offset += ret;
std::swap(data, data2);
write = true;
read = false;
}
}
// Receive an empty packet if size is a multiple of the endpoint size.
if (ret % packet_size == 0 || zero_packet) {
if (TEMP_FAILURE_RETRY(::read(mBulkOut, data, packet_size)) != 0) {
return -1;
}
}
return 0;
}
long long BSONTool::processFile( const boost::filesystem::path& root ) {
std::string fileName = root.string();
unsigned long long fileLength = file_size( root );
if ( fileLength == 0 ) {
toolInfoOutput() << "file " << fileName << " empty, skipping" << std::endl;
return 0;
}
FILE* file = fopen( fileName.c_str() , "rb" );
if ( ! file ) {
toolError() << "error opening file: " << fileName << " " << errnoWithDescription()
<< std::endl;
return 0;
}
#ifdef POSIX_FADV_SEQUENTIAL
posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL);
#endif
if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) {
toolInfoOutput() << "\t file size: " << fileLength << std::endl;
}
unsigned long long read = 0;
unsigned long long num = 0;
unsigned long long processed = 0;
const int BUF_SIZE = BSONObjMaxUserSize + ( 1024 * 1024 );
boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
char * buf = buf_holder.get();
ProgressMeter m(fileLength);
if (!toolGlobalParams.quiet) {
m.setUnits( "bytes" );
}
while ( read < fileLength ) {
size_t amt = fread(buf, 1, 4, file);
verify( amt == 4 );
int size = ((int*)buf)[0];
uassert( 10264 , str::stream() << "invalid object size: " << size , size < BUF_SIZE );
amt = fread(buf+4, 1, size-4, file);
verify( amt == (size_t)( size - 4 ) );
BSONObj o( buf );
if (bsonToolGlobalParams.objcheck) {
const Status status = validateBSON(buf, size);
if (!status.isOK()) {
toolError() << "INVALID OBJECT - going to try and print out " << std::endl;
toolError() << "size: " << size << std::endl;
toolError() << "error: " << status.reason() << std::endl;
StringBuilder sb;
try {
o.toString(sb); // using StringBuilder version to get as much as possible
} catch (...) {
toolError() << "object up to error: " << sb.str() << endl;
throw;
}
toolError() << "complete object: " << sb.str() << endl;
// NOTE: continuing with object even though we know it is invalid.
}
}
if (!bsonToolGlobalParams.hasFilter || _matcher->matches(o)) {
gotObject( o );
processed++;
}
read += o.objsize();
num++;
if (!toolGlobalParams.quiet) {
m.hit(o.objsize());
}
}
fclose( file );
uassert(10265, "counts don't match", read == fileLength);
toolInfoOutput() << num << ((num == 1) ? " document" : " documents")
<< " found" << std::endl;
if (bsonToolGlobalParams.hasFilter) {
toolInfoOutput() << processed
<< ((processed == 1) ? " document" : " documents")
<< " processed" << std::endl;
}
return processed;
}
int MtpFfsCompatHandle::sendFile(mtp_file_range mfr) {
uint64_t file_length = mfr.length;
uint32_t given_length = std::min(static_cast<uint64_t>(MAX_MTP_FILE_SIZE),
file_length + sizeof(mtp_data_header));
uint64_t offset = mfr.offset;
int packet_size = getPacketSize(mBulkIn);
// If file_length is larger than a size_t, truncating would produce the wrong comparison.
// Instead, promote the left side to 64 bits, then truncate the small result.
int init_read_len = std::min(
static_cast<uint64_t>(packet_size - sizeof(mtp_data_header)), file_length);
unsigned char *data = mIobuf[0].bufs.data();
unsigned char *data2 = mIobuf[1].bufs.data();
posix_fadvise(mfr.fd, 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
struct aiocb aio;
aio.aio_fildes = mfr.fd;
struct aiocb *aiol[] = {&aio};
int ret, length;
int error = 0;
bool read = false;
bool write = false;
// Send the header data
mtp_data_header *header = reinterpret_cast<mtp_data_header*>(data);
header->length = htole32(given_length);
header->type = htole16(2); /* data packet */
header->command = htole16(mfr.command);
header->transaction_id = htole32(mfr.transaction_id);
// Some hosts don't support header/data separation even though MTP allows it
// Handle by filling first packet with initial file data
if (TEMP_FAILURE_RETRY(pread(mfr.fd, reinterpret_cast<char*>(data) +
sizeof(mtp_data_header), init_read_len, offset))
!= init_read_len) return -1;
if (writeHandle(mBulkIn, data, sizeof(mtp_data_header) + init_read_len) == -1) return -1;
file_length -= init_read_len;
offset += init_read_len;
ret = init_read_len + sizeof(mtp_data_header);
// Break down the file into pieces that fit in buffers
while (file_length > 0) {
if (read) {
// Wait for the previous read to finish
aio_suspend(aiol, 1, nullptr);
ret = aio_return(&aio);
if (ret == -1) {
errno = aio_error(&aio);
return -1;
}
if (static_cast<size_t>(ret) < aio.aio_nbytes) {
errno = EIO;
return -1;
}
file_length -= ret;
offset += ret;
std::swap(data, data2);
read = false;
write = true;
}
if (error == -1) {
return -1;
}
if (file_length > 0) {
length = std::min(static_cast<uint64_t>(MAX_FILE_CHUNK_SIZE), file_length);
// Queue up another read
aio_prepare(&aio, data, length, offset);
aio_read(&aio);
read = true;
}
if (write) {
if (writeHandle(mBulkIn, data2, ret) == -1) {
error = -1;
}
write = false;
}
}
if (ret % packet_size == 0) {
// If the last packet wasn't short, send a final empty packet
if (TEMP_FAILURE_RETRY(::write(mBulkIn, data, 0)) != 0) {
return -1;
}
}
return 0;
}
static void dig_holes(int fd, off_t off, off_t len)
{
off_t end = len ? off + len : 0;
off_t hole_start = 0, hole_sz = 0;
uintmax_t ct = 0;
size_t bufsz;
char *buf;
struct stat st;
#if defined(POSIX_FADV_SEQUENTIAL) && defined(HAVE_POSIX_FADVISE)
off_t cache_start = off;
/*
* We don't want to call POSIX_FADV_DONTNEED to discard cached
* data in PAGE_SIZE steps. IMHO it's overkill (too many syscalls).
*
* Let's assume that 1MiB (on system with 4K page size) is just
* a good compromise.
* -- kzak Feb-2014
*/
const size_t cachesz = getpagesize() * 256;
#endif
if (fstat(fd, &st) != 0)
err(EXIT_FAILURE, _("stat of %s failed"), filename);
bufsz = st.st_blksize;
if (lseek(fd, off, SEEK_SET) < 0)
err(EXIT_FAILURE, _("seek on %s failed"), filename);
/* buffer + extra space for is_nul() sentinel */
buf = xmalloc(bufsz + sizeof(uintptr_t));
#if defined(POSIX_FADV_SEQUENTIAL) && defined(HAVE_POSIX_FADVISE)
posix_fadvise(fd, off, 0, POSIX_FADV_SEQUENTIAL);
#endif
while (end == 0 || off < end) {
ssize_t rsz;
rsz = pread(fd, buf, bufsz, off);
if (rsz < 0 && errno)
err(EXIT_FAILURE, _("%s: read failed"), filename);
if (end && rsz > 0 && off > end - rsz)
rsz = end - off;
if (rsz <= 0)
break;
if (is_nul(buf, rsz)) {
if (!hole_sz) { /* new hole detected */
int rc = skip_hole(fd, &off);
if (rc == 0)
continue; /* hole skipped */
else if (rc == 1)
break; /* end of file */
hole_start = off;
}
hole_sz += rsz;
} else if (hole_sz) {
xfallocate(fd, FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE,
hole_start, hole_sz);
ct += hole_sz;
hole_sz = hole_start = 0;
}
#if defined(POSIX_FADV_DONTNEED) && defined(HAVE_POSIX_FADVISE)
/* discard cached data */
if (off - cache_start > (off_t) cachesz) {
size_t clen = off - cache_start;
clen = (clen / cachesz) * cachesz;
posix_fadvise(fd, cache_start, clen, POSIX_FADV_DONTNEED);
cache_start = cache_start + clen;
}
#endif
off += rsz;
}
if (hole_sz) {
xfallocate(fd, FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE,
hole_start, hole_sz);
ct += hole_sz;
}
free(buf);
if (verbose) {
char *str = size_to_human_string(SIZE_SUFFIX_3LETTER | SIZE_SUFFIX_SPACE, ct);
fprintf(stdout, _("%s: %s (%ju bytes) converted to sparse holes.\n"),
filename, str, ct);
free(str);
}
}
bool warmup(const char *path, enum FileType file_type){
int fd = 0;
DIR *dir = NULL;
int time_used = 0;
char file_name[256];
struct stat st;
struct timeval begin, end;
struct dirent *dp = NULL;
if(stat(path, &st)<0){
goto ERROR;
}
gettimeofday(&begin, NULL);
if(file_type == REGFILE){
fd = open(path, O_RDONLY);
if(fd<0){
goto ERROR;
}
if(posix_fadvise(fd, 0, st.st_size, POSIX_FADV_WILLNEED) != 0){
goto ERROR;
}
}else if(file_type == DIRECTORY){
if((dir = opendir(path)) == NULL){
goto ERROR;
}
while((dp = readdir(dir)) != NULL){
if(dp->d_name[0] != '.'){
memset(file_name, 0, sizeof(path));
strcat(file_name, path);
strcat(file_name, "/");
strcat(file_name, dp->d_name);
if(dp->d_type == DT_REG){
warmup(file_name, REGFILE);
}else if(dp->d_type == DT_DIR){
warmup(file_name, DIRECTORY);
}else{
fprintf(stdout, "%s:%c type unsupported!\n", dp->d_name, dp->d_type);
}
}
}
}
gettimeofday(&end, NULL);
time_used = getUsedTime(&begin, &end);
if(file_type == REGFILE){
fprintf(stdout, "Warmup File:%s TimeUsed:%d ms\n", path, time_used);
close(fd);
}else if(file_type == DIRECTORY){
fprintf(stdout, "Warmup Dir:%s TimeUsed:%d ms\n", path, time_used);
closedir(dir);
}
return true;
ERROR:
fprintf(stderr, "File:%s %s\n", path, strerror(errno));
if(fd) close(fd);
if(dir) closedir(dir);
return false;
}
