void vmtouch_crawl(char *path) {
struct stat sb;
DIR *dirp;
struct dirent *de;
char npath[PATH_MAX];
int res;
int tp_path_len = strlen(path);
int i;
if (path[tp_path_len-1] == '/') path[tp_path_len-1] = '\0'; // prevent ugly double slashes when printing path names
res = o_followsymlinks ? stat(path, &sb) : lstat(path, &sb);
if (res) {
warning("unable to stat %s (%s)", path, strerror(errno));
return;
} else {
if (S_ISLNK(sb.st_mode)) {
warning("not following symbolic link %s", path);
return;
}
if (!o_ignorehardlinkeduplictes && sb.st_nlink > 1) {
/*
* For files with more than one link to it, ignore it if we already know
* inode. Without this check files copied as hardlinks (cp -al) are
* counted twice (which may lead to a cache usage of more than 100% of
* RAM).
*/
if (find_object(&sb)) {
// we already saw the device and inode referenced by this file
return;
} else {
add_object(&sb);
}
}
if (S_ISDIR(sb.st_mode)) {
for (i=0; i<curr_crawl_depth; i++) {
if (crawl_inodes[i] == sb.st_ino) {
warning("symbolic link loop detected: %s", path);
return;
}
}
if (curr_crawl_depth == MAX_CRAWL_DEPTH)
fatal("maximum directory crawl depth reached: %s", path);
total_dirs++;
crawl_inodes[curr_crawl_depth] = sb.st_ino;
retry_opendir:
dirp = opendir(path);
if (dirp == NULL) {
if (errno == ENFILE || errno == EMFILE) {
increment_nofile_rlimit();
goto retry_opendir;
}
warning("unable to opendir %s (%s), skipping", path, strerror(errno));
return;
}
while((de = readdir(dirp)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
if (snprintf(npath, sizeof(npath), "%s/%s", path, de->d_name) >= sizeof(npath)) {
warning("path too long %s", path);
goto bail;
}
curr_crawl_depth++;
vmtouch_crawl(npath);
curr_crawl_depth--;
}
bail:
if (closedir(dirp)) {
warning("unable to closedir %s (%s)", path, strerror(errno));
return;
}
} else if (S_ISREG(sb.st_mode)) {
total_files++;
vmtouch_file(path);
} else {
warning("skipping non-regular file: %s", path);
}
}
}
void crawler::_crawl(std::string &path_std_string, file_index &already_seen_files) {
const char *path = path_std_string.c_str();
struct ::stat sb;
int res = follow_symlinks ? stat(path, &sb) : lstat(path, &sb);
if (res) {
warning("unable to stat %s (%s)", path, strerror(errno));
return;
}
if (S_ISLNK(sb.st_mode)) {
warning("not following symbolic link %s", path);
return;
}
if (skip_duplicate_hardlinks && sb.st_nlink > 1) {
file_index::iterator dev = already_seen_files.find(sb.st_dev);
// Haven't seen this device before, initialize it
if (dev == already_seen_files.end()) {
already_seen_files.emplace(sb.st_dev, std::unordered_set<ino_t> {sb.st_ino});
}
// Already seen this inode before
else if (!dev->second.insert(sb.st_ino).second) {
warning("skipping duplicate hardlink %s", path);
return;
}
}
if (S_ISDIR(sb.st_mode)) {
if (curr_crawl_depth == max_crawl_depth) {
warning("maximum directory crawl depth reached: %s", path);
return;
}
num_dirs++;
retry_opendir:
DIR *dirp = opendir(path);
if (dirp == NULL) {
if (errno == ENFILE || errno == EMFILE) {
increment_nofile_rlimit();
goto retry_opendir;
}
warning("unable to opendir %s (%s), skipping", path, strerror(errno));
return;
}
struct dirent *de;
std::vector<std::string> dir_entries;
while((de = readdir(dirp)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
dir_entries.emplace_back(de->d_name);
}
if (closedir(dirp)) fatal("unable to closedir %s (%s)", path, strerror(errno));
std::sort(dir_entries.begin(), dir_entries.end());
for (auto &s : dir_entries) {
std::string npath = path_std_string + std::string("/") + s;
curr_crawl_depth++;
_crawl(npath, already_seen_files);
curr_crawl_depth--;
}
} else if (S_ISREG(sb.st_mode)) {
num_files++;
// FIXME: catch exceptions thrown by the callback
file_handler(path_std_string, sb);
} else {
warning("skipping non-regular file: %s", path);
}
}
void vmtouch_file(char *path) {
int fd;
void *mem;
struct stat sb;
int64_t len_of_file;
int64_t pages_in_file;
int i;
int res;
res = o_followsymlinks ? stat(path, &sb) : lstat(path, &sb);
if (res) {
warning("unable to stat %s (%s), skipping", path, strerror(errno));
return;
}
if (S_ISLNK(sb.st_mode)) {
warning("not following symbolic link %s", path);
return;
}
if (sb.st_size == 0) return;
if (sb.st_size > o_max_file_size) {
warning("file %s too large, skipping", path);
return;
}
len_of_file = sb.st_size;
retry_open:
fd = open(path, O_RDONLY, 0);
if (fd == -1) {
if (errno == ENFILE || errno == EMFILE) {
increment_nofile_rlimit();
goto retry_open;
}
warning("unable to open %s (%s), skipping", path, strerror(errno));
return;
}
mem = mmap(NULL, len_of_file, PROT_READ, MAP_SHARED, fd, 0);
if (mem == MAP_FAILED) {
warning("unable to mmap file %s (%s), skipping", path, strerror(errno));
close(fd);
return;
}
if (!aligned_p(mem)) fatal("mmap(%s) wasn't page aligned", path);
pages_in_file = bytes2pages(len_of_file);
total_pages += pages_in_file;
if (o_evict) {
if (o_verbose) printf("Evicting %s\n", path);
#if defined(__linux__) || defined(__hpux)
if (posix_fadvise(fd, 0, len_of_file, POSIX_FADV_DONTNEED))
warning("unable to posix_fadvise file %s (%s)", path, strerror(errno));
#elif defined(__FreeBSD__) || defined(__sun__) || defined(__APPLE__)
if (msync(mem, len_of_file, MS_INVALIDATE))
warning("unable to msync invalidate file %s (%s)", path, strerror(errno));
#else
fatal("cache eviction not (yet?) supported on this platform");
#endif
} else {
int64_t pages_in_core=0;
double last_chart_print_time=0.0, temp_time;
char *mincore_array = malloc(pages_in_file);
if (mincore_array == NULL) fatal("Failed to allocate memory for mincore array (%s)", strerror(errno));
// 3rd arg to mincore is char* on BSD and unsigned char* on linux
if (mincore(mem, len_of_file, (void*)mincore_array)) fatal("mincore %s (%s)", path, strerror(errno));
for (i=0; i<pages_in_file; i++) {
if (is_mincore_page_resident(mincore_array[i])) {
pages_in_core++;
total_pages_in_core++;
}
}
if (o_verbose) {
printf("%s\n", path);
last_chart_print_time = gettimeofday_as_double();
print_page_residency_chart(stdout, mincore_array, pages_in_file);
}
if (o_touch) {
for (i=0; i<pages_in_file; i++) {
junk_counter += ((char*)mem)[i*pagesize];
mincore_array[i] = 1;
if (o_verbose) {
temp_time = gettimeofday_as_double();
if (temp_time > (last_chart_print_time+CHART_UPDATE_INTERVAL)) {
last_chart_print_time = temp_time;
print_page_residency_chart(stdout, mincore_array, pages_in_file);
}
}
}
}
if (o_verbose) {
print_page_residency_chart(stdout, mincore_array, pages_in_file);
printf("\n");
}
free(mincore_array);
}
if (o_lock) {
if (mlock(mem, len_of_file))
fatal("mlock: %s (%s)", path, strerror(errno));
}
if (!o_lock && !o_lockall) {
if (munmap(mem, len_of_file)) warning("unable to munmap file %s (%s)", path, strerror(errno));
close(fd);
}
}
