/* zero all the stats structures */
void
stats_zero(void)
{
int dir;
unsigned i;
char *fname;
fname = format("%s/stats", cache_dir);
x_unlink(fname);
free(fname);
for (dir = 0; dir <= 0xF; dir++) {
struct counters *counters = counters_init(STATS_END);
fname = format("%s/%1x/stats", cache_dir, dir);
if (lockfile_acquire(fname, lock_staleness_limit)) {
stats_read(fname, counters);
for (i = 0; stats_info[i].message; i++) {
if (!(stats_info[i].flags & FLAG_NOZERO)) {
counters->data[stats_info[i].stat] = 0;
}
}
stats_write(fname, counters);
lockfile_release(fname);
}
counters_free(counters);
free(fname);
}
}
// Zero all the stats structures.
void
stats_zero(void)
{
assert(conf);
char *fname = format("%s/stats", conf->cache_dir);
x_unlink(fname);
free(fname);
time_t timestamp = time(NULL);
for (int dir = 0; dir <= 0xF; dir++) {
struct counters *counters = counters_init(STATS_END);
struct stat st;
fname = format("%s/%1x/stats", conf->cache_dir, dir);
if (stat(fname, &st) != 0) {
// No point in trying to reset the stats file if it doesn't exist.
free(fname);
continue;
}
if (lockfile_acquire(fname, lock_staleness_limit)) {
stats_read(fname, counters);
for (unsigned i = 0; stats_info[i].message; i++) {
if (!(stats_info[i].flags & FLAG_NOZERO)) {
counters->data[stats_info[i].stat] = 0;
}
}
counters->data[STATS_ZEROTIMESTAMP] = timestamp;
stats_write(fname, counters);
lockfile_release(fname);
}
counters_free(counters);
free(fname);
}
}
static void
delete_file(const char *path, size_t size)
{
if (x_unlink(path) == 0) {
cache_size -= size;
files_in_cache--;
} else if (errno != ENOENT) {
cc_log("Failed to unlink %s (%s)", path, strerror(errno));
}
}
/* traverse function for wiping files */
static void wipe_fn(const char *fname, struct stat *st)
{
char *p;
if (!S_ISREG(st->st_mode)) return;
p = basename(fname);
if (str_eq(p, "stats")) {
free(p);
return;
}
free(p);
x_unlink(fname);
}
/* this builds the list of files in the cache */
static void
traverse_fn(const char *fname, struct stat *st)
{
char *p;
if (!S_ISREG(st->st_mode)) {
return;
}
p = basename(fname);
if (str_eq(p, "stats")) {
goto out;
}
if (str_startswith(p, ".nfs")) {
/* Ignore temporary NFS files that may be left for open but deleted
* files. */
goto out;
}
if (strstr(p, ".tmp.")) {
/* delete any tmp files older than 1 hour */
if (st->st_mtime + 3600 < time(NULL)) {
x_unlink(fname);
goto out;
}
}
if (strstr(p, "CACHEDIR.TAG")) {
goto out;
}
if (num_files == allocated) {
allocated = 10000 + num_files*2;
files = (struct files **)x_realloc(files, sizeof(struct files *)*allocated);
}
files[num_files] = (struct files *)x_malloc(sizeof(struct files));
files[num_files]->fname = x_strdup(fname);
files[num_files]->mtime = st->st_mtime;
files[num_files]->size = file_size(st);
cache_size += files[num_files]->size;
files_in_cache++;
num_files++;
out:
free(p);
}
// Traverse function for wiping files.
static void wipe_fn(const char *fname, struct stat *st)
{
if (!S_ISREG(st->st_mode)) {
return;
}
char *p = basename(fname);
if (str_eq(p, "stats")) {
free(p);
return;
}
free(p);
files_in_cache++;
x_unlink(fname);
}
// Put the object name into a manifest file given a set of included files.
// Returns true on success, otherwise false.
bool
manifest_put(const char *manifest_path, struct file_hash *object_hash,
struct hashtable *included_files)
{
int ret = 0;
gzFile f2 = NULL;
struct manifest *mf = NULL;
char *tmp_file = NULL;
// We don't bother to acquire a lock when writing the manifest to disk. A
// race between two processes will only result in one lost entry, which is
// not a big deal, and it's also very unlikely.
int fd1 = open(manifest_path, O_RDONLY | O_BINARY);
if (fd1 == -1) {
// New file.
mf = create_empty_manifest();
} else {
gzFile f1 = gzdopen(fd1, "rb");
if (!f1) {
cc_log("Failed to gzdopen manifest file");
close(fd1);
goto out;
}
mf = read_manifest(f1);
gzclose(f1);
if (!mf) {
cc_log("Failed to read manifest file; deleting it");
x_unlink(manifest_path);
mf = create_empty_manifest();
}
}
if (mf->n_objects > MAX_MANIFEST_ENTRIES) {
// Normally, there shouldn't be many object entries in the manifest since
// new entries are added only if an include file has changed but not the
// source file, and you typically change source files more often than
// header files. However, it's certainly possible to imagine cases where
// the manifest will grow large (for instance, a generated header file that
// changes for every build), and this must be taken care of since
// processing an ever growing manifest eventually will take too much time.
// A good way of solving this would be to maintain the object entries in
// LRU order and discarding the old ones. An easy way is to throw away all
// entries when there are too many. Let's do that for now.
cc_log("More than %u entries in manifest file; discarding",
MAX_MANIFEST_ENTRIES);
free_manifest(mf);
mf = create_empty_manifest();
} else if (mf->n_file_infos > MAX_MANIFEST_FILE_INFO_ENTRIES) {
// Rarely, file_info entries can grow large in pathological cases where
// many included files change, but the main file does not. This also puts
// an upper bound on the number of file_info entries.
cc_log("More than %u file_info entries in manifest file; discarding",
MAX_MANIFEST_FILE_INFO_ENTRIES);
free_manifest(mf);
mf = create_empty_manifest();
}
tmp_file = format("%s.tmp", manifest_path);
int fd2 = create_tmp_fd(&tmp_file);
f2 = gzdopen(fd2, "wb");
if (!f2) {
cc_log("Failed to gzdopen %s", tmp_file);
goto out;
}
add_object_entry(mf, object_hash, included_files);
if (write_manifest(f2, mf)) {
gzclose(f2);
f2 = NULL;
if (x_rename(tmp_file, manifest_path) == 0) {
ret = 1;
} else {
cc_log("Failed to rename %s to %s", tmp_file, manifest_path);
goto out;
}
} else {
cc_log("Failed to write manifest file");
goto out;
}
out:
if (mf) {
free_manifest(mf);
}
if (tmp_file) {
free(tmp_file);
}
if (f2) {
gzclose(f2);
}
return ret;
}
/*
* Put the object name into a manifest file given a set of included files.
* Returns true on success, otherwise false.
*/
bool
manifest_put(const char *manifest_path, struct file_hash *object_hash,
struct hashtable *included_files)
{
int ret = 0;
int fd1;
int fd2;
gzFile f2 = NULL;
struct manifest *mf = NULL;
char *tmp_file = NULL;
/*
* We don't bother to acquire a lock when writing the manifest to disk. A
* race between two processes will only result in one lost entry, which is
* not a big deal, and it's also very unlikely.
*/
fd1 = open(manifest_path, O_RDONLY | O_BINARY);
if (fd1 == -1) {
/* New file. */
mf = create_empty_manifest();
} else {
gzFile f1 = gzdopen(fd1, "rb");
if (!f1) {
cc_log("Failed to gzdopen manifest file");
close(fd1);
goto out;
}
mf = read_manifest(f1);
gzclose(f1);
if (!mf) {
cc_log("Failed to read manifest file; deleting it");
x_unlink(manifest_path);
mf = create_empty_manifest();
}
}
if (mf->n_objects > MAX_MANIFEST_ENTRIES) {
/*
* Normally, there shouldn't be many object entries in the manifest since
* new entries are added only if an include file has changed but not the
* source file, and you typically change source files more often than
* header files. However, it's certainly possible to imagine cases where
* the manifest will grow large (for instance, a generated header file that
* changes for every build), and this must be taken care of since
* processing an ever growing manifest eventually will take too much time.
* A good way of solving this would be to maintain the object entries in
* LRU order and discarding the old ones. An easy way is to throw away all
* entries when there are too many. Let's do that for now.
*/
cc_log("More than %u entries in manifest file; discarding",
MAX_MANIFEST_ENTRIES);
free_manifest(mf);
mf = create_empty_manifest();
}
tmp_file = format("%s.tmp.%s", manifest_path, tmp_string());
fd2 = safe_create_wronly(tmp_file);
if (fd2 == -1) {
cc_log("Failed to open %s", tmp_file);
goto out;
}
f2 = gzdopen(fd2, "wb");
if (!f2) {
cc_log("Failed to gzdopen %s", tmp_file);
goto out;
}
add_object_entry(mf, object_hash, included_files);
if (write_manifest(f2, mf)) {
gzclose(f2);
f2 = NULL;
if (x_rename(tmp_file, manifest_path) == 0) {
ret = 1;
} else {
cc_log("Failed to rename %s to %s", tmp_file, manifest_path);
goto out;
}
} else {
cc_log("Failed to write manifest file");
goto out;
}
out:
if (mf) {
free_manifest(mf);
}
if (tmp_file) {
free(tmp_file);
}
if (f2) {
gzclose(f2);
}
return ret;
}
/*
* This function acquires a lockfile for the given path. Returns true if the
* lock was acquired, otherwise false. If the lock has been considered stale
* for the number of microseconds specified by staleness_limit, the function
* will (if possible) break the lock and then try to acquire it again. The
* staleness limit should be reasonably larger than the longest time the lock
* can be expected to be held, and the updates of the locked path should
* probably be made with an atomic rename(2) to avoid corruption in the rare
* case that the lock is broken by another process.
*/
bool
lockfile_acquire(const char *path, unsigned staleness_limit)
{
char *lockfile = format("%s.lock", path);
char *my_content = NULL, *content = NULL, *initial_content = NULL;
const char *hostname = get_hostname();
bool acquired = false;
#ifdef _WIN32
const size_t bufsize = 1024;
int fd, len;
#else
int ret;
#endif
unsigned to_sleep = 1000, slept = 0; /* Microseconds. */
while (1) {
free(my_content);
my_content = format("%s:%d:%d", hostname, (int)getpid(), (int)time(NULL));
#ifdef _WIN32
fd = open(lockfile, O_WRONLY|O_CREAT|O_EXCL|O_BINARY, 0666);
if (fd == -1) {
cc_log("lockfile_acquire: open WRONLY %s: %s", lockfile, strerror(errno));
if (errno == ENOENT) {
/* Directory doesn't exist? */
if (create_parent_dirs(lockfile) == 0) {
/* OK. Retry. */
continue;
}
}
if (errno != EEXIST) {
/* Directory doesn't exist or isn't writable? */
goto out;
}
/* Someone else has the lock. */
fd = open(lockfile, O_RDONLY|O_BINARY);
if (fd == -1) {
if (errno == ENOENT) {
/*
* The file was removed after the open() call above, so retry
* acquiring it.
*/
continue;
} else {
cc_log("lockfile_acquire: open RDONLY %s: %s",
lockfile, strerror(errno));
goto out;
}
}
free(content);
content = x_malloc(bufsize);
if ((len = read(fd, content, bufsize - 1)) == -1) {
cc_log("lockfile_acquire: read %s: %s", lockfile, strerror(errno));
close(fd);
goto out;
}
close(fd);
content[len] = '\0';
} else {
/* We got the lock. */
if (write(fd, my_content, strlen(my_content)) == -1) {
cc_log("lockfile_acquire: write %s: %s", lockfile, strerror(errno));
close(fd);
x_unlink(lockfile);
goto out;
}
close(fd);
acquired = true;
goto out;
}
#else
ret = symlink(my_content, lockfile);
if (ret == 0) {
/* We got the lock. */
acquired = true;
goto out;
}
cc_log("lockfile_acquire: symlink %s: %s", lockfile, strerror(errno));
if (errno == ENOENT) {
/* Directory doesn't exist? */
if (create_parent_dirs(lockfile) == 0) {
/* OK. Retry. */
continue;
}
}
if (errno != EEXIST) {
/* Directory doesn't exist or isn't writable? */
goto out;
}
free(content);
content = x_readlink(lockfile);
if (!content) {
if (errno == ENOENT) {
/*
* The symlink was removed after the symlink() call above, so retry
* acquiring it.
*/
continue;
} else {
cc_log("lockfile_acquire: readlink %s: %s", lockfile, strerror(errno));
goto out;
}
}
#endif
if (str_eq(content, my_content)) {
/* Lost NFS reply? */
cc_log("lockfile_acquire: symlink %s failed but we got the lock anyway",
lockfile);
acquired = true;
goto out;
}
/*
* A possible improvement here would be to check if the process holding the
* lock is still alive and break the lock early if it isn't.
*/
cc_log("lockfile_acquire: lock info for %s: %s", lockfile, content);
if (!initial_content) {
initial_content = x_strdup(content);
}
if (slept > staleness_limit) {
if (str_eq(content, initial_content)) {
/* The lock seems to be stale -- break it. */
cc_log("lockfile_acquire: breaking %s", lockfile);
if (lockfile_acquire(lockfile, staleness_limit)) {
lockfile_release(path);
lockfile_release(lockfile);
to_sleep = 1000;
slept = 0;
continue;
}
}
cc_log("lockfile_acquire: gave up acquiring %s", lockfile);
goto out;
}
cc_log("lockfile_acquire: failed to acquire %s; sleeping %u microseconds",
lockfile, to_sleep);
usleep(to_sleep);
slept += to_sleep;
to_sleep *= 2;
}
out:
if (acquired) {
cc_log("Acquired lock %s", lockfile);
} else {
cc_log("Failed to acquire lock %s", lockfile);
}
free(lockfile);
free(my_content);
free(initial_content);
free(content);
return acquired;
}
