static char *
build_cache_tree(const char *path)
{
LOG(LOG_DEBUG, "building cache dir for '%s'", path);
char *local = NULL;
char *tmp_local = NULL;
char *dir = NULL;
struct stat st;
/* ignore the leading spaces */
while (path && '/' == *path)
path++;
local = tmpstr_printf("%s/%s", conf->cache_dir, path);
tmp_local = strdup(local);
if (! tmp_local) {
LOG(LOG_CRIT, "strdup(%s): %s", tmp_local, strerror(errno));
return NULL;
}
dir = tmpstr_printf("%s", dirname(tmp_local));
if (-1 == stat(dir, &st)) {
if (ENOENT == errno)
mkdir_tree(dir);
else
LOG(LOG_ERR, "stat(%s): %s", dir, strerror(errno));
}
free(tmp_local);
return local;
}
void
assign_meta_to_dict(dpl_dict_t *dict,
char *meta,
unsigned long val)
{
char *buf = NULL;
buf = tmpstr_printf("%lu", val);
LOG(LOG_DEBUG, "meta='%s', value='%s'", meta, buf);
if (DPL_SUCCESS != dpl_dict_add(dict, meta, buf, 1))
LOG(LOG_ERR, "can't update value '%s' for '%s'", buf, meta);
}
void
pentry_unlink_cache_file(tpath_entry *pe)
{
char *local = NULL;
assert(pe);
if (! pe->path)
return;
local = tmpstr_printf("%s/%s", conf->cache_dir, pe->path);
if (-1 == unlink(local))
LOG(LOG_INFO, "unlink(%s): %s", local, strerror(errno));
}
tpath_entry *
pentry_get_parent(tpath_entry *pe)
{
tpath_entry *parent = NULL;
char *path = NULL;
char *p = NULL;
char *dirname = NULL;
/* sanity check */
if (! pe) {
LOG(LOG_ERR, "NULL entry, no parent");
goto end;
}
LOG(LOG_DEBUG, "path=%s", pe->path);
path = tmpstr_printf("%s", pe->path);
p = strrchr(path, '/');
if (! p) {
LOG(LOG_ERR, "malformed path: %s", pe->path);
goto end;
}
if (p == path)
dirname = "/";
else
*p = 0;
parent = g_hash_table_lookup(hash, dirname);
end:
LOG(LOG_DEBUG, "parent path=%s", parent ? parent->path : "null");
return parent;
}
/* return the fd of a local copy, to operate on */
int
dfs_get_local_copy(pentry_t *pe,
const char * const remote,
int flags)
{
int fd;
dpl_dict_t *metadata = NULL;
dpl_dict_t *headers = NULL;
struct get_data get_data = { .fd = -1, .buf = NULL };
dpl_status_t rc = DPL_FAILURE;
char *local = NULL;
unsigned encryption = 0;
local = tmpstr_printf("%s%s", conf->cache_dir, remote);
LOG(LOG_DEBUG, "bucket=%s, path=%s, local=%s",
ctx->cur_bucket, remote, local);
if (-1 == download_headers((char *)remote, &headers)) {
LOG(LOG_NOTICE, "%s: can't download headers", remote);
fd = -1;
goto end;
}
metadata = dpl_dict_new(13);
if (! metadata) {
LOG(LOG_ERR, "dpl_dict_new: can't allocate memory");
fd = -1;
goto end;
}
if (DPL_FAILURE == dpl_get_metadata_from_headers(
#if defined(DPL_VERSION_MAJOR) && defined(DPL_VERSION_MINOR) && (DPL_VERSION_MAJOR == 0 && DPL_VERSION_MINOR >= 2) || (DPL_VERSION_MAJOR > 0)
ctx,
#endif
headers, metadata)) {
LOG(LOG_ERR, "%s: metadata extraction failed", remote);
fd = -1;
goto end;
}
if (-1 == pentry_set_metadata(pe, metadata)) {
LOG(LOG_ERR, "can't update metadata");
fd = -1;
goto end;
}
if (-1 == check_permissions(pe, metadata)) {
LOG(LOG_NOTICE, "permission denied");
fd = -1;
goto end;
}
/* If the remote MD5 matches a cache file, we don't have to download
* it again, just return the (open) file descriptor of the cache file
*/
if (0 == compare_digests(pe, headers)) {
fd = pentry_get_fd(pe);
goto end;
}
/* a cache file already exists, its MD5 digest is different, so
* just remove it */
if (0 == access(local, F_OK)) {
LOG(LOG_DEBUG, "removing cache file '%s'", local);
if (-1 == unlink(local))
LOG(LOG_ERR, "unlink(%s): %s", local, strerror(errno));
}
get_data.fd = open(local, O_RDWR|O_CREAT|O_TRUNC, 0600);
if (-1 == get_data.fd) {
LOG(LOG_ERR, "open: %s: %s (%d)",
local, strerror(errno), errno);
fd = -1;
goto end;
}
encryption = check_encryption_flag(metadata);
rc = dpl_openread(ctx,
(char *)remote,
encryption,
NULL,
cb_get_buffered,
&get_data,
&metadata);
if (DPL_SUCCESS != rc) {
LOG(LOG_ERR, "dpl_openread: %s", dpl_status_str(rc));
close(get_data.fd);
fd = -1;
goto end;
}
/* If the file is compressed, uncompress it! */
if(-1 == handle_compression(remote, local, &get_data, metadata)) {
fd = -1;
goto end;
}
if (-1 == close(get_data.fd)) {
LOG(LOG_ERR, "close(path=%s, fd=%d): %s",
local, get_data.fd, strerror(errno));
fd = -1;
goto end;
}
fd = open(local, flags, 0600);
if (-1 == fd) {
LOG(LOG_ERR, "open(path=%s, fd=%d): %s",
local, fd, strerror(errno));
fd = -1;
goto end;
}
end:
if (metadata)
dpl_dict_free(metadata);
if (headers)
dpl_dict_free(headers);
return fd;
}
static int
handle_compression(const char *remote,
char *local,
struct get_data *get_data,
dpl_dict_t *metadata)
{
dpl_status_t rc;
char *uzlocal = NULL;
FILE *fpsrc = NULL;
FILE *fpdst = NULL;
char *compressed = NULL;
int zret;
int ret;
compressed = dpl_dict_get_value(metadata, "compression");
if (! compressed) {
LOG(LOG_INFO, "%s: uncompressed remote file", remote);
ret = 0;
goto end;
}
#define NONE "none"
#define ZLIB "zlib"
if (0 == strncmp(compressed, NONE, strlen(NONE))) {
LOG(LOG_INFO, "compression method: 'none'");
ret = 0;
goto end;
}
if (0 != strncmp(compressed, ZLIB, strlen(ZLIB))) {
LOG(LOG_ERR, "compression method not supported '%s'",
compressed);
ret = -1;
goto end;
}
uzlocal = tmpstr_printf("%s.tmp", local);
fpsrc = fopen(local, "r");
if (! fpsrc) {
LOG(LOG_ERR, "fopen: %s", strerror(errno));
ret = -1;
goto end;
}
fpdst = fopen(uzlocal, "w");
if (! fpdst) {
LOG(LOG_ERR, "fopen: %s", strerror(errno));
ret = -1;
goto end;
}
LOG(LOG_INFO, "uncompressing local file '%s'", local);
zret = unzip(fpsrc, fpdst);
if (Z_OK != zret) {
LOG(LOG_ERR, "unzip failed: %s", zerr_to_str(zret));
ret = -1;
goto end;
}
rc = dpl_dict_update_value(metadata, "compression", "none");
if (DPL_SUCCESS != rc) {
LOG(LOG_ERR, "unable to update 'compression' metadata");
ret = -1;
goto end;
}
if (-1 == rename(uzlocal, local)) {
LOG(LOG_ERR, "rename: %s", strerror(errno));
ret = 1;
goto end;
}
close(get_data->fd);
get_data->fd = open(local, O_RDONLY);
if (-1 == get_data->fd) {
LOG(LOG_ERR, "open: %s", strerror(errno));
ret = -1;
goto end;
}
#undef ZLIB
#undef NONE
ret = 0;
end:
if (fpsrc)
fclose(fpsrc);
if (fpdst)
fclose(fpdst);
return ret;
}
static void
gc_callback(gpointer key,
gpointer value,
gpointer user_data)
{
tfs_ctx *ctx = user_data;
GHashTable *hash = ctx->hash;
char *path = key;
tpath_entry *pe = value;
struct stat st;
time_t t;
char *local = NULL;
int refcount = 0;
int threshold = ctx->conf->gc_age_threshold;
assert(pe);
refcount = pentry_get_refcount(pe);
if (refcount)
/* open (either r or rw), don't touch this cell */
return;
if (pentry_trylock(pe))
return;
if (-1 == pe->fd)
/* nothing to do, the tpath_entry cell is allocated but no
* file descriptor/path is affected now
*/
goto release;
if (-1 == fstat(pe->fd, &st)) {
LOG(LOG_ERR, "fstat(fd=%d, %p): %s, remove the cell",
pe->fd, (void *) &st, strerror(errno));
goto remove;
}
if (pe->exclude)
threshold *= 10;
t = time(NULL);
if (t < st.st_atime + threshold &&
t < st.st_mtime + threshold &&
t < st.st_ctime + threshold)
goto release;
LOG(LOG_DEBUG, "%s file too old: now=%d, atime=%d, mtime=%d, ctime=%d",
path, (int)t, (int)st.st_atime, (int)st.st_mtime, (int)st.st_ctime);
remove:
local = tmpstr_printf("%s/%s", ctx->conf->cache_dir, path);
LOG(LOG_INFO, "removing cache file '%s'", local);
if (-1 == unlink(local))
LOG(LOG_ERR, "unlink(%s): %s", local, strerror(errno));
LOG(LOG_DEBUG, "path=%s remove from the hashtable", path);
if (FALSE == g_hash_table_remove(hash, path))
LOG(LOG_WARNING, "can't remove the cell from the hashtable");
return;
release:
(void)pentry_unlock(pe);
}
