static char * build_cache_tree(const char *path) { LOG(LOG_DEBUG, "building cache dir for '%s'", path); char *local = NULL; char *tmp_local = NULL; char *dir = NULL; struct stat st; /* ignore the leading spaces */ while (path && '/' == *path) path++; local = tmpstr_printf("%s/%s", conf->cache_dir, path); tmp_local = strdup(local); if (! tmp_local) { LOG(LOG_CRIT, "strdup(%s): %s", tmp_local, strerror(errno)); return NULL; } dir = tmpstr_printf("%s", dirname(tmp_local)); if (-1 == stat(dir, &st)) { if (ENOENT == errno) mkdir_tree(dir); else LOG(LOG_ERR, "stat(%s): %s", dir, strerror(errno)); } free(tmp_local); return local; }
void assign_meta_to_dict(dpl_dict_t *dict, char *meta, unsigned long val) { char *buf = NULL; buf = tmpstr_printf("%lu", val); LOG(LOG_DEBUG, "meta='%s', value='%s'", meta, buf); if (DPL_SUCCESS != dpl_dict_add(dict, meta, buf, 1)) LOG(LOG_ERR, "can't update value '%s' for '%s'", buf, meta); }
void pentry_unlink_cache_file(tpath_entry *pe) { char *local = NULL; assert(pe); if (! pe->path) return; local = tmpstr_printf("%s/%s", conf->cache_dir, pe->path); if (-1 == unlink(local)) LOG(LOG_INFO, "unlink(%s): %s", local, strerror(errno)); }
tpath_entry * pentry_get_parent(tpath_entry *pe) { tpath_entry *parent = NULL; char *path = NULL; char *p = NULL; char *dirname = NULL; /* sanity check */ if (! pe) { LOG(LOG_ERR, "NULL entry, no parent"); goto end; } LOG(LOG_DEBUG, "path=%s", pe->path); path = tmpstr_printf("%s", pe->path); p = strrchr(path, '/'); if (! p) { LOG(LOG_ERR, "malformed path: %s", pe->path); goto end; } if (p == path) dirname = "/"; else *p = 0; parent = g_hash_table_lookup(hash, dirname); end: LOG(LOG_DEBUG, "parent path=%s", parent ? parent->path : "null"); return parent; }
/* return the fd of a local copy, to operate on */ int dfs_get_local_copy(pentry_t *pe, const char * const remote, int flags) { int fd; dpl_dict_t *metadata = NULL; dpl_dict_t *headers = NULL; struct get_data get_data = { .fd = -1, .buf = NULL }; dpl_status_t rc = DPL_FAILURE; char *local = NULL; unsigned encryption = 0; local = tmpstr_printf("%s%s", conf->cache_dir, remote); LOG(LOG_DEBUG, "bucket=%s, path=%s, local=%s", ctx->cur_bucket, remote, local); if (-1 == download_headers((char *)remote, &headers)) { LOG(LOG_NOTICE, "%s: can't download headers", remote); fd = -1; goto end; } metadata = dpl_dict_new(13); if (! metadata) { LOG(LOG_ERR, "dpl_dict_new: can't allocate memory"); fd = -1; goto end; } if (DPL_FAILURE == dpl_get_metadata_from_headers( #if defined(DPL_VERSION_MAJOR) && defined(DPL_VERSION_MINOR) && (DPL_VERSION_MAJOR == 0 && DPL_VERSION_MINOR >= 2) || (DPL_VERSION_MAJOR > 0) ctx, #endif headers, metadata)) { LOG(LOG_ERR, "%s: metadata extraction failed", remote); fd = -1; goto end; } if (-1 == pentry_set_metadata(pe, metadata)) { LOG(LOG_ERR, "can't update metadata"); fd = -1; goto end; } if (-1 == check_permissions(pe, metadata)) { LOG(LOG_NOTICE, "permission denied"); fd = -1; goto end; } /* If the remote MD5 matches a cache file, we don't have to download * it again, just return the (open) file descriptor of the cache file */ if (0 == compare_digests(pe, headers)) { fd = pentry_get_fd(pe); goto end; } /* a cache file already exists, its MD5 digest is different, so * just remove it */ if (0 == access(local, F_OK)) { LOG(LOG_DEBUG, "removing cache file '%s'", local); if (-1 == unlink(local)) LOG(LOG_ERR, "unlink(%s): %s", local, strerror(errno)); } get_data.fd = open(local, O_RDWR|O_CREAT|O_TRUNC, 0600); if (-1 == get_data.fd) { LOG(LOG_ERR, "open: %s: %s (%d)", local, strerror(errno), errno); fd = -1; goto end; } encryption = check_encryption_flag(metadata); rc = dpl_openread(ctx, (char *)remote, encryption, NULL, cb_get_buffered, &get_data, &metadata); if (DPL_SUCCESS != rc) { LOG(LOG_ERR, "dpl_openread: %s", dpl_status_str(rc)); close(get_data.fd); fd = -1; goto end; } /* If the file is compressed, uncompress it! */ if(-1 == handle_compression(remote, local, &get_data, metadata)) { fd = -1; goto end; } if (-1 == close(get_data.fd)) { LOG(LOG_ERR, "close(path=%s, fd=%d): %s", local, get_data.fd, strerror(errno)); fd = -1; goto end; } fd = open(local, flags, 0600); if (-1 == fd) { LOG(LOG_ERR, "open(path=%s, fd=%d): %s", local, fd, strerror(errno)); fd = -1; goto end; } end: if (metadata) dpl_dict_free(metadata); if (headers) dpl_dict_free(headers); return fd; }
static int handle_compression(const char *remote, char *local, struct get_data *get_data, dpl_dict_t *metadata) { dpl_status_t rc; char *uzlocal = NULL; FILE *fpsrc = NULL; FILE *fpdst = NULL; char *compressed = NULL; int zret; int ret; compressed = dpl_dict_get_value(metadata, "compression"); if (! compressed) { LOG(LOG_INFO, "%s: uncompressed remote file", remote); ret = 0; goto end; } #define NONE "none" #define ZLIB "zlib" if (0 == strncmp(compressed, NONE, strlen(NONE))) { LOG(LOG_INFO, "compression method: 'none'"); ret = 0; goto end; } if (0 != strncmp(compressed, ZLIB, strlen(ZLIB))) { LOG(LOG_ERR, "compression method not supported '%s'", compressed); ret = -1; goto end; } uzlocal = tmpstr_printf("%s.tmp", local); fpsrc = fopen(local, "r"); if (! fpsrc) { LOG(LOG_ERR, "fopen: %s", strerror(errno)); ret = -1; goto end; } fpdst = fopen(uzlocal, "w"); if (! fpdst) { LOG(LOG_ERR, "fopen: %s", strerror(errno)); ret = -1; goto end; } LOG(LOG_INFO, "uncompressing local file '%s'", local); zret = unzip(fpsrc, fpdst); if (Z_OK != zret) { LOG(LOG_ERR, "unzip failed: %s", zerr_to_str(zret)); ret = -1; goto end; } rc = dpl_dict_update_value(metadata, "compression", "none"); if (DPL_SUCCESS != rc) { LOG(LOG_ERR, "unable to update 'compression' metadata"); ret = -1; goto end; } if (-1 == rename(uzlocal, local)) { LOG(LOG_ERR, "rename: %s", strerror(errno)); ret = 1; goto end; } close(get_data->fd); get_data->fd = open(local, O_RDONLY); if (-1 == get_data->fd) { LOG(LOG_ERR, "open: %s", strerror(errno)); ret = -1; goto end; } #undef ZLIB #undef NONE ret = 0; end: if (fpsrc) fclose(fpsrc); if (fpdst) fclose(fpdst); return ret; }
static void gc_callback(gpointer key, gpointer value, gpointer user_data) { tfs_ctx *ctx = user_data; GHashTable *hash = ctx->hash; char *path = key; tpath_entry *pe = value; struct stat st; time_t t; char *local = NULL; int refcount = 0; int threshold = ctx->conf->gc_age_threshold; assert(pe); refcount = pentry_get_refcount(pe); if (refcount) /* open (either r or rw), don't touch this cell */ return; if (pentry_trylock(pe)) return; if (-1 == pe->fd) /* nothing to do, the tpath_entry cell is allocated but no * file descriptor/path is affected now */ goto release; if (-1 == fstat(pe->fd, &st)) { LOG(LOG_ERR, "fstat(fd=%d, %p): %s, remove the cell", pe->fd, (void *) &st, strerror(errno)); goto remove; } if (pe->exclude) threshold *= 10; t = time(NULL); if (t < st.st_atime + threshold && t < st.st_mtime + threshold && t < st.st_ctime + threshold) goto release; LOG(LOG_DEBUG, "%s file too old: now=%d, atime=%d, mtime=%d, ctime=%d", path, (int)t, (int)st.st_atime, (int)st.st_mtime, (int)st.st_ctime); remove: local = tmpstr_printf("%s/%s", ctx->conf->cache_dir, path); LOG(LOG_INFO, "removing cache file '%s'", local); if (-1 == unlink(local)) LOG(LOG_ERR, "unlink(%s): %s", local, strerror(errno)); LOG(LOG_DEBUG, "path=%s remove from the hashtable", path); if (FALSE == g_hash_table_remove(hash, path)) LOG(LOG_WARNING, "can't remove the cell from the hashtable"); return; release: (void)pentry_unlock(pe); }