static uint64_t curl_get_object_size(const char *url)
{
CURL *curl;
CURLcode res;
double content_length = 0;
curl = curl_easy_init();
if (!curl) {
sheepfs_pr("Failed to init curl");
goto out;
}
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_NOBODY, true);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
curl_easy_setopt(curl, CURLOPT_URL, url);
res = curl_easy_perform(curl);
if (res == CURLE_OK) {
res = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
&content_length);
if (res != CURLE_OK) {
sheepfs_pr("Failed to get size of object %s",
curl_easy_strerror(res));
content_length = 0;
}
}
out:
curl_easy_cleanup(curl);
return (uint64_t)content_length;
}
static bool curl_object_exists(const char *url)
{
CURL *curl;
CURLcode res;
bool ret = false;
curl = curl_easy_init();
if (!curl) {
sheepfs_pr("Failed to init curl");
goto out;
}
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_NOBODY, true);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
curl_easy_setopt(curl, CURLOPT_URL, url);
res = curl_easy_perform(curl);
if (res == CURLE_OK)
ret = true;
else
sheepfs_pr("Failed to call libcurl res: %s, url: %s",
curl_easy_strerror(res), url);
out:
curl_easy_cleanup(curl);
return ret;
}
static size_t curl_read_object(const char *url, char *buf, size_t size,
off_t offset)
{
CURL *curl;
CURLcode res;
char header[PATH_MAX];
double content_length;
struct buffer_s buffer = { 0 };
struct curl_slist *headers = NULL;
curl = curl_easy_init();
if (!curl) {
sheepfs_pr("Failed to init curl");
goto out;
}
snprintf(header, sizeof(header), "Range: bytes=%"PRIu64"-%"PRIu64,
offset, offset + size - 1);
headers = curl_slist_append(headers, header);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
buffer.mem = buf;
buffer.total_size = size;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
res = curl_easy_perform(curl);
if (res == CURLE_OK) {
res = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
&content_length);
if (res != CURLE_OK) {
sheepfs_pr("Failed to getinfo res: %s",
curl_easy_strerror(res));
size = 0;
goto out;
}
if ((size_t)content_length > size) {
sheepfs_pr("Failed to get correct CONTENT_LENGTH, "
"content_length: %"PRIu64", get_size: %"
PRIu64, (size_t)content_length, size);
size = 0;
} else {
sheepfs_pr("Read out %"PRIu64" data from %s",
size, url);
size = (size_t)content_length;
}
} else {
sheepfs_pr("Failed to call libcurl res: %s, url: %s",
curl_easy_strerror(res), url);
size = 0;
}
out:
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
return size;
}
static void *fetch_thread_run(void *arg)
{
struct cache_handle *ch = (struct cache_handle *)arg;
char url[PATH_MAX];
char *pos = strstr(ch->path, PATH_HTTP) + strlen(PATH_HTTP);
int ret;
while (true) {
sem_wait(&ch->prepare_sem);
if (ch->stop)
break;
/* update cache */
ret = generate_url(pos, strlen(ch->path) - strlen(PATH_HTTP),
url, PATH_MAX);
if (ret)
sheepfs_pr("failed to generate url for %s", ch->path);
else {
ret = curl_read_object(url, ch->prepare->mem,
CACHE_SIZE, ch->fetch_offset);
ch->prepare->offset = ch->fetch_offset;
ch->prepare->size = ret;
}
sem_post(&ch->ready_sem);
}
return NULL;
}
static int object_create_entry(const char *entry, const char *url)
{
struct strbuf buf = STRBUF_INIT;
char *args[3], path[PATH_MAX];
int nr, ret = -EINVAL;
uint64_t size;
nr = split_path(entry, ARRAY_SIZE(args), args);
if (nr != ARRAY_SIZE(args)) {
sheepfs_pr("Invalid argument %d, %s", nr, entry);
goto out;
}
strbuf_addf(&buf, "%s", PATH_HTTP);
for (int i = 0; i < nr; i++) {
strbuf_addf(&buf, "/%s", args[i]);
snprintf(path, sizeof(path), "%.*s", (int)buf.len, buf.buf);
if (i == (nr - 1)) {
if (shadow_file_create(path) < 0) {
sheepfs_pr("Create file %s fail", path);
goto out;
}
size = curl_get_object_size(url);
if (size <= 0) {
sheepfs_pr("Failed to get size of object");
shadow_file_delete(path);
goto out;
}
if (shadow_file_setxattr(path, HTTP_SIZE_NAME, &size,
HTTP_SIZE_SIZE) < 0) {
sheepfs_pr("Failed to setxattr for %s",
HTTP_SIZE_NAME);
shadow_file_delete(path);
goto out;
}
if (sheepfs_set_op(path, OP_OBJECT) < 0) {
sheepfs_pr("Set_op %s fail", path);
shadow_file_delete(path);
goto out;
}
} else {
if (shadow_dir_create(path) < 0) {
sheepfs_pr("Create dir %s fail", path);
goto out;
}
if (sheepfs_set_op(path, OP_CONTAINER) < 0) {
sheepfs_pr("Set_op %s fail", path);
shadow_dir_delete(path);
goto out;
}
}
}
ret = 0;
out:
for (int i = 0; i < ARRAY_SIZE(args); i++)
free(args[i]);
strbuf_release(&buf);
return ret;
}
int shadow_dir_create(const char *path)
{
char p[PATH_MAX];
snprintf(p, sizeof(p), "%s%s", sheepfs_shadow, path);
if (xmkdir(p, 0755) < 0) {
sheepfs_pr("%m\n");
return -1;
}
return 0;
}
size_t shadow_file_write(const char *path, char *buf, size_t size)
{
char p[PATH_MAX];
int fd;
size_t len = 0;
snprintf(p, sizeof(p), "%s%s", sheepfs_shadow, path);
fd = open(p, O_WRONLY | O_TRUNC);
if (fd < 0) {
sheepfs_pr("%m\n");
return 0;
}
len = xwrite(fd, buf, size);
if (len != size) {
sheepfs_pr("failed to write\n");
len = 0;
}
close(fd);
return len;
}
int shadow_file_setxattr(const char *path, const char *name,
const void *value, size_t size)
{
char p[PATH_MAX];
snprintf(p, sizeof(p), "%s%s", sheepfs_shadow, path);
if (setxattr(p, name, value, size, 0) < 0) {
sheepfs_pr("%m\n");
return -1;
}
return 0;
}
bool shadow_file_exsit(const char *path)
{
char p[PATH_MAX];
snprintf(p, sizeof(p), "%s%s", sheepfs_shadow, path);
if (access(p, R_OK | W_OK) < 0) {
if (errno != ENOENT)
sheepfs_pr("%m\n");
return false;
}
return true;
}
int shadow_file_delete(const char *path)
{
char p[PATH_MAX];
snprintf(p, sizeof(p), "%s%s", sheepfs_shadow, path);
if (unlink(p) < 0) {
if (errno != ENOENT) {
sheepfs_pr("%m\n");
return -1;
}
}
return 0;
}
int object_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi)
{
struct cache_handle *ch;
struct read_cache *cache;
char *pos;
int ret;
pos = strstr(path, PATH_HTTP);
if (!pos) {
sheepfs_pr("Invalid Path %s", path);
ret = -EINVAL;
goto out;
}
ch = (struct cache_handle *)fi->fh;
while (true) {
cache = ch->ready;
/* try to read from cache first */
if (offset >= cache->offset &&
(cache->offset + cache->size) > offset) {
if ((cache->offset + cache->size) > (offset + size))
ret = size;
else
ret = (cache->offset + cache->size) - offset;
memcpy(buf, cache->mem + (offset - cache->offset), ret);
/* read next cache if not fulfill the 'size' */
if (ret < size && object_wait_cache(ch) > 0) {
int extra_read;
buf += ret;
offset += ret;
cache = ch->ready;
extra_read = min(cache->size, size - ret);
memcpy(buf, cache->mem +
(offset - cache->offset), extra_read);
ret += extra_read;
}
break;
} else if (offset >= ch->obj_size) {
ret = 0;
break;
} else if (!object_wait_cache(ch)) {
ret = 0;
break;
}
}
out:
return ret;
}
int shadow_file_create(const char *path)
{
char p[PATH_MAX];
int fd;
snprintf(p, sizeof(p), "%s%s", sheepfs_shadow, path);
fd = creat(p, 0644);
if (fd < 0) {
if (errno != EEXIST) {
sheepfs_pr("%m\n");
return -1;
}
}
close(fd);
return 0;
}
int shadow_file_read(const char *path, char *buf, size_t size, off_t offset)
{
char p[PATH_MAX];
int fd, len;
snprintf(p, sizeof(p), "%s%s", sheepfs_shadow, path);
fd = open(p, O_RDONLY);
if (fd < 0) {
sheepfs_pr("%m\n");
return -errno;
}
len = xpread(fd, buf, size, offset);
close(fd);
return len;
}
int object_open(const char *path, struct fuse_file_info *fi)
{
struct cache_handle *ch;
char *pos;
int ret;
pos = strstr(path, PATH_HTTP);
if (!pos) {
sheepfs_pr("Invalid Path %s", path);
return -EINVAL;
}
/* don't need page cache of fuse */
fi->direct_io = 1;
ch = xzalloc(sizeof(*ch));
ch->ready = xzalloc(sizeof(struct read_cache));
ch->ready->mem = xmalloc(CACHE_SIZE);
ch->prepare = xzalloc(sizeof(struct read_cache));
ch->prepare->mem = xmalloc(CACHE_SIZE);
ch->stop = false;
ch->fetch_offset = 0;
ch->obj_size = object_get_size(path);
fi->fh = (uint64_t)ch;
sem_init(&ch->ready_sem, 0, 0);
sem_init(&ch->prepare_sem, 0, 1);
strncpy(ch->path, path, PATH_MAX);
ret = pthread_create(&ch->fetch_thread, NULL, fetch_thread_run, ch);
if (ret != 0) {
sheepfs_pr("failed to create thread to fetch data");
release_cache_handle(ch);
return -1;
}
return 0;
}
static int generate_url(const char *buff, int size, char *url, int url_len)
{
char address[PATH_MAX], *ch;
int len, ret = 0;
len = shadow_file_read(PATH_HTTP_ADDRESS, address, sizeof(address), 0);
if (len <= 0) {
sheepfs_pr("Can't get address of http");
ret = -EINVAL;
goto out;
}
/* remove last '\n' of address */
ch = strchr(address, '\n');
if (ch != NULL)
*ch = '\0';
snprintf(url, url_len, "http://%s/v1%.*s", address, size, buff);
out:
return ret;
}
