dpl_req_t *
dpl_req_new(dpl_ctx_t *ctx)
{
dpl_req_t *req = NULL;
req = malloc(sizeof (*req));
if (NULL == req)
goto bad;
memset(req, 0, sizeof (*req));
req->ctx = ctx;
req->metadata = dpl_dict_new(13);
if (NULL == req->metadata)
goto bad;
//virtual hosting is prefered since it "disperses" connections
req->behavior_flags = DPL_BEHAVIOR_KEEP_ALIVE|DPL_BEHAVIOR_VIRTUAL_HOSTING;
return req;
bad:
if (NULL != req)
dpl_req_free(req);
return NULL;
}
int
pentry_set_metadata(pentry_t *pe,
dpl_dict_t *dict)
{
int ret;
assert(pe);
if (pe->metadata)
dpl_dict_free(pe->metadata);
pe->metadata = dpl_dict_new(13);
if (! pe->metadata) {
LOG(LOG_ERR, "dpl_dict_new: can't allocate memory");
ret = -1;
goto err;
}
if (DPL_FAILURE == dpl_dict_copy(pe->metadata, dict)) {
ret = -1;
goto err;
}
ret = 0;
err:
return ret;
}
int
pentry_set_usermd(tpath_entry *pe,
dpl_dict_t *dict)
{
int ret;
assert(pe);
if (pe->usermd)
dpl_dict_free(pe->usermd);
pe->usermd = dpl_dict_new(13);
if (! pe->usermd) {
LOG(LOG_ERR, "path=%s: dpl_dict_new: can't allocate memory",
pe->path);
ret = -1;
goto err;
}
if (DPL_FAILURE == dpl_dict_copy(pe->usermd, dict)) {
LOG(LOG_ERR, "path=%s: dpl_dict_copy: failed", pe->path);
ret = -1;
goto err;
}
ret = 0;
err:
return ret;
}
END_TEST
START_TEST(remove_test)
{
dpl_dict_t *dict;
int i;
dpl_status_t r;
char valbuf[128];
/* create with a small table to ensure we have some chains */
dict = dpl_dict_new(5);
dpl_assert_ptr_not_null(dict);
/* add all the keys */
for (i = 0 ; i < nkeys ; i++)
{
dpl_dict_add(dict, keys[i],
make_value(i, valbuf, sizeof(valbuf)),
/* lowered */0);
}
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
/* remove the keys again */
for (i = 0 ; i < nkeys ; i++)
{
dpl_dict_var_t *var = dpl_dict_get(dict, keys[i]);
dpl_assert_ptr_not_null(var);
dpl_assert_str_eq(var->key, keys[i]);
dpl_dict_remove(dict, var);
dpl_assert_int_eq(nkeys-1-i, dpl_dict_count(dict));
}
/* add all the keys back again */
for (i = 0 ; i < nkeys ; i++)
{
dpl_dict_add(dict, keys[i],
make_value(i, valbuf, sizeof(valbuf)),
/* lowered */0);
}
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
/* remove the keys again in reverse order; we do
* this to exercise some hash chain manipulation
* corner cases */
for (i = nkeys-1 ; i >= 0 ; i--)
{
dpl_dict_var_t *var = dpl_dict_get(dict, keys[i]);
dpl_assert_ptr_not_null(var);
dpl_assert_str_eq(var->key, keys[i]);
dpl_dict_remove(dict, var);
dpl_assert_int_eq(i, dpl_dict_count(dict));
}
dpl_dict_free(dict);
}
dpl_status_t
dpl_cdmi_head_all(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
dpl_dict_t **metadatap)
{
int ret, ret2;
char *md_buf = NULL;
u_int md_len;
dpl_dict_t *metadata = NULL;
//fetch metadata from JSON content
ret2 = dpl_cdmi_get(ctx, bucket, resource, NULL != subresource ? subresource : "metadata", object_type, condition, &md_buf, &md_len, NULL);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
goto end;
}
ret2 = dpl_cdmi_parse_metadata(ctx, md_buf, md_len, metadata);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != metadatap)
{
*metadatap = metadata;
metadata = NULL;
}
ret = DPL_SUCCESS;
end:
if (NULL != metadata)
dpl_dict_free(metadata);
if (NULL != md_buf)
free(md_buf);
return ret;
}
dpl_status_t
dpl_cdmi_head(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
dpl_dict_t **metadatap)
{
int ret, ret2;
dpl_dict_t *all_mds = NULL;
dpl_dict_t *metadata = NULL;
ret2 = dpl_cdmi_head_all(ctx, bucket, resource, subresource, object_type, condition, &all_mds);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
goto end;
}
ret2 = dpl_cdmi_get_metadata_from_json_metadata(all_mds, metadata);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
if (NULL != metadatap)
{
*metadatap = metadata;
metadata = NULL;
}
ret = DPL_SUCCESS;
end:
if (NULL != metadata)
dpl_dict_free(metadata);
if (NULL != all_mds)
dpl_dict_free(all_mds);
return ret;
}
void
copy_nameless_object_with_new_md()
{
dpl_async_task_t *atask = NULL;
dpl_buf_t *buf = NULL;
dpl_status_t ret;
dpl_dict_t *metadata = NULL;
banner("11 - copy nameless object with new metadata");
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
exit(1);
}
ret = dpl_dict_add(metadata, "bar", "qux", 0);
if (DPL_SUCCESS != ret)
{
fprintf(stderr, "error updating metadatum: %s (%d)\n", dpl_status_str(ret), ret);
exit(1);
}
/*
* note: With Dewpoint, it would be possible to copy nameless object into another nameless object.
* Does it make sense ? for now we copy it into a named object
*/
atask = (dpl_async_task_t *) dpl_copy_id_async_prepare(ctx,
NULL, //no src bucket
id1, //the src resource
NULL, //no dst bucket
file1_path, //dst resource
NULL, //no option
DPL_FTYPE_REG, //regular file
DPL_COPY_DIRECTIVE_COPY, //rename
metadata, //metadata
NULL, //no sysmd
NULL); //no server side condition
if (NULL == atask)
{
fprintf(stderr, "error preparing task\n");
exit(1);
}
atask->cb_func = cb_copy_nameless_object_with_new_md;
atask->cb_arg = atask;
dpl_task_pool_put(pool, (dpl_task_t *) atask);
}
void
update_metadata_named_object()
{
dpl_async_task_t *atask = NULL;
dpl_status_t ret;
dpl_option_t option;
dpl_dict_t *metadata = NULL;
banner("6 - append metadata to existing named object");
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
exit(1);
}
ret = dpl_dict_add(metadata, "foo", "bar", 0);
if (DPL_SUCCESS != ret)
{
fprintf(stderr, "error updating metadatum: %s (%d)\n", dpl_status_str(ret), ret);
exit(1);
}
option.mask = DPL_OPTION_APPEND_METADATA;
atask = (dpl_async_task_t *) dpl_put_async_prepare(ctx,
NULL, //no bucket
file3_path, //the id
&option, //option
DPL_FTYPE_REG, //regular object
NULL, //condition
NULL, //range
metadata, //the metadata
NULL, //no sysmd
NULL); //object body
dpl_dict_free(metadata);
if (NULL == atask)
{
fprintf(stderr, "error preparing task\n");
exit(1);
}
atask->cb_func = cb_update_metadata_named_object;
atask->cb_arg = atask;
dpl_task_pool_put(pool, (dpl_task_t *) atask);
}
/**
* get metadata from values
*
* @param values
* @param metadatap
* @param sysmdp
*
* @return
*/
dpl_status_t
dpl_cdmi_get_metadata_from_values(const dpl_dict_t *values,
dpl_dict_t **metadatap,
dpl_sysmd_t *sysmdp)
{
dpl_dict_t *metadata = NULL;
dpl_status_t ret, ret2;
struct metadata_conven mc;
if (metadatap)
{
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
goto end;
}
}
memset(&mc, 0, sizeof (mc));
mc.metadata = metadata;
mc.sysmdp = sysmdp;
if (sysmdp)
sysmdp->mask = 0;
ret2 = dpl_dict_iterate(values, cb_values_iterate, &mc);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != metadatap)
{
*metadatap = metadata;
metadata = NULL;
}
ret = DPL_SUCCESS;
end:
if (NULL != metadata)
dpl_dict_free(metadata);
return ret;
}
void
copy_named_object_with_new_md()
{
dpl_async_task_t *atask = NULL;
dpl_buf_t *buf = NULL;
dpl_status_t ret;
dpl_dict_t *metadata = NULL;
banner("12 - copy named object with new metadata");
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
exit(1);
}
ret = dpl_dict_add(metadata, "qux", "baz", 0);
if (DPL_SUCCESS != ret)
{
fprintf(stderr, "error updating metadatum: %s (%d)\n", dpl_status_str(ret), ret);
exit(1);
}
atask = (dpl_async_task_t *) dpl_copy_async_prepare(ctx,
NULL, //no src bucket
file1_path, //the src resource
NULL, //no dst bucket
file4_path, //dst resource
NULL, //no option
DPL_FTYPE_REG, //regular file
DPL_COPY_DIRECTIVE_COPY, //rename
metadata, //metadata
NULL, //no sysmd
NULL); //no server side condition
if (NULL == atask)
{
fprintf(stderr, "error preparing task\n");
exit(1);
}
atask->cb_func = cb_copy_named_object_with_new_md;
atask->cb_arg = atask;
dpl_task_pool_put(pool, (dpl_task_t *) atask);
}
/**
* parse a string of the form metadata1=value1;metadata2=value2...
*
* @param metadata
*
* @return
*/
dpl_dict_t *
dpl_parse_metadata(char *metadata)
{
char *saveptr = NULL;
char *str, *tok, *p;
int ret;
dpl_dict_t *dict;
char *nmetadata;
nmetadata = strdup(metadata);
if (NULL == nmetadata)
return NULL;
dict = dpl_dict_new(13);
if (NULL == dict)
{
free(nmetadata);
return NULL;
}
for (str = metadata;;str = NULL)
{
tok = strtok_r(str, ";", &saveptr);
if (NULL == tok)
break ;
DPRINTF("tok=%s\n", tok);
p = index(tok, '=');
if (NULL == p)
p = "";
else
*p++ = 0;
ret = dpl_dict_add(dict, tok, p, 0);
if (DPL_SUCCESS != ret)
{
dpl_dict_free(dict);
free(nmetadata);
return NULL;
}
}
free(nmetadata);
return dict;
}
/**
* parse a string of the form query_params1=value1;query_params2=value2...
*
* @param query_params
*
* @return
*/
dpl_dict_t *
dpl_parse_query_params(char *query_params)
{
char *saveptr = NULL;
char *str, *tok, *p;
int ret;
dpl_dict_t *dict;
char *nquery_params;
nquery_params = strdup(query_params);
if (NULL == nquery_params)
return NULL;
dict = dpl_dict_new(13);
if (NULL == dict)
{
free(nquery_params);
return NULL;
}
for (str = query_params;;str = NULL)
{
tok = strtok_r(str, ";", &saveptr);
if (NULL == tok)
break ;
DPRINTF("tok=%s\n", tok);
p = index(tok, '=');
if (NULL == p)
p = "";
else
*p++ = 0;
ret = dpl_dict_add(dict, tok, p, 0);
if (DPL_SUCCESS != ret)
{
dpl_dict_free(dict);
free(nquery_params);
return NULL;
}
}
free(nquery_params);
return dict;
}
END_TEST
/*
* Test replacing an existing value with dpl_dict_add(
*/
START_TEST(replace_test)
{
dpl_dict_t *dict;
int i;
dpl_status_t r;
char valbuf[128];
/* strings courtesy http://hipsteripsum.me/ */
static const char key0[] = "Sriracha";
static const char val0[] = "Banksy";
static const char val0_new[] = "polaroid";
static const char key1[] = "trust";
static const char val1[] = "fund";
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
/* add the values */
dpl_dict_add(dict, key0, val0, /* lowered */0);
dpl_dict_add(dict, key1, val1, /* lowered */0);
dpl_assert_int_eq(2, dpl_dict_count(dict));
/* check the values are there */
dpl_assert_str_eq(dpl_dict_get_value(dict, key0), val0);
dpl_assert_str_eq(dpl_dict_get_value(dict, key1), val1);
/* replace one of the values */
dpl_dict_add(dict, key0, val0_new, /* lowered */0);
/* check the element count is correct */
dpl_assert_int_eq(2, dpl_dict_count(dict));
/* check the new value is there */
dpl_assert_str_eq(dpl_dict_get_value(dict, key0), val0_new);
/* check the other key is unaffected */
dpl_assert_str_eq(dpl_dict_get_value(dict, key1), val1);
dpl_dict_free(dict);
}
static dpl_status_t
cb_httpreply_header(void *cb_arg,
const char *header,
const char *value)
{
struct httreply_conven *hc = (struct httreply_conven *) cb_arg;
int ret;
if (NULL == hc->headers)
{
hc->headers = dpl_dict_new(13);
if (NULL == hc->headers)
return DPL_ENOMEM;
}
ret = dpl_dict_add(hc->headers, header, value, 1);
if (DPL_SUCCESS != ret)
return DPL_ENOMEM;
return DPL_SUCCESS;
}
static void
setup(void)
{
int r;
unsetenv("DPLDIR");
unsetenv("DPLPROFILE");
dpl_init();
r = toyserver_start(NULL, &state);
dpl_assert_int_eq(r, 0);
profile = dpl_dict_new(13);
dpl_assert_ptr_not_null(profile);
dpl_assert_int_eq(DPL_SUCCESS, dpl_dict_add(profile, "host", toyserver_addrlist(state), 0));
dpl_assert_int_eq(DPL_SUCCESS, dpl_dict_add(profile, "backend", "sproxyd", 0));
dpl_assert_int_eq(DPL_SUCCESS, dpl_dict_add(profile, "base_path", "/proxy/chord", 0));
dpl_assert_int_eq(DPL_SUCCESS, dpl_dict_add(profile, "droplet_dir", "/never/seen", 0));
dpl_assert_int_eq(DPL_SUCCESS, dpl_dict_add(profile, "profile_name", "viral", 0));
/* need this to disable the event log, otherwise the droplet_dir needs to exist */
dpl_assert_int_eq(DPL_SUCCESS, dpl_dict_add(profile, "pricing_dir", "", 0));
}
END_TEST
/*
* Use the dict with long key strings; tests
* some corner cases e.g. in the hash function.
*/
START_TEST(long_key_test)
{
#define N 512
dpl_dict_t *dict;
int i;
int j;
const char *act;
const char *exp;
char keybuf[1024];
char valbuf[1024];
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
for (i = 0 ; i < N ; i++)
{
dpl_dict_add(dict, make_key(i, keybuf, sizeof(keybuf)),
make_value(i, valbuf, sizeof(valbuf)),
/* lowered */0);
}
dpl_assert_int_eq(N, dpl_dict_count(dict));
for (i = 0 ; i < N ; i++)
{
act = dpl_dict_get_value(dict, make_key(i, keybuf, sizeof(keybuf)));
exp = make_value(i, valbuf, sizeof(valbuf));
dpl_assert_str_eq(act, exp);
}
dpl_dict_free(dict);
#undef N
}
/**
* post processing of profile, e.g. init SSL
*
* @param ctx
*
* @return
*/
dpl_status_t
dpl_profile_post(dpl_ctx_t *ctx)
{
int ret, ret2;
//sanity checks
if (NULL == ctx->host)
{
fprintf(stderr, "missing 'host' in profile\n");
ret = DPL_FAILURE;
goto end;
}
if (-1 == ctx->port)
{
if (0 == ctx->use_https)
ctx->port = 80;
else
ctx->port = 443;
}
//ssl stuff
if (1 == ctx->use_https)
{
#if OPENSSL_VERSION_NUMBER >= 0x10000000L
const SSL_METHOD *method;
#else
SSL_METHOD *method;
#endif
if (NULL == ctx->ssl_cert_file ||
NULL == ctx->ssl_key_file ||
NULL == ctx->ssl_password)
return DPL_EINVAL;
ctx->ssl_bio_err = BIO_new_fp(stderr, BIO_NOCLOSE);
if (NULL == ctx->ssl_bio_err)
{
ret = DPL_FAILURE;
goto end;
}
method = SSLv23_method();
ctx->ssl_ctx = SSL_CTX_new(method);
if (!(SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, ctx->ssl_cert_file)))
{
BIO_printf(ctx->ssl_bio_err, "use_certificate_chain_file: ");
ERR_print_errors(ctx->ssl_bio_err);
BIO_printf(ctx->ssl_bio_err, "\n");
ret = DPL_FAILURE;
goto end;
}
SSL_CTX_set_default_passwd_cb(ctx->ssl_ctx, passwd_cb);
SSL_CTX_set_default_passwd_cb_userdata(ctx->ssl_ctx, ctx);
if (!(SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, ctx->ssl_key_file, SSL_FILETYPE_PEM)))
{
BIO_printf(ctx->ssl_bio_err, "use_private_key_file: ");
ERR_print_errors(ctx->ssl_bio_err);
BIO_printf(ctx->ssl_bio_err, "\n");
ret = DPL_FAILURE;
goto end;
}
if (NULL != ctx->ssl_ca_list)
{
if (!(SSL_CTX_load_verify_locations(ctx->ssl_ctx, ctx->ssl_ca_list, 0)))
{
BIO_printf(ctx->ssl_bio_err, "load_verify_location: ");
ERR_print_errors(ctx->ssl_bio_err);
BIO_printf(ctx->ssl_bio_err, "\n");
ret = DPL_FAILURE;
goto end;
}
}
}
//pricing
if (NULL != ctx->pricing)
{
ret2 = dpl_pricing_load(ctx);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
}
//encrypt
OpenSSL_add_all_digests();
OpenSSL_add_all_ciphers();
//event log
ret2 = dpl_open_event_log(ctx);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
//connection pool
ret2 = dpl_conn_pool_init(ctx);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
ctx->cwds = dpl_dict_new(13);
if (NULL == ctx->cwds)
{
ret = DPL_FAILURE;
goto end;
}
ctx->cur_bucket = strdup("");
if (NULL == ctx->cur_bucket)
{
ret = DPL_FAILURE;
goto end;
}
ret = DPL_SUCCESS;
end:
return ret;
}
void
create_object()
{
dpl_async_task_t *atask = NULL;
dpl_buf_t *buf = NULL;
dpl_status_t ret;
buf = dpl_buf_new();
if (NULL == buf)
exit(1);
buf->size = DATA_LEN;
buf->ptr = malloc(DATA_LEN);
if (NULL == buf->ptr)
{
fprintf(stderr, "alloc data failed\n");
exit(1);
}
memset(buf->ptr, 'z', buf->size);
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
fprintf(stderr, "dpl_dict_new failed\n");
exit(1);
}
ret = dpl_dict_add(metadata, "foo", "bar", 0);
if (DPL_SUCCESS != ret)
{
fprintf(stderr, "dpl_dict_add failed\n");
exit(1);
}
ret = dpl_dict_add(metadata, "foo2", "qux", 0);
if (DPL_SUCCESS != ret)
{
fprintf(stderr, "dpl_dict_add failed\n");
exit(1);
}
fprintf(stderr, "atomic creation of an object+MD\n");
atask = (dpl_async_task_t *) dpl_post_async_prepare(ctx, //the context
NULL, //no bucket
folder, //the folder
NULL, //no option
DPL_FTYPE_REG, //regular object
NULL, //condition
NULL, //range
metadata, //the metadata
NULL, //no sysmd
buf, //object body
NULL); //no query params
if (NULL == atask)
{
fprintf(stderr, "error preparing task\n");
exit(1);
}
atask->cb_func = cb_create_object;
atask->cb_arg = atask;
dpl_task_pool_put(pool, (dpl_task_t *) atask);
}
/* return the fd of a local copy, to operate on */
int
dfs_get_local_copy(pentry_t *pe,
const char * const remote,
int flags)
{
int fd;
dpl_dict_t *metadata = NULL;
dpl_dict_t *headers = NULL;
struct get_data get_data = { .fd = -1, .buf = NULL };
dpl_status_t rc = DPL_FAILURE;
char *local = NULL;
unsigned encryption = 0;
local = tmpstr_printf("%s%s", conf->cache_dir, remote);
LOG(LOG_DEBUG, "bucket=%s, path=%s, local=%s",
ctx->cur_bucket, remote, local);
if (-1 == download_headers((char *)remote, &headers)) {
LOG(LOG_NOTICE, "%s: can't download headers", remote);
fd = -1;
goto end;
}
metadata = dpl_dict_new(13);
if (! metadata) {
LOG(LOG_ERR, "dpl_dict_new: can't allocate memory");
fd = -1;
goto end;
}
if (DPL_FAILURE == dpl_get_metadata_from_headers(
#if defined(DPL_VERSION_MAJOR) && defined(DPL_VERSION_MINOR) && (DPL_VERSION_MAJOR == 0 && DPL_VERSION_MINOR >= 2) || (DPL_VERSION_MAJOR > 0)
ctx,
#endif
headers, metadata)) {
LOG(LOG_ERR, "%s: metadata extraction failed", remote);
fd = -1;
goto end;
}
if (-1 == pentry_set_metadata(pe, metadata)) {
LOG(LOG_ERR, "can't update metadata");
fd = -1;
goto end;
}
if (-1 == check_permissions(pe, metadata)) {
LOG(LOG_NOTICE, "permission denied");
fd = -1;
goto end;
}
/* If the remote MD5 matches a cache file, we don't have to download
* it again, just return the (open) file descriptor of the cache file
*/
if (0 == compare_digests(pe, headers)) {
fd = pentry_get_fd(pe);
goto end;
}
/* a cache file already exists, its MD5 digest is different, so
* just remove it */
if (0 == access(local, F_OK)) {
LOG(LOG_DEBUG, "removing cache file '%s'", local);
if (-1 == unlink(local))
LOG(LOG_ERR, "unlink(%s): %s", local, strerror(errno));
}
get_data.fd = open(local, O_RDWR|O_CREAT|O_TRUNC, 0600);
if (-1 == get_data.fd) {
LOG(LOG_ERR, "open: %s: %s (%d)",
local, strerror(errno), errno);
fd = -1;
goto end;
}
encryption = check_encryption_flag(metadata);
rc = dpl_openread(ctx,
(char *)remote,
encryption,
NULL,
cb_get_buffered,
&get_data,
&metadata);
if (DPL_SUCCESS != rc) {
LOG(LOG_ERR, "dpl_openread: %s", dpl_status_str(rc));
close(get_data.fd);
fd = -1;
goto end;
}
/* If the file is compressed, uncompress it! */
if(-1 == handle_compression(remote, local, &get_data, metadata)) {
fd = -1;
goto end;
}
if (-1 == close(get_data.fd)) {
LOG(LOG_ERR, "close(path=%s, fd=%d): %s",
local, get_data.fd, strerror(errno));
fd = -1;
goto end;
}
fd = open(local, flags, 0600);
if (-1 == fd) {
LOG(LOG_ERR, "open(path=%s, fd=%d): %s",
local, fd, strerror(errno));
fd = -1;
goto end;
}
end:
if (metadata)
dpl_dict_free(metadata);
if (headers)
dpl_dict_free(headers);
return fd;
}
dpl_status_t
dpl_s3_req_gen_url(const dpl_req_t *req,
dpl_dict_t *headers,
char *buf, int len,
unsigned int *lenp)
{
int bucket;
char resource_ue[DPL_URL_LENGTH(strlen(req->resource)) + 1], *p;
dpl_status_t ret;
dpl_dict_t *query_params;
unsigned char is_first_param;
DPL_TRACE(req->ctx, DPL_TRACE_REQ, "req_gen_query_string");
query_params = dpl_dict_new(32);
if (query_params == NULL)
return DPL_FAILURE;
if (req->resource[0] != '/') {
resource_ue[0] = '/';
dpl_url_encode(req->resource, resource_ue + 1);
} else
dpl_url_encode(req->resource + 1, resource_ue);
p = buf;
if (req->ctx->use_https)
DPL_APPEND_STR("https");
else
DPL_APPEND_STR("http");
DPL_APPEND_STR("://");
DPL_APPEND_STR(req->host);
if (( req->ctx->use_https && strcmp(req->port, "443")) ||
(!req->ctx->use_https && strcmp(req->port, "80"))) {
DPL_APPEND_STR(":");
DPL_APPEND_STR(req->port);
}
DPL_APPEND_STR(resource_ue);
if (req->ctx->aws_auth_sign_version == 2)
ret = dpl_s3_get_authorization_v2_params(req, query_params, NULL);
else if (req->ctx->aws_auth_sign_version == 4)
ret = dpl_s3_get_authorization_v4_params(req, query_params, NULL);
else
ret = DPL_FAILURE;
is_first_param = 1;
for (bucket = 0; bucket < query_params->n_buckets; bucket++) {
dpl_dict_var_t *param = query_params->buckets[bucket];
while (param != NULL) {
if (ret == DPL_SUCCESS) {
if (is_first_param) {
DPL_APPEND_STR("?");
is_first_param = 0;
} else
DPL_APPEND_STR("&");
DPL_APPEND_STR(param->key);
DPL_APPEND_STR("=");
DPL_APPEND_STR(dpl_sbuf_get_str(param->val->string));
}
param = param->prev;
}
}
if (lenp != NULL)
*lenp = (p - buf);
dpl_dict_free(query_params);
return ret;
}
dpl_status_t
dpl_posix_head_raw(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
const dpl_option_t *option,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
dpl_dict_t **metadatap,
char **locationp)
{
dpl_status_t ret, ret2;
char path[MAXPATHLEN];
int iret;
struct stat st;
char buf[256];
dpl_dict_t *metadata = NULL;
dpl_dict_t *subdict = NULL;
dpl_value_t value;
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");
snprintf(path, sizeof (path), "/%s/%s",
ctx->base_path ? ctx->base_path : "",
resource ? resource : "");
iret = stat(path, &st);
if (-1 == iret)
{
ret = dpl_posix_map_errno();
goto end;
}
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
goto end;
}
snprintf(buf, sizeof (buf), "%ld", st.st_dev);
ret2 = dpl_dict_add(metadata, "dev", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lX", st.st_ino);
ret2 = dpl_dict_add(metadata, "ino", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%u", st.st_mode);
ret2 = dpl_dict_add(metadata, "mode", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%ld", st.st_nlink);
ret2 = dpl_dict_add(metadata, "nlink", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%u", st.st_uid);
ret2 = dpl_dict_add(metadata, "uid", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%u", st.st_gid);
ret2 = dpl_dict_add(metadata, "gid", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_rdev);
ret2 = dpl_dict_add(metadata, "rdev", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_size);
ret2 = dpl_dict_add(metadata, "size", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_blksize);
ret2 = dpl_dict_add(metadata, "blksize", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_blocks);
ret2 = dpl_dict_add(metadata, "blocks", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_atime);
ret2 = dpl_dict_add(metadata, "atime", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_mtime);
ret2 = dpl_dict_add(metadata, "mtime", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_ctime);
ret2 = dpl_dict_add(metadata, "ctime", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
subdict = dpl_dict_new(13);
if (NULL == subdict)
{
ret = DPL_ENOMEM;
goto end;
}
ret2 = dpl_get_xattrs(path,
subdict,
DPL_POSIX_XATTR_PREFIX,
XATTRS_NO_ENCODING);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
value.type = DPL_VALUE_SUBDICT;
value.subdict = subdict;
// dpl_dict_add_value dups the value, so don't prevent freeing subdict
ret2 = dpl_dict_add_value(metadata, "xattr", &value, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != metadatap)
{
*metadatap = metadata;
metadata = NULL;
}
ret = DPL_SUCCESS;
end:
if (NULL != subdict)
dpl_dict_free(subdict);
if (NULL != metadata)
dpl_dict_free(metadata);
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);
return ret;
}
dpl_status_t
dpl_s3_get_buffered(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
const dpl_option_t *option,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
const dpl_range_t *range,
dpl_metadatum_func_t metadatum_func,
dpl_dict_t **metadatap,
dpl_sysmd_t *sysmdp,
dpl_buffer_func_t buffer_func,
void *cb_arg,
char **locationp)
{
int ret, ret2;
dpl_conn_t *conn = NULL;
char header[dpl_header_size];
u_int header_len;
struct iovec iov[10];
int n_iov = 0;
dpl_dict_t *headers_request = NULL;
dpl_req_t *req = NULL;
struct get_conven gc;
int http_status;
dpl_s3_req_mask_t req_mask = 0u;
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");
memset(&gc, 0, sizeof (gc));
gc.metadata = dpl_dict_new(13);
if (NULL == gc.metadata)
{
ret = DPL_ENOMEM;
goto end;
}
gc.sysmdp = sysmdp;
gc.metadatum_func = metadatum_func;
gc.buffer_func = buffer_func;
gc.cb_arg = cb_arg;
req = dpl_req_new(ctx);
if (NULL == req)
{
ret = DPL_ENOMEM;
goto end;
}
dpl_req_set_method(req, DPL_METHOD_GET);
if (NULL == bucket)
{
ret = DPL_EINVAL;
goto end;
}
ret2 = dpl_req_set_bucket(req, bucket);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_set_resource(req, resource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != subresource)
{
ret2 = dpl_req_set_subresource(req, subresource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (NULL != condition)
{
dpl_req_set_condition(req, condition);
}
//build request
ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_try_connect(ctx, req, &conn);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_add_host_to_headers(req, headers_request);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
iov[n_iov].iov_base = header;
iov[n_iov].iov_len = header_len;
n_iov++;
//final crlf
iov[n_iov].iov_base = "\r\n";
iov[n_iov].iov_len = 2;
n_iov++;
ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
if (DPL_SUCCESS != ret2)
{
DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
gc.connection_close = 1;
ret = ret2;
goto end;
}
ret2 = dpl_read_http_reply_buffered(conn, 1, &http_status, cb_get_header, cb_get_buffer, &gc);
if (DPL_SUCCESS != ret2)
{
DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "read http answer failed");
gc.connection_close = 1;
ret = ret2;
goto end;
}
if (!conn->ctx->keep_alive)
gc.connection_close = 1;
if (NULL != metadatap)
{
*metadatap = gc.metadata;
gc.metadata = NULL;
}
//map http_status to relevant value
ret = dpl_map_http_status(http_status);
//caller is responsible for logging the event
end:
if (NULL != conn)
{
if (1 == gc.connection_close)
dpl_conn_terminate(conn);
else
dpl_conn_release(conn);
}
if (NULL != gc.metadata)
dpl_dict_free(gc.metadata);
if (NULL != headers_request)
dpl_dict_free(headers_request);
if (NULL != req)
dpl_req_free(req);
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);
return ret;
}
dpl_status_t
dpl_s3_list_bucket(dpl_ctx_t *ctx,
const char *bucket,
const char *prefix,
const char *delimiter,
const int max_keys,
dpl_vec_t **objectsp,
dpl_vec_t **common_prefixesp,
char **locationp)
{
int ret, ret2;
dpl_conn_t *conn = NULL;
char header[dpl_header_size];
u_int header_len;
struct iovec iov[10];
int n_iov = 0;
int connection_close = 0;
char *data_buf = NULL;
u_int data_len;
dpl_vec_t *objects = NULL;
dpl_vec_t *common_prefixes = NULL;
dpl_dict_t *query_params = NULL;
dpl_dict_t *headers_request = NULL;
dpl_dict_t *headers_reply = NULL;
dpl_req_t *req = NULL;
dpl_s3_req_mask_t req_mask = 0u;
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");
req = dpl_req_new(ctx);
if (NULL == req)
{
ret = DPL_ENOMEM;
goto end;
}
dpl_req_set_method(req, DPL_METHOD_GET);
if (NULL == bucket)
{
ret = DPL_EINVAL;
goto end;
}
ret2 = dpl_req_set_bucket(req, bucket);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_set_resource(req, "/");
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
query_params = dpl_dict_new(13);
if (NULL == query_params)
{
ret = DPL_ENOMEM;
goto end;
}
if (NULL != prefix)
{
ret2 = dpl_dict_add(query_params, "prefix", prefix, 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
if (NULL != delimiter)
{
ret2 = dpl_dict_add(query_params, "delimiter", delimiter, 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
if (-1 != max_keys)
{
char tmp[32] = "";
snprintf(tmp, sizeof tmp, "%d", max_keys);
ret2 = dpl_dict_add(query_params, "max-keys", tmp, 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_try_connect(ctx, req, &conn);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_add_host_to_headers(req, headers_request);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_gen_http_request(ctx, req, headers_request, query_params, header, sizeof (header), &header_len);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
iov[n_iov].iov_base = header;
iov[n_iov].iov_len = header_len;
n_iov++;
//final crlf
iov[n_iov].iov_base = "\r\n";
iov[n_iov].iov_len = 2;
n_iov++;
ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
if (DPL_SUCCESS != ret2)
{
DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
connection_close = 1;
ret = ret2;
goto end;
}
ret2 = dpl_read_http_reply(conn, 1, &data_buf, &data_len, &headers_reply, &connection_close);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
objects = dpl_vec_new(2, 2);
if (NULL == objects)
{
ret = DPL_ENOMEM;
goto end;
}
common_prefixes = dpl_vec_new(2, 2);
if (NULL == common_prefixes)
{
ret = DPL_ENOMEM;
goto end;
}
ret = dpl_s3_parse_list_bucket(ctx, data_buf, data_len, objects, common_prefixes);
if (DPL_SUCCESS != ret)
{
ret = ret2;
goto end;
}
if (NULL != objectsp)
{
*objectsp = objects;
objects = NULL; //consume it
}
if (NULL != common_prefixesp)
{
*common_prefixesp = common_prefixes;
common_prefixes = NULL; //consume it
}
ret = DPL_SUCCESS;
end:
if (NULL != objects)
dpl_vec_objects_free(objects);
if (NULL != common_prefixes)
dpl_vec_common_prefixes_free(common_prefixes);
if (NULL != data_buf)
free(data_buf);
if (NULL != conn)
{
if (1 == connection_close)
dpl_conn_terminate(conn);
else
dpl_conn_release(conn);
}
if (NULL != query_params)
dpl_dict_free(query_params);
if (NULL != headers_reply)
dpl_dict_free(headers_reply);
if (NULL != headers_request)
dpl_dict_free(headers_request);
if (NULL != req)
dpl_req_free(req);
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);
return ret;
}
END_TEST
/*
* Test the dpl_dict_filter_prefix() and
* dpl_dict_filter_no_prefix() functions.
*/
START_TEST(filter_test)
{
dpl_dict_t *dict;
dpl_dict_t *d2;
int i;
dpl_status_t r;
const char *exp;
const char *act;
char valbuf[128];
/* strings courtesy http://hipsteripsum.me/ */
static const char * const keys[] = {
"Sriracha", "mehBanksy", "trust", "fund", "Brooklyn",
"mehpolaroid", "Viral", "mehselfies", "mehkogi", "Austin" };
static const int nkeys = sizeof(keys)/sizeof(keys[0]);
static const int is_meh[] = { 0, 1, 0, 0, 0,
1, 0, 1, 1, 0 };
static const int nmehs = 4;
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
/* add all the keys */
for (i = 0 ; i < nkeys ; i++)
{
dpl_dict_add(dict, keys[i],
make_value(i, valbuf, sizeof(valbuf)),
/* lowered */0);
}
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
/* create a new dict and copy everything across using an empty prefix */
d2 = dpl_dict_new(13);
r = dpl_dict_filter_prefix(d2, dict, "");
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
dpl_assert_int_eq(nkeys, dpl_dict_count(d2));
for (i = 0 ; i < nkeys ; i++)
{
exp = make_value(i, valbuf, sizeof(valbuf));
act = dpl_dict_get_value(d2, keys[i]);
dpl_assert_str_eq(exp, act);
}
dpl_dict_free(d2);
/* create a new dict and copy just the matching ones */
d2 = dpl_dict_new(13);
r = dpl_dict_filter_prefix(d2, dict, "meh");
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
dpl_assert_int_eq(nmehs, dpl_dict_count(d2));
for (i = 0 ; i < nkeys ; i++)
{
if (is_meh[i])
{
exp = make_value(i, valbuf, sizeof(valbuf));
/*
* dpl_dict_filter_prefix() inexplicably removes the prefix
* from the keys it copies across. This was probably very
* convenient for the person who wrote the code, but isn't
* all that easy to explain. But this is why we add a
* strlen() here.
*/
act = dpl_dict_get_value(d2, keys[i]+strlen("meh"));
dpl_assert_str_eq(exp, act);
}
}
dpl_dict_free(d2);
/* create a new dict and copy just the non-matching ones */
d2 = dpl_dict_new(13);
r = dpl_dict_filter_no_prefix(d2, dict, "meh");
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
dpl_assert_int_eq(nkeys-nmehs, dpl_dict_count(d2));
for (i = 0 ; i < nkeys ; i++)
{
if (!is_meh[i])
{
exp = make_value(i, valbuf, sizeof(valbuf));
act = dpl_dict_get_value(d2, keys[i]);
dpl_assert_str_eq(exp, act);
}
}
dpl_dict_free(d2);
dpl_dict_free(dict);
}
dpl_status_t
dpl_posix_get_buffered(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
const dpl_option_t *option,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
const dpl_range_t *range,
dpl_metadatum_func_t metadatum_func,
dpl_dict_t **metadatap,
dpl_sysmd_t *sysmdp,
dpl_buffer_func_t buffer_func,
void *cb_arg,
char **locationp)
{
dpl_status_t ret, ret2;
char path[MAXPATHLEN];
int fd = -1;
int iret;
char buf[8192];
ssize_t cc;
dpl_dict_t *all_mds = NULL;
struct get_conven gc;
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");
memset(&gc, 0, sizeof (gc));
gc.metadata = dpl_dict_new(13);
if (NULL == gc.metadata)
{
ret = DPL_ENOMEM;
goto end;
}
gc.sysmdp = sysmdp;
gc.metadatum_func = metadatum_func;
gc.buffer_func = buffer_func;
gc.cb_arg = cb_arg;
snprintf(path, sizeof (path), "/%s/%s", ctx->base_path ? ctx->base_path : "", resource);
switch (object_type)
{
case DPL_FTYPE_UNDEF:
case DPL_FTYPE_CAP:
case DPL_FTYPE_DOM:
case DPL_FTYPE_DIR:
case DPL_FTYPE_CHRDEV:
case DPL_FTYPE_BLKDEV:
case DPL_FTYPE_FIFO:
case DPL_FTYPE_SOCKET:
case DPL_FTYPE_SYMLINK:
ret = DPL_EINVAL;
goto end;
case DPL_FTYPE_ANY:
case DPL_FTYPE_REG:
fd = open(path, O_RDONLY);
if (-1 == fd)
{
perror("open");
ret = DPL_FAILURE;
goto end;
}
break ;
}
if (range)
{
iret = lseek(fd, range->start, SEEK_SET);
if (-1 == iret)
{
perror("lseek");
ret = DPL_FAILURE;
goto end;
}
}
ret2 = dpl_posix_head_raw(ctx, bucket, resource, subresource, option,
object_type, condition, &all_mds, locationp);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_dict_iterate(all_mds, cb_get_value, &gc);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
while (1)
{
cc = read(fd, buf, sizeof (buf));
if (-1 == cc)
{
perror("read");
ret = DPL_FAILURE;
goto end;
}
ret2 = cb_get_buffer(&gc, buf, cc);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (0 == cc)
break ;
}
if (NULL != metadatap)
{
*metadatap = gc.metadata;
gc.metadata = NULL;
}
ret = DPL_SUCCESS;
end:
if (NULL != all_mds)
dpl_dict_free(all_mds);
if (NULL != gc.metadata)
dpl_dict_free(gc.metadata);
if (-1 == fd)
close(fd);
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);
return ret;
}
static dpl_status_t
convert_obj_to_value(dpl_ctx_t *ctx,
struct json_object *obj,
int level,
dpl_value_t **valp)
{
int ret, ret2;
dpl_value_t *val = NULL;
char *key;
struct lh_entry *entry;
json_object *child;
dpl_dict_t *subdict = NULL;
dpl_vec_t *vector = NULL;
DPRINTF("convert_obj_to_value level=%d type=%d\n", level, json_object_get_type(obj));
val = malloc(sizeof (*val));
if (NULL == val)
{
ret = DPL_ENOMEM;
goto end;
}
memset(val, 0, sizeof (*val));
switch (json_object_get_type(obj))
{
case json_type_null:
return DPL_ENOTSUPP;
case json_type_array:
{
int n_items = json_object_array_length(obj);
int i;
vector = dpl_vec_new(2, 2);
if (NULL == vector)
{
ret = DPL_ENOMEM;
goto end;
}
for (i = 0;i < n_items;i++)
{
child = json_object_array_get_idx(obj, i);
dpl_value_t *subval;
ret2 = convert_obj_to_value(ctx, child, level+1, &subval);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_vec_add_value(vector, subval);
dpl_value_free(subval);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
val->type = DPL_VALUE_VECTOR;
val->vector = vector;
vector = NULL;
break ;
}
case json_type_object:
{
subdict = dpl_dict_new(13);
if (NULL == subdict)
{
ret = DPL_ENOMEM;
goto end;
}
for (entry = json_object_get_object(obj)->head; (entry ? (key = (char*)entry->k, child = (struct json_object*)entry->v, entry) : 0); entry = entry->next)
{
dpl_value_t *subval;
DPRINTF("key='%s'\n", key);
ret2 = convert_obj_to_value(ctx, child, level+1, &subval);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_dict_add_value(subdict, key, subval, 0);
dpl_value_free(subval);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
val->type = DPL_VALUE_SUBDICT;
val->subdict = subdict;
subdict = NULL;
break ;
}
case json_type_boolean:
case json_type_double:
case json_type_int:
case json_type_string:
{
pthread_mutex_lock(&ctx->lock); //lock for objects other than string
val->string = strdup((char *) json_object_get_string(obj));
pthread_mutex_unlock(&ctx->lock);
if (NULL == val->string)
{
ret = DPL_ENOMEM;
goto end;
}
val->type = DPL_VALUE_STRING;
break ;
}
}
if (NULL != valp)
{
*valp = val;
val = NULL;
}
ret = DPL_SUCCESS;
end:
if (NULL != vector)
dpl_vec_free(vector);
if (NULL != subdict)
dpl_dict_free(subdict);
if (NULL != val)
dpl_value_free(val);
DPRINTF("level=%d ret=%d\n", level, ret);
return ret;
}
/**
* build headers from request
*
* @param req
* @param headersp
*
* @return
*/
dpl_status_t
dpl_s3_req_build(const dpl_req_t *req,
dpl_s3_req_mask_t req_mask,
dpl_dict_t **headersp)
{
dpl_dict_t *headers = NULL;
int ret, ret2;
const char *method = dpl_method_str(req->method);
char resource_ue[DPL_URL_LENGTH(strlen(req->resource) + (req->subresource ? strlen(req->subresource) : 0)) + 1];
DPL_TRACE(req->ctx, DPL_TRACE_REQ, "req_build method=%s bucket=%s resource=%s subresource=%s", method, req->bucket, req->resource, req->subresource);
//resource
if ('/' != req->resource[0])
{
resource_ue[0] = '/';
dpl_url_encode(req->resource, resource_ue + 1);
}
else
{
resource_ue[0] = '/'; //some servers do not like encoded slash
dpl_url_encode(req->resource + 1, resource_ue + 1);
}
// Append subresource
if (req->subresource)
{
dpl_url_encode("?", resource_ue + strlen(resource_ue));
dpl_url_encode(req->subresource, resource_ue + strlen(resource_ue));
}
headers = dpl_dict_new(13);
if (NULL == headers)
{
ret = DPL_ENOMEM;
goto end;
}
/*
* per method headers
*/
if (DPL_METHOD_GET == req->method ||
DPL_METHOD_HEAD == req->method)
{
if (req->range_enabled)
{
ret2 = dpl_add_range_to_headers(&req->range, headers);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
ret2 = add_conditions_to_headers(&req->condition, headers, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
else if (DPL_METHOD_PUT == req->method ||
DPL_METHOD_POST == req->method)
{
char buf[64];
if (NULL != req->cache_control)
{
ret2 = dpl_dict_add(headers, "Cache-Control", req->cache_control, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (NULL != req->content_disposition)
{
ret2 = dpl_dict_add(headers, "Content-Disposition", req->content_disposition, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (NULL != req->content_encoding)
{
ret2 = dpl_dict_add(headers, "Content-Encoding", req->content_encoding, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (req->behavior_flags & DPL_BEHAVIOR_MD5)
{
MD5_CTX ctx;
u_char digest[MD5_DIGEST_LENGTH];
char b64_digest[DPL_BASE64_LENGTH(MD5_DIGEST_LENGTH) + 1];
u_int b64_digest_len;
if (!req->data_enabled)
{
ret = DPL_EINVAL;
goto end;
}
MD5_Init(&ctx);
MD5_Update(&ctx, req->data_buf, req->data_len);
MD5_Final(digest, &ctx);
b64_digest_len = dpl_base64_encode(digest, MD5_DIGEST_LENGTH, (u_char *) b64_digest);
b64_digest[b64_digest_len] = 0;
ret2 = dpl_dict_add(headers, "Content-MD5", b64_digest, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (req->data_enabled)
{
snprintf(buf, sizeof (buf), "%u", req->data_len);
ret2 = dpl_dict_add(headers, "Content-Length", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
if (NULL != req->content_type)
{
ret2 = dpl_dict_add(headers, "Content-Type", req->content_type, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (req->behavior_flags & DPL_BEHAVIOR_EXPECT)
{
ret2 = dpl_dict_add(headers, "Expect", "100-continue", 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
if (DPL_CANNED_ACL_UNDEF != req->canned_acl)
{
char *str;
str = dpl_canned_acl_str(req->canned_acl);
if (NULL == str)
{
ret = DPL_FAILURE;
goto end;
}
ret2 = dpl_dict_add(headers, "x-amz-acl", str, 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
ret2 = add_metadata_to_headers(req->metadata, headers);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (DPL_STORAGE_CLASS_UNDEF != req->storage_class)
{
char *str;
str = dpl_storage_class_str(req->storage_class);
if (NULL == str)
{
ret = DPL_FAILURE;
goto end;
}
ret2 = dpl_dict_add(headers, "x-amz-storage-class", str, 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
/*
* copy
*/
if (req_mask & DPL_S3_REQ_COPY)
{
ret2 = add_source_to_headers(req, headers);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (DPL_COPY_DIRECTIVE_UNDEF != req->copy_directive)
{
char *str;
switch (req->copy_directive)
{
case DPL_COPY_DIRECTIVE_UNDEF:
case DPL_COPY_DIRECTIVE_COPY:
case DPL_COPY_DIRECTIVE_LINK:
case DPL_COPY_DIRECTIVE_SYMLINK:
case DPL_COPY_DIRECTIVE_MOVE:
case DPL_COPY_DIRECTIVE_MKDENT:
case DPL_COPY_DIRECTIVE_RMDENT:
case DPL_COPY_DIRECTIVE_MVDENT:
ret = DPL_ENOTSUPP;
goto end;
case DPL_COPY_DIRECTIVE_METADATA_REPLACE:
str = "REPLACE";
break ;
}
ret2 = dpl_dict_add(headers, "x-amz-metadata-directive", str, 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
ret2 = add_conditions_to_headers(&req->copy_source_condition, headers, 1);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
}
else if (DPL_METHOD_DELETE == req->method)
{
//XXX todo x-amz-mfa
}
else
{
ret = DPL_EINVAL;
goto end;
}
/*
* common headers
*/
if (req->behavior_flags & DPL_BEHAVIOR_KEEP_ALIVE)
{
ret2 = dpl_dict_add(headers, "Connection", "Keep-Alive", 0);
if (DPL_SUCCESS != ret2)
{
ret = DPL_ENOMEM;
goto end;
}
}
ret2 = add_date_to_headers(headers);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != headersp)
{
*headersp = headers;
headers = NULL; //consume it
}
ret = DPL_SUCCESS;
end:
if (NULL != headers)
dpl_dict_free(headers);
return ret;
}
END_TEST
/*
* Test the "lowered" feature, which is almost but not quite
* the same as case insensitivity.
*/
START_TEST(lowered_test)
{
dpl_dict_t *dict;
dpl_status_t r;
const char *s;
dpl_dict_var_t *var;
const char value[] = "World";
const char value2[] = "Mondo";
const char value3[] = "Monde";
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
/* Add a value, lowered */
r = dpl_dict_add(dict, "Hello", value, /*lowered*/1);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(1, dpl_dict_count(dict));
/* The actual key used is internally lowercased, so
* doing a normal get on the key originally given
* should fail */
dpl_assert_str_eq(NULL, dpl_dict_get_value(dict, "Hello"));
/* Likewise, doing a normal get on a lowercase version
* of the original key should succeed */
dpl_assert_str_eq(value, dpl_dict_get_value(dict, "hello"));
/* dpl_dict_get_lowered internally lowercases the key it's
* given, so it can be used with keys of any casing */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "hello", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "Hello", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "HELLO", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "hElLo", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
/* check that dpl_dict_get_lowered() will report as missing
* some keys that we know not to be present */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "HellonEarth", &var);
dpl_assert_int_eq(DPL_ENOENT, r);
/* the value of `var' on failure is not documented */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "Hell", &var);
dpl_assert_int_eq(DPL_ENOENT, r);
/* the value of `var' on failure is not documented */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "daffyduck", &var);
dpl_assert_int_eq(DPL_ENOENT, r);
/* the value of `var' on failure is not documented */
/* Verify that inserting another key which maps to the
* same lowercased string, replaces the first key */
r = dpl_dict_add(dict, "hello", value2, /*lowered*/1);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(1, dpl_dict_count(dict));
dpl_assert_str_eq(value2, dpl_dict_get_value(dict, "hello"));
r = dpl_dict_add(dict, "hELLo", value3, /*lowered*/1);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(1, dpl_dict_count(dict));
dpl_assert_str_eq(value3, dpl_dict_get_value(dict, "hello"));
dpl_dict_free(dict);
}
int
dfs_create(const char *path,
mode_t mode,
struct fuse_file_info *info)
{
tfs_ctx *ctx = fuse_get_context()->private_data;
int ret = -1;
dpl_status_t rc = DPL_FAILURE;
tpath_entry *pe = NULL;
struct stat st;
dpl_dict_t *usermd = NULL;
int exclude;
LOG(LOG_DEBUG, "%s, mode=0x%x, %s",
path, (unsigned)mode, flags_to_str(info->flags));
if (! S_ISREG(mode)) {
LOG(LOG_ERR, "%s: not a regular file", path);
ret = -1;
goto err;
}
exclude = re_matcher(&ctx->conf->regex, path);
if (-1 == dfs_open(path, info)) {
ret = -1;
goto err;
}
pe = (tpath_entry *) info->fh;
if (! pe) {
ret = -1;
goto err;
}
pe->exclude = exclude;
if (-1 == pe->fd) {
ret = -1;
goto err;
}
if (-1 == fchmod(pe->fd, mode)) {
LOG(LOG_ERR, "fchmod(fd=%d): %s", pe->fd, strerror(errno));
ret = -errno;
goto err;
}
if (-1 == fstat(pe->fd, &st)) {
LOG(LOG_ERR, "fstat(fd=%d): %s", pe->fd, strerror(errno));
ret = -errno;
goto err;
}
usermd = pe->usermd;
if (! usermd) {
usermd = dpl_dict_new(13);
if (! usermd) {
LOG(LOG_ERR, "allocation failure");
ret = -1;
goto err;
}
}
fill_metadata_from_stat(usermd, &st);
assign_meta_to_dict(usermd, "scal_mode", (unsigned long) mode);
pentry_md_lock(pe);
pentry_set_usermd(pe, usermd);
pentry_md_unlock(pe);
if (! exclude) {
rc = dfs_mknod_timeout(ctx, path);
if (DPL_SUCCESS != rc) {
LOG(LOG_ERR, "dfs_mknod_timeout: %s", dpl_status_str(rc));
ret = -1;
goto err;
}
}
ret = 0;
err:
if (usermd)
dpl_dict_free(usermd);
LOG(LOG_DEBUG, "path=%s ret=%s", path, dpl_status_str(ret));
return ret;
}
dpl_status_t
dpl_posix_head_raw(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
const dpl_option_t *option,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
dpl_dict_t **metadatap,
char **locationp)
{
dpl_status_t ret, ret2;
char path[MAXPATHLEN];
int iret;
struct stat st;
char buf[256];
dpl_dict_t *metadata = NULL;
char xattr[64*1024];
ssize_t ssize_ret, off;
dpl_dict_t *subdict = NULL;
dpl_value_t value;
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");
snprintf(path, sizeof (path), "/%s/%s", ctx->base_path ? ctx->base_path : "", resource);
iret = stat(path, &st);
if (-1 == iret)
{
ret = DPL_FAILURE;
goto end;
}
metadata = dpl_dict_new(13);
if (NULL == metadata)
{
ret = DPL_ENOMEM;
goto end;
}
snprintf(buf, sizeof (buf), "%ld", st.st_dev);
ret2 = dpl_dict_add(metadata, "dev", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lX", st.st_ino);
ret2 = dpl_dict_add(metadata, "ino", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%u", st.st_mode);
ret2 = dpl_dict_add(metadata, "mode", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%ld", st.st_nlink);
ret2 = dpl_dict_add(metadata, "nlink", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%u", st.st_uid);
ret2 = dpl_dict_add(metadata, "uid", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%u", st.st_gid);
ret2 = dpl_dict_add(metadata, "gid", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_rdev);
ret2 = dpl_dict_add(metadata, "rdev", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_size);
ret2 = dpl_dict_add(metadata, "size", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_blksize);
ret2 = dpl_dict_add(metadata, "blksize", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_blocks);
ret2 = dpl_dict_add(metadata, "blocks", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_atime);
ret2 = dpl_dict_add(metadata, "atime", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_mtime);
ret2 = dpl_dict_add(metadata, "mtime", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
snprintf(buf, sizeof (buf), "%lu", st.st_ctime);
ret2 = dpl_dict_add(metadata, "ctime", buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
subdict = dpl_dict_new(13);
if (NULL == subdict)
{
ret = DPL_ENOMEM;
goto end;
}
ssize_ret = llistxattr(path, xattr, sizeof (xattr));
if (-1 == ssize_ret)
{
ret = DPL_FAILURE;
goto end;
}
off = 0;
while (off < ssize_ret)
{
char *key;
int key_len;
ssize_t val_len;
key = (xattr + off);
key_len = strlen(key);
val_len = lgetxattr(path, key, buf, sizeof (buf) - 1);
if (val_len == -1)
{
ret = DPL_FAILURE;
goto end;
}
if (strncmp(key, XATTR_PREFIX, strlen(XATTR_PREFIX)))
{
ret = DPL_EINVAL;
goto end;
}
buf[val_len] = 0;
ret2 = dpl_dict_add(subdict, key + strlen(XATTR_PREFIX), buf, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
off += key_len + 1;
}
value.type = DPL_VALUE_SUBDICT;
value.subdict = subdict;
subdict = NULL;
ret2 = dpl_dict_add_value(metadata, "xattr", &value, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != metadatap)
{
*metadatap = metadata;
metadata = NULL;
}
ret = DPL_SUCCESS;
end:
if (NULL != subdict)
dpl_dict_free(subdict);
if (NULL != metadata)
dpl_dict_free(metadata);
DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);
return ret;
}