static unsigned int
check_encryption_flag(dpl_dict_t *metadata)
{
char *cipher = NULL;
char *cipher_type = NULL;
unsigned int ret = 0;
if (strncasecmp(conf->encryption_method, "aes", strlen("aes")))
goto end;
cipher = dpl_dict_get_value(metadata, "cipher");
cipher_type = dpl_dict_get_value(metadata, "cipher-type");
if (! cipher || ! cipher_type) {
ret = 0;
goto end;
}
if (strncasecmp(cipher, "yes", strlen("yes")))
ret = 0;
goto end;
if (strncasecmp(cipher_type, "aes-256-cfb", strlen("aes-256-cfb"))) {
LOG(LOG_ERR, "unsupported cipher-type: %s", cipher_type);
ret = 0;
goto end;
}
ret |= DPL_VFILE_FLAG_ENCRYPT;
end:
LOG(LOG_DEBUG, "flags = 0x%x", ret);
return ret;
}
dpl_status_t
dpl_s3_add_authorization_v2_to_headers(const dpl_req_t *req,
dpl_dict_t *headers,
const dpl_dict_t UNUSED *query_params,
struct tm UNUSED *tm)
{
int ret;
const char *method = dpl_method_str(req->method);
char resource_ue[DPL_URL_LENGTH(strlen(req->resource)) + 1];
char sign_str[1024];
u_int sign_len;
char hmac_str[1024];
u_int hmac_len;
char base64_str[1024];
u_int base64_len;
char auth_str[1024];
char *date_str = NULL;
//resource
if ('/' != req->resource[0]) {
resource_ue[0] = '/';
dpl_url_encode_no_slashes(req->resource, resource_ue + 1);
} else
dpl_url_encode_no_slashes(req->resource, resource_ue);
date_str = dpl_dict_get_value(headers, "x-amz-date");
if (date_str == NULL)
date_str = dpl_dict_get_value(headers, "Date");
ret = dpl_s3_make_signature_v2(req->ctx, method, req->bucket, resource_ue, req->subresource,
date_str, headers,
sign_str, sizeof (sign_str), &sign_len);
if (DPL_SUCCESS != ret)
return DPL_FAILURE;
/* DPL_TRACE(req->ctx, DPL_TRACE_REQ, "stringtosign=%.*s", sign_len, sign_str); */
hmac_len = dpl_hmac_sha1(req->ctx->secret_key, strlen(req->ctx->secret_key), sign_str, sign_len, hmac_str);
base64_len = dpl_base64_encode((const u_char *) hmac_str, hmac_len, (u_char *) base64_str);
snprintf(auth_str, sizeof (auth_str), "AWS %s:%.*s", req->ctx->access_key, base64_len, base64_str);
return dpl_dict_add(headers, "Authorization", auth_str, 0);
}
END_TEST
/*
* Test replacing an existing value with dpl_dict_add(
*/
START_TEST(replace_test)
{
dpl_dict_t *dict;
int i;
dpl_status_t r;
char valbuf[128];
/* strings courtesy http://hipsteripsum.me/ */
static const char key0[] = "Sriracha";
static const char val0[] = "Banksy";
static const char val0_new[] = "polaroid";
static const char key1[] = "trust";
static const char val1[] = "fund";
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
/* add the values */
dpl_dict_add(dict, key0, val0, /* lowered */0);
dpl_dict_add(dict, key1, val1, /* lowered */0);
dpl_assert_int_eq(2, dpl_dict_count(dict));
/* check the values are there */
dpl_assert_str_eq(dpl_dict_get_value(dict, key0), val0);
dpl_assert_str_eq(dpl_dict_get_value(dict, key1), val1);
/* replace one of the values */
dpl_dict_add(dict, key0, val0_new, /* lowered */0);
/* check the element count is correct */
dpl_assert_int_eq(2, dpl_dict_count(dict));
/* check the new value is there */
dpl_assert_str_eq(dpl_dict_get_value(dict, key0), val0_new);
/* check the other key is unaffected */
dpl_assert_str_eq(dpl_dict_get_value(dict, key1), val1);
dpl_dict_free(dict);
}
int
dfs_readlink(const char *path,
char *buf,
size_t bufsiz)
{
dpl_dict_t *dict = NULL;
dpl_status_t rc;
int ret;
char *dest = NULL;
size_t dest_size = 0;
rc = dfs_getattr_timeout(ctx, path, &dict);
if (DPL_SUCCESS != rc) {
LOG(LOG_ERR, "dfs_getattr_timeout: %s", dpl_status_str(rc));
ret = -1;
goto err;
}
if (! dict) {
LOG(LOG_ERR, "dpl_getattr: %s", dpl_status_str(rc));
ret = -1;
goto err;
}
dest = dpl_dict_get_value(dict, "symlink");
if (! dest) {
LOG(LOG_ERR, "empty link path");
ret = -1;
goto err;
}
dest_size = strlen(dest);
if (dest_size > bufsiz) {
LOG(LOG_NOTICE, "link length too big: '%s'", dest);
dest_size = bufsiz;
}
if (! strncpy(buf, dest, dest_size)) {
LOG(LOG_ERR, "path=%s: strcpy: %s", path, strerror(errno));
ret = -1;
goto err;
}
ret = 0;
err:
if (dict)
dpl_dict_free(dict);
LOG(LOG_DEBUG, "%s", path);
return 0;
}
static int
compare_digests(pentry_t *pe,
dpl_dict_t *dict)
{
char *remote = NULL;
int ret;
char *digest = NULL;
if (FILE_LOCAL != pentry_get_placeholder(pe)) {
LOG(LOG_DEBUG, "no local file");
ret = -1;
goto err;
}
digest = pentry_get_digest(pe);
if (! digest) {
LOG(LOG_NOTICE, "no digest");
ret = -1;
goto err;
}
print_metadata(dict);
ret = -1;
remote = dpl_dict_get_value(dict, "etag");
if (remote) {
LOG(LOG_DEBUG, "remote md5=%s", remote);
LOG(LOG_DEBUG, "local md5=\"%.*s\"", MD5_DIGEST_LENGTH, digest);
if (0 == memcmp(digest, remote, MD5_DIGEST_LENGTH)) {
ret = 0;
} else {
pentry_set_digest(pe, remote);
LOG(LOG_DEBUG, "updated local md5=\"%.*s\"",
MD5_DIGEST_LENGTH, pentry_get_digest(pe));
}
}
err:
return ret;
}
static long long
metadatatoll(dpl_dict_t *dict,
const char *const name)
{
char *value = NULL;
long long v = 0;
value = dpl_dict_get_value(dict, (char *)name);
if (! value) {
return -1;
}
v = strtoull(value, NULL, 10);
if (0 == strcmp(name, "scal_mode"))
LOG(LOG_DEBUG, "meta=%s, value=O%x", name, (unsigned)v);
else
LOG(LOG_DEBUG, "meta=%s, value=%s", name, value);
return v;
}
END_TEST
/*
* Use the dict with long key strings; tests
* some corner cases e.g. in the hash function.
*/
START_TEST(long_key_test)
{
#define N 512
dpl_dict_t *dict;
int i;
int j;
const char *act;
const char *exp;
char keybuf[1024];
char valbuf[1024];
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
for (i = 0 ; i < N ; i++)
{
dpl_dict_add(dict, make_key(i, keybuf, sizeof(keybuf)),
make_value(i, valbuf, sizeof(valbuf)),
/* lowered */0);
}
dpl_assert_int_eq(N, dpl_dict_count(dict));
for (i = 0 ; i < N ; i++)
{
act = dpl_dict_get_value(dict, make_key(i, keybuf, sizeof(keybuf)));
exp = make_value(i, valbuf, sizeof(valbuf));
dpl_assert_str_eq(act, exp);
}
dpl_dict_free(dict);
#undef N
}
static dpl_status_t
create_canonical_request(const dpl_req_t *req,
dpl_dict_t *headers,
dpl_vec_t *canonical_headers,
dpl_vec_t *canonical_params,
char *p, unsigned int len)
{
// Method
{
const char *method = dpl_method_str(req->method);
DPL_APPEND_STR(method);
DPL_APPEND_STR("\n");
}
// Resource
if (req->resource != NULL) {
char resource_ue[DPL_URL_LENGTH(strlen(req->resource)) + 1];
if (req->resource[0] != '/')
DPL_APPEND_STR("/");
dpl_url_encode_no_slashes(req->resource, resource_ue);
DPL_APPEND_STR(resource_ue);
}
DPL_APPEND_STR("\n");
// Query params
{
int item;
for (item = 0; item < canonical_params->n_items; item++) {
dpl_dict_var_t *param = (dpl_dict_var_t *) dpl_vec_get(canonical_params, item);
if (param == NULL)
continue;
if (item > 0)
DPL_APPEND_STR("&");
DPL_APPEND_STR(param->key);
DPL_APPEND_STR("=");
DPL_APPEND_STR(dpl_sbuf_get_str(param->val->string));
}
DPL_APPEND_STR("\n");
}
// Headers
if (canonical_headers != NULL) {
int item;
char *c;
dpl_dict_var_t *header;
for (item = 0; item < canonical_headers->n_items; item++) {
header = (dpl_dict_var_t *) dpl_vec_get(canonical_headers, item);
if (header == NULL)
continue;
for (c = header->key; *c != '\0'; c++)
DPL_APPEND_CHAR(tolower(*c));
DPL_APPEND_STR(":");
DPL_APPEND_STR(dpl_sbuf_get_str(header->val->string));
DPL_APPEND_STR("\n");
}
DPL_APPEND_STR("\n");
for (item = 0; item < canonical_headers->n_items; item++) {
header = (dpl_dict_var_t *) dpl_vec_get(canonical_headers, item);
if (header == NULL)
continue;
if (item > 0)
DPL_APPEND_STR(";");
for (c = header->key; *c != '\0'; c++)
DPL_APPEND_CHAR(tolower(*c));
}
DPL_APPEND_STR("\n");
} else {
DPL_APPEND_STR("host:");
DPL_APPEND_STR(req->host);
DPL_APPEND_STR("\n\n");
DPL_APPEND_STR("host\n");
}
// Hashed payload
if (headers != NULL) {
char *value = dpl_dict_get_value(headers, "x-amz-content-sha256");
if (value != NULL)
DPL_APPEND_STR(value);
} else
DPL_APPEND_STR("UNSIGNED-PAYLOAD");
return DPL_SUCCESS;
}
static int
handle_compression(const char *remote,
char *local,
struct get_data *get_data,
dpl_dict_t *metadata)
{
dpl_status_t rc;
char *uzlocal = NULL;
FILE *fpsrc = NULL;
FILE *fpdst = NULL;
char *compressed = NULL;
int zret;
int ret;
compressed = dpl_dict_get_value(metadata, "compression");
if (! compressed) {
LOG(LOG_INFO, "%s: uncompressed remote file", remote);
ret = 0;
goto end;
}
#define NONE "none"
#define ZLIB "zlib"
if (0 == strncmp(compressed, NONE, strlen(NONE))) {
LOG(LOG_INFO, "compression method: 'none'");
ret = 0;
goto end;
}
if (0 != strncmp(compressed, ZLIB, strlen(ZLIB))) {
LOG(LOG_ERR, "compression method not supported '%s'",
compressed);
ret = -1;
goto end;
}
uzlocal = tmpstr_printf("%s.tmp", local);
fpsrc = fopen(local, "r");
if (! fpsrc) {
LOG(LOG_ERR, "fopen: %s", strerror(errno));
ret = -1;
goto end;
}
fpdst = fopen(uzlocal, "w");
if (! fpdst) {
LOG(LOG_ERR, "fopen: %s", strerror(errno));
ret = -1;
goto end;
}
LOG(LOG_INFO, "uncompressing local file '%s'", local);
zret = unzip(fpsrc, fpdst);
if (Z_OK != zret) {
LOG(LOG_ERR, "unzip failed: %s", zerr_to_str(zret));
ret = -1;
goto end;
}
rc = dpl_dict_update_value(metadata, "compression", "none");
if (DPL_SUCCESS != rc) {
LOG(LOG_ERR, "unable to update 'compression' metadata");
ret = -1;
goto end;
}
if (-1 == rename(uzlocal, local)) {
LOG(LOG_ERR, "rename: %s", strerror(errno));
ret = 1;
goto end;
}
close(get_data->fd);
get_data->fd = open(local, O_RDONLY);
if (-1 == get_data->fd) {
LOG(LOG_ERR, "open: %s", strerror(errno));
ret = -1;
goto end;
}
#undef ZLIB
#undef NONE
ret = 0;
end:
if (fpsrc)
fclose(fpsrc);
if (fpdst)
fclose(fpdst);
return ret;
}
dpl_status_t
dpl_cdmi_list_bucket(dpl_ctx_t *ctx,
const char *bucket,
const char *prefix,
const char *delimiter,
dpl_vec_t **objectsp,
dpl_vec_t **common_prefixesp)
{
char *host;
int ret, ret2;
dpl_conn_t *conn = NULL;
char header[1024];
u_int header_len;
struct iovec iov[10];
int n_iov = 0;
int connection_close = 0;
char *data_buf = NULL;
u_int data_len;
dpl_dict_t *headers_request = NULL;
dpl_dict_t *headers_reply = NULL;
dpl_req_t *req = NULL;
dpl_vec_t *common_prefixes = NULL;
dpl_vec_t *objects = NULL;
req = dpl_req_new(ctx);
if (NULL == req)
{
ret = DPL_ENOMEM;
goto end;
}
dpl_req_set_method(req, DPL_METHOD_GET);
ret2 = dpl_req_set_resource(req, NULL != prefix ? prefix : "/");
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
//contact default host
dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);
dpl_req_set_object_type(req, DPL_FTYPE_DIR);
//build request
ret2 = dpl_cdmi_req_build(req, &headers_request, NULL, NULL);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
host = dpl_dict_get_value(headers_request, "Host");
if (NULL == host)
{
ret = DPL_FAILURE;
goto end;
}
conn = dpl_conn_open_host(ctx, host, ctx->port);
if (NULL == conn)
{
ret = DPL_FAILURE;
goto end;
}
//bucket emulation
ret2 = dpl_dict_add(headers_request, "X-Scality-Bucket", bucket, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
iov[n_iov].iov_base = header;
iov[n_iov].iov_len = header_len;
n_iov++;
//final crlf
iov[n_iov].iov_base = "\r\n";
iov[n_iov].iov_len = 2;
n_iov++;
ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
if (DPL_SUCCESS != ret2)
{
DPLERR(1, "writev failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
ret2 = dpl_read_http_reply(conn, 1, &data_buf, &data_len, &headers_reply);
if (DPL_SUCCESS != ret2)
{
if (DPL_ENOENT == ret2)
{
ret = DPL_ENOENT;
goto end;
}
else
{
DPLERR(0, "read http answer failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
}
else
{
connection_close = dpl_connection_close(ctx, headers_reply);
}
(void) dpl_log_event(ctx, "DATA", "OUT", data_len);
objects = dpl_vec_new(2, 2);
if (NULL == objects)
{
ret = DPL_FAILURE;
goto end;
}
common_prefixes = dpl_vec_new(2, 2);
if (NULL == common_prefixes)
{
ret = DPL_FAILURE;
goto end;
}
ret = dpl_cdmi_parse_list_bucket(ctx, data_buf, data_len, prefix, objects, common_prefixes);
if (DPL_SUCCESS != ret)
{
ret = DPL_FAILURE;
goto end;
}
if (NULL != objectsp)
{
*objectsp = objects;
objects = NULL; //consume it
}
if (NULL != common_prefixesp)
{
*common_prefixesp = common_prefixes;
common_prefixes = NULL; //consume it
}
ret = DPL_SUCCESS;
end:
if (NULL != objects)
dpl_vec_objects_free(objects);
if (NULL != common_prefixes)
dpl_vec_common_prefixes_free(common_prefixes);
if (NULL != data_buf)
free(data_buf);
if (NULL != conn)
{
if (1 == connection_close)
dpl_conn_terminate(conn);
else
dpl_conn_release(conn);
}
if (NULL != headers_reply)
dpl_dict_free(headers_reply);
if (NULL != headers_request)
dpl_dict_free(headers_request);
if (NULL != req)
dpl_req_free(req);
DPRINTF("ret=%d\n", ret);
return ret;
}
dpl_status_t
dpl_cdmi_delete(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
dpl_ftype_t object_type)
{
char *host;
int ret, ret2;
dpl_conn_t *conn = NULL;
char header[1024];
u_int header_len;
struct iovec iov[10];
int n_iov = 0;
int connection_close = 0;
dpl_dict_t *headers_request = NULL;
dpl_dict_t *headers_reply = NULL;
dpl_req_t *req = NULL;
req = dpl_req_new(ctx);
if (NULL == req)
{
ret = DPL_ENOMEM;
goto end;
}
dpl_req_set_method(req, DPL_METHOD_DELETE);
ret2 = dpl_req_set_resource(req, resource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != subresource)
{
ret2 = dpl_req_set_subresource(req, subresource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
//contact default host
dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);
dpl_req_add_behavior(req, DPL_BEHAVIOR_HTTP_COMPAT);
//build request
ret2 = dpl_cdmi_req_build(req, &headers_request, NULL, NULL);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
host = dpl_dict_get_value(headers_request, "Host");
if (NULL == host)
{
ret = DPL_FAILURE;
goto end;
}
conn = dpl_conn_open_host(ctx, host, ctx->port);
if (NULL == conn)
{
ret = DPL_FAILURE;
goto end;
}
//bucket emulation
ret2 = dpl_dict_add(headers_request, "X-Scality-Bucket", bucket, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
iov[n_iov].iov_base = header;
iov[n_iov].iov_len = header_len;
n_iov++;
//final crlf
iov[n_iov].iov_base = "\r\n";
iov[n_iov].iov_len = 2;
n_iov++;
ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
if (DPL_SUCCESS != ret2)
{
DPLERR(1, "writev failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply);
if (DPL_SUCCESS != ret2)
{
if (DPL_ENOENT == ret2)
{
ret = DPL_ENOENT;
goto end;
}
else
{
DPLERR(0, "read http answer failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
}
else
{
connection_close = dpl_connection_close(ctx, headers_reply);
}
(void) dpl_log_event(ctx, "REQUEST", "DELETE", 0);
ret = DPL_SUCCESS;
end:
if (NULL != conn)
{
if (1 == connection_close)
dpl_conn_terminate(conn);
else
dpl_conn_release(conn);
}
if (NULL != headers_reply)
dpl_dict_free(headers_reply);
if (NULL != headers_request)
dpl_dict_free(headers_request);
if (NULL != req)
dpl_req_free(req);
DPRINTF("ret=%d\n", ret);
return ret;
}
dpl_status_t
dpl_cdmi_get_buffered(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
dpl_header_func_t header_func,
dpl_buffer_func_t buffer_func,
void *cb_arg)
{
char *host;
int ret, ret2;
dpl_conn_t *conn = NULL;
char header[1024];
u_int header_len;
struct iovec iov[10];
int n_iov = 0;
dpl_dict_t *headers_request = NULL;
dpl_req_t *req = NULL;
struct get_conven gc;
memset(&gc, 0, sizeof (gc));
gc.header_func = header_func;
gc.buffer_func = buffer_func;
gc.cb_arg = cb_arg;
req = dpl_req_new(ctx);
if (NULL == req)
{
ret = DPL_ENOMEM;
goto end;
}
dpl_req_set_method(req, DPL_METHOD_GET);
if (NULL != bucket)
{
ret2 = dpl_req_set_bucket(req, bucket);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
ret2 = dpl_req_set_resource(req, resource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != subresource)
{
ret2 = dpl_req_set_subresource(req, subresource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (NULL != condition)
{
dpl_req_set_condition(req, condition);
}
//contact default host
dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);
dpl_req_add_behavior(req, DPL_BEHAVIOR_HTTP_COMPAT);
//build request
ret2 = dpl_cdmi_req_build(req, &headers_request, NULL, NULL);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
host = dpl_dict_get_value(headers_request, "Host");
if (NULL == host)
{
ret = DPL_FAILURE;
goto end;
}
conn = dpl_conn_open_host(ctx, host, ctx->port);
if (NULL == conn)
{
ret = DPL_FAILURE;
goto end;
}
//bucket emulation
ret2 = dpl_dict_add(headers_request, "X-Scality-Bucket", bucket, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
iov[n_iov].iov_base = header;
iov[n_iov].iov_len = header_len;
n_iov++;
//final crlf
iov[n_iov].iov_base = "\r\n";
iov[n_iov].iov_len = 2;
n_iov++;
ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
if (DPL_SUCCESS != ret2)
{
DPLERR(1, "writev failed");
gc.connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
ret2 = dpl_read_http_reply_buffered(conn, 1, cb_get_header, cb_get_buffer, &gc);
if (DPL_SUCCESS != ret2)
{
if (DPL_ENOENT == ret2)
{
ret = DPL_ENOENT;
goto end;
}
else
{
DPLERR(0, "read http answer failed");
gc.connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
}
//caller is responsible for logging the event
ret = DPL_SUCCESS;
end:
if (NULL != conn)
{
if (1 == gc.connection_close)
dpl_conn_terminate(conn);
else
dpl_conn_release(conn);
}
if (NULL != headers_request)
dpl_dict_free(headers_request);
if (NULL != req)
dpl_req_free(req);
DPRINTF("ret=%d\n", ret);
return ret;
}
dpl_status_t
dpl_cdmi_get(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
dpl_ftype_t object_type,
const dpl_condition_t *condition,
char **data_bufp,
unsigned int *data_lenp,
dpl_dict_t **metadatap)
{
char *host;
int ret, ret2;
dpl_conn_t *conn = NULL;
char header[1024];
u_int header_len;
struct iovec iov[10];
int n_iov = 0;
int connection_close = 0;
char *data_buf = NULL;
u_int data_len;
dpl_dict_t *headers_request = NULL;
dpl_dict_t *headers_reply = NULL;
dpl_dict_t *metadata = NULL;
dpl_req_t *req = NULL;
req = dpl_req_new(ctx);
if (NULL == req)
{
ret = DPL_ENOMEM;
goto end;
}
dpl_req_set_method(req, DPL_METHOD_GET);
if (NULL != bucket)
{
ret2 = dpl_req_set_bucket(req, bucket);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
ret2 = dpl_req_set_resource(req, resource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != subresource)
{
ret2 = dpl_req_set_subresource(req, subresource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
if (NULL != condition)
{
dpl_req_set_condition(req, condition);
}
//contact default host
dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);
dpl_req_set_object_type(req, object_type);
//build request
ret2 = dpl_cdmi_req_build(req, &headers_request, NULL, NULL);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
host = dpl_dict_get_value(headers_request, "Host");
if (NULL == host)
{
ret = DPL_FAILURE;
goto end;
}
conn = dpl_conn_open_host(ctx, host, ctx->port);
if (NULL == conn)
{
ret = DPL_FAILURE;
goto end;
}
ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
iov[n_iov].iov_base = header;
iov[n_iov].iov_len = header_len;
n_iov++;
//final crlf
iov[n_iov].iov_base = "\r\n";
iov[n_iov].iov_len = 2;
n_iov++;
ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
if (DPL_SUCCESS != ret2)
{
DPLERR(1, "writev failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
ret2 = dpl_read_http_reply(conn, 1, &data_buf, &data_len, &headers_reply);
if (DPL_SUCCESS != ret2)
{
if (DPL_ENOENT == ret2)
{
ret = DPL_ENOENT;
goto end;
}
else
{
DPLERR(0, "read http answer failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
}
else
{
connection_close = dpl_connection_close(ctx, headers_reply);
}
(void) dpl_log_event(ctx, "DATA", "OUT", data_len);
if (NULL != data_bufp)
{
*data_bufp = data_buf;
data_buf = NULL; //consume it
}
if (NULL != data_lenp)
*data_lenp = data_len;
if (NULL != metadatap)
{
*metadatap = metadata;
metadata = NULL; //consume it
}
ret = DPL_SUCCESS;
end:
if (NULL != data_buf)
free(data_buf);
if (NULL != conn)
{
if (1 == connection_close)
dpl_conn_terminate(conn);
else
dpl_conn_release(conn);
}
if (NULL != metadata)
dpl_dict_free(metadata);
if (NULL != headers_reply)
dpl_dict_free(headers_reply);
if (NULL != headers_request)
dpl_dict_free(headers_request);
if (NULL != req)
dpl_req_free(req);
DPRINTF("ret=%d\n", ret);
return ret;
}
END_TEST
/*
* Test the "lowered" feature, which is almost but not quite
* the same as case insensitivity.
*/
START_TEST(lowered_test)
{
dpl_dict_t *dict;
dpl_status_t r;
const char *s;
dpl_dict_var_t *var;
const char value[] = "World";
const char value2[] = "Mondo";
const char value3[] = "Monde";
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
/* Add a value, lowered */
r = dpl_dict_add(dict, "Hello", value, /*lowered*/1);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(1, dpl_dict_count(dict));
/* The actual key used is internally lowercased, so
* doing a normal get on the key originally given
* should fail */
dpl_assert_str_eq(NULL, dpl_dict_get_value(dict, "Hello"));
/* Likewise, doing a normal get on a lowercase version
* of the original key should succeed */
dpl_assert_str_eq(value, dpl_dict_get_value(dict, "hello"));
/* dpl_dict_get_lowered internally lowercases the key it's
* given, so it can be used with keys of any casing */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "hello", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "Hello", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "HELLO", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "hElLo", &var);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_ptr_not_null(var);
dpl_assert_ptr_ne(var, BADPOINTER);
dpl_assert_str_eq(value, dpl_sbuf_get_str(var->val->string));
/* check that dpl_dict_get_lowered() will report as missing
* some keys that we know not to be present */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "HellonEarth", &var);
dpl_assert_int_eq(DPL_ENOENT, r);
/* the value of `var' on failure is not documented */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "Hell", &var);
dpl_assert_int_eq(DPL_ENOENT, r);
/* the value of `var' on failure is not documented */
var = BADPOINTER;
r = dpl_dict_get_lowered(dict, "daffyduck", &var);
dpl_assert_int_eq(DPL_ENOENT, r);
/* the value of `var' on failure is not documented */
/* Verify that inserting another key which maps to the
* same lowercased string, replaces the first key */
r = dpl_dict_add(dict, "hello", value2, /*lowered*/1);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(1, dpl_dict_count(dict));
dpl_assert_str_eq(value2, dpl_dict_get_value(dict, "hello"));
r = dpl_dict_add(dict, "hELLo", value3, /*lowered*/1);
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(1, dpl_dict_count(dict));
dpl_assert_str_eq(value3, dpl_dict_get_value(dict, "hello"));
dpl_dict_free(dict);
}
END_TEST
/*
* Test the dpl_dict_filter_prefix() and
* dpl_dict_filter_no_prefix() functions.
*/
START_TEST(filter_test)
{
dpl_dict_t *dict;
dpl_dict_t *d2;
int i;
dpl_status_t r;
const char *exp;
const char *act;
char valbuf[128];
/* strings courtesy http://hipsteripsum.me/ */
static const char * const keys[] = {
"Sriracha", "mehBanksy", "trust", "fund", "Brooklyn",
"mehpolaroid", "Viral", "mehselfies", "mehkogi", "Austin" };
static const int nkeys = sizeof(keys)/sizeof(keys[0]);
static const int is_meh[] = { 0, 1, 0, 0, 0,
1, 0, 1, 1, 0 };
static const int nmehs = 4;
dict = dpl_dict_new(13);
dpl_assert_ptr_not_null(dict);
/* add all the keys */
for (i = 0 ; i < nkeys ; i++)
{
dpl_dict_add(dict, keys[i],
make_value(i, valbuf, sizeof(valbuf)),
/* lowered */0);
}
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
/* create a new dict and copy everything across using an empty prefix */
d2 = dpl_dict_new(13);
r = dpl_dict_filter_prefix(d2, dict, "");
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
dpl_assert_int_eq(nkeys, dpl_dict_count(d2));
for (i = 0 ; i < nkeys ; i++)
{
exp = make_value(i, valbuf, sizeof(valbuf));
act = dpl_dict_get_value(d2, keys[i]);
dpl_assert_str_eq(exp, act);
}
dpl_dict_free(d2);
/* create a new dict and copy just the matching ones */
d2 = dpl_dict_new(13);
r = dpl_dict_filter_prefix(d2, dict, "meh");
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
dpl_assert_int_eq(nmehs, dpl_dict_count(d2));
for (i = 0 ; i < nkeys ; i++)
{
if (is_meh[i])
{
exp = make_value(i, valbuf, sizeof(valbuf));
/*
* dpl_dict_filter_prefix() inexplicably removes the prefix
* from the keys it copies across. This was probably very
* convenient for the person who wrote the code, but isn't
* all that easy to explain. But this is why we add a
* strlen() here.
*/
act = dpl_dict_get_value(d2, keys[i]+strlen("meh"));
dpl_assert_str_eq(exp, act);
}
}
dpl_dict_free(d2);
/* create a new dict and copy just the non-matching ones */
d2 = dpl_dict_new(13);
r = dpl_dict_filter_no_prefix(d2, dict, "meh");
dpl_assert_int_eq(DPL_SUCCESS, r);
dpl_assert_int_eq(nkeys, dpl_dict_count(dict));
dpl_assert_int_eq(nkeys-nmehs, dpl_dict_count(d2));
for (i = 0 ; i < nkeys ; i++)
{
if (!is_meh[i])
{
exp = make_value(i, valbuf, sizeof(valbuf));
act = dpl_dict_get_value(d2, keys[i]);
dpl_assert_str_eq(exp, act);
}
}
dpl_dict_free(d2);
dpl_dict_free(dict);
}
static dpl_status_t
dpl_s3_make_signature_v2(dpl_ctx_t *ctx,
const char *method,
const char *bucket,
const char *resource,
const char *subresource,
char *date,
dpl_dict_t *headers,
char *buf,
unsigned int len,
unsigned int *lenp)
{
char *p;
char *value;
int ret;
p = buf;
//method
DPL_APPEND_STR(method);
DPL_APPEND_STR("\n");
//md5
if (headers != NULL) {
value = dpl_dict_get_value(headers, "Content-MD5");
if (NULL != value)
DPL_APPEND_STR(value);
}
DPL_APPEND_STR("\n");
//content type
if (headers != NULL) {
value = dpl_dict_get_value(headers, "Content-Type");
if (NULL != value)
DPL_APPEND_STR(value);
}
DPL_APPEND_STR("\n");
//expires or date
if (date != NULL)
DPL_APPEND_STR(date);
DPL_APPEND_STR("\n");
//x-amz headers
if (headers != NULL) {
int bucket;
dpl_dict_var_t *var;
dpl_vec_t *vec;
int i;
vec = dpl_vec_new(2, 2);
if (NULL == vec)
return DPL_ENOMEM;
for (bucket = 0; bucket < headers->n_buckets; bucket++)
{
for (var = headers->buckets[bucket];var;var = var->prev)
{
if (!strncmp(var->key, "x-amz-", 6) && strcmp(var->key, "x-amz-date"))
{
assert(DPL_VALUE_STRING == var->val->type);
ret = dpl_vec_add(vec, var);
if (DPL_SUCCESS != ret)
{
dpl_vec_free(vec);
return DPL_FAILURE;
}
}
}
}
dpl_vec_sort(vec, var_cmp);
for (i = 0;i < vec->n_items;i++) {
var = (dpl_dict_var_t *) dpl_vec_get(vec, i);
if (var == NULL)
continue;
assert(DPL_VALUE_STRING == var->val->type);
DPL_APPEND_STR(var->key);
DPL_APPEND_STR(":");
DPL_APPEND_STR(dpl_sbuf_get_str(var->val->string));
DPL_APPEND_STR("\n");
}
dpl_vec_free(vec);
}
//resource
if (NULL != bucket) {
DPL_APPEND_STR("/");
DPL_APPEND_STR(bucket);
}
if (NULL != resource)
DPL_APPEND_STR(resource);
if (NULL != subresource) {
DPL_APPEND_STR("?");
DPL_APPEND_STR(subresource);
}
if (NULL != lenp)
*lenp = p - buf;
return DPL_SUCCESS;
}
dpl_status_t
dpl_cdmi_put_buffered(dpl_ctx_t *ctx,
const char *bucket,
const char *resource,
const char *subresource,
dpl_ftype_t object_type,
const dpl_dict_t *metadata,
const dpl_sysmd_t *sysmd,
unsigned int data_len,
dpl_conn_t **connp)
{
char *host;
int ret, ret2;
dpl_conn_t *conn = NULL;
char header[1024];
u_int header_len;
struct iovec iov[10];
int n_iov = 0;
int connection_close = 0;
dpl_dict_t *headers_request = NULL;
dpl_dict_t *headers_reply = NULL;
dpl_req_t *req = NULL;
dpl_chunk_t chunk;
req = dpl_req_new(ctx);
if (NULL == req)
{
ret = DPL_ENOMEM;
goto end;
}
dpl_req_set_method(req, DPL_METHOD_PUT);
if (NULL != bucket)
{
ret2 = dpl_req_set_bucket(req, bucket);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
ret2 = dpl_req_set_resource(req, resource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
if (NULL != subresource)
{
ret2 = dpl_req_set_subresource(req, subresource);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
chunk.buf = NULL;
chunk.len = data_len;
dpl_req_set_chunk(req, &chunk);
//contact default host
dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);
dpl_req_add_behavior(req, DPL_BEHAVIOR_HTTP_COMPAT);
dpl_req_set_object_type(req, object_type);
dpl_req_add_behavior(req, DPL_BEHAVIOR_EXPECT);
if (NULL != metadata)
{
ret2 = dpl_req_add_metadata(req, metadata);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
}
ret2 = dpl_cdmi_req_build(req, &headers_request, NULL, NULL);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
host = dpl_dict_get_value(headers_request, "Host");
if (NULL == host)
{
ret = DPL_FAILURE;
goto end;
}
conn = dpl_conn_open_host(ctx, host, ctx->port);
if (NULL == conn)
{
ret = DPL_FAILURE;
goto end;
}
//bucket emulation
ret2 = dpl_dict_add(headers_request, "X-Scality-Bucket", bucket, 0);
if (DPL_SUCCESS != ret2)
{
ret = ret2;
goto end;
}
ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
if (DPL_SUCCESS != ret2)
{
ret = DPL_FAILURE;
goto end;
}
iov[n_iov].iov_base = header;
iov[n_iov].iov_len = header_len;
n_iov++;
//final crlf
iov[n_iov].iov_base = "\r\n";
iov[n_iov].iov_len = 2;
n_iov++;
ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
if (DPL_SUCCESS != ret2)
{
DPLERR(1, "writev failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply);
if (DPL_SUCCESS != ret2)
{
if (DPL_ENOENT == ret2)
{
ret = DPL_ENOENT;
goto end;
}
else
{
DPLERR(0, "read http answer failed");
connection_close = 1;
ret = DPL_ENOENT; //mapped to 404
goto end;
}
}
else
{
if (NULL != headers_reply) //possible if continue succeeded
connection_close = dpl_connection_close(ctx, headers_reply);
}
(void) dpl_log_event(ctx, "DATA", "IN", data_len);
if (NULL != connp)
{
*connp = conn;
conn = NULL; //consume it
}
ret = DPL_SUCCESS;
end:
if (NULL != conn)
{
if (1 == connection_close)
dpl_conn_terminate(conn);
else
dpl_conn_release(conn);
}
if (NULL != headers_reply)
dpl_dict_free(headers_reply);
if (NULL != headers_request)
dpl_dict_free(headers_request);
if (NULL != req)
dpl_req_free(req);
DPRINTF("ret=%d\n", ret);
return ret;
}
