Beispiel #1
0
dpl_status_t
dpl_cdmi_copy(dpl_ctx_t *ctx,
              const char *src_bucket,
              const char *src_resource,
              const char *src_subresource,
              const char *dst_bucket,
              const char *dst_resource,
              const char *dst_subresource,
              const dpl_option_t *option,
              dpl_ftype_t object_type,
              dpl_copy_directive_t copy_directive,
              const dpl_dict_t *metadata,
              const dpl_sysmd_t *sysmd,
              const dpl_condition_t *condition,
              char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  char          *body_str = NULL;
  int           body_len = 0;
  int           add_base_path;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  if (DPL_COPY_DIRECTIVE_METADATA_REPLACE == copy_directive)
    {
      ret = dpl_cdmi_put(ctx, dst_bucket, dst_resource, NULL, NULL,
                         object_type, condition, NULL, metadata, DPL_CANNED_ACL_UNDEF, NULL, 0, 
                         NULL, NULL, locationp);
      goto end;
    }
      
  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_PUT);

  ret2 = dpl_cdmi_req_set_resource(req, dst_resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != dst_subresource)
    {
      ret2 = dpl_req_set_subresource(req, dst_subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (copy_directive == DPL_COPY_DIRECTIVE_MKDENT ||
      copy_directive == DPL_COPY_DIRECTIVE_RMDENT)
    add_base_path = 0;
  else
    add_base_path = 1;

  ret2 = dpl_req_set_src_resource_ext(req, src_resource, add_base_path);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != src_subresource)
    {
      ret2 = dpl_req_set_src_subresource(req, src_subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  dpl_req_set_copy_directive(req, copy_directive);

  dpl_req_set_object_type(req, object_type);

  dpl_req_add_behavior(req, DPL_BEHAVIOR_MD5);

  if (NULL != sysmd)
    {
      ret2 = dpl_cdmi_add_sysmd_to_req(sysmd, req);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (NULL != metadata)
    {
      ret2 = dpl_cdmi_req_add_metadata(req, metadata, option ? option->mask & DPL_OPTION_APPEND_METADATA : 0);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  //build request
  ret2 = dpl_cdmi_req_build(req, 0, &headers_request, &body_str, &body_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  //buffer
  iov[n_iov].iov_base = body_str;
  iov[n_iov].iov_len = body_len;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != body_str)
    free(body_str);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #2
0
dpl_status_t
dpl_cdmi_put_internal(dpl_ctx_t *ctx,
                      int post,
                      const char *bucket,
                      const char *resource,
                      const char *subresource,
                      const dpl_option_t *option,
                      dpl_ftype_t object_type,
                      const dpl_condition_t *condition,
                      const dpl_range_t *range,
                      const dpl_dict_t *metadata,
                      const dpl_sysmd_t *sysmd,
                      const char *data_buf,
                      unsigned int data_len,
                      const dpl_dict_t *query_params,
                      dpl_sysmd_t *returned_sysmdp,
                      char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  char          *body_str = NULL;
  int           body_len = 0;
  char *data_buf_returned = NULL;
  u_int data_len_returned;
  dpl_value_t *val = NULL;
  dpl_cdmi_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  if (option)
    {
      if (option->mask & DPL_OPTION_HTTP_COMPAT)
        req_mask |= DPL_CDMI_REQ_HTTP_COMPAT;
    }

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  if (post)
    dpl_req_set_method(req, DPL_METHOD_POST);
  else
    dpl_req_set_method(req, DPL_METHOD_PUT);

  ret2 = dpl_cdmi_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (NULL != condition)
    {
      dpl_req_set_condition(req, condition);
    }

  if (range)
    {
      ret2 = dpl_cdmi_req_add_range(req, req_mask, range);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  dpl_req_set_object_type(req, object_type);

  dpl_req_set_data(req, data_buf, data_len);

  dpl_req_add_behavior(req, DPL_BEHAVIOR_MD5);

  if (NULL != sysmd)
    {
      ret2 = dpl_cdmi_add_sysmd_to_req(sysmd, req);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (NULL != metadata)
    {
      ret2 = dpl_cdmi_req_add_metadata(req, metadata, option ? option->mask & DPL_OPTION_APPEND_METADATA : 0);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  //build request
  ret2 = dpl_cdmi_req_build(req, 0, &headers_request, &body_str, &body_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  //buffer
  iov[n_iov].iov_base = body_str;
  iov[n_iov].iov_len = body_len;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, &data_buf_returned, &data_len_returned, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (post)
    {
      if (req_mask & DPL_CDMI_REQ_HTTP_COMPAT)
        {
          ret2 = dpl_cdmi_get_metadata_from_headers(headers_reply, NULL, returned_sysmdp);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }
        }
      else
        {
          ret2 = dpl_cdmi_parse_json_buffer(ctx, data_buf_returned, data_len_returned, &val);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }
          
          if (DPL_VALUE_SUBDICT != val->type)
            {
              ret = DPL_EINVAL;
              goto end;
            }
          
          ret2 = dpl_cdmi_get_metadata_from_values(val->subdict, NULL, returned_sysmdp);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }
        }
      
      if (NULL != returned_sysmdp)
        {
          dpl_dict_var_t *var;
          int base_path_len = strlen(ctx->base_path);
          
          //location contains the path to new location
          ret2 = dpl_dict_get_lowered(headers_reply, "location", &var);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }
          
          if (DPL_VALUE_STRING != var->val->type)
            {
              ret = DPL_EINVAL;
              goto end;
            }
          
          //remove the base_path from the answer
          if (strncmp(dpl_sbuf_get_str(var->val->string),
                      ctx->base_path, base_path_len))
            {
              ret = DPL_EINVAL;
              goto end;
            }
          
          returned_sysmdp->mask |= DPL_SYSMD_MASK_PATH;
          strncpy(returned_sysmdp->path,
                  dpl_sbuf_get_str(var->val->string) + base_path_len,
                  DPL_MAXPATHLEN);
        }
    }
  
  ret = DPL_SUCCESS;

 end:

  if (NULL != val)
    dpl_value_free(val);

  if (NULL != data_buf_returned)
    free(data_buf_returned);

  if (NULL != body_str)
    free(body_str);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #3
0
dpl_status_t
dpl_s3_stream_multipart_put(dpl_ctx_t *ctx,
                            const char *bucket,
                            const char *resource,
                            const char *uploadid,
                            unsigned int partnb,
                            char *buf, unsigned int len,
                            const char **etagp)
{
    dpl_status_t  ret;
    dpl_conn_t    *conn = NULL;
    char          header[dpl_header_size];
    u_int         header_len;
    struct iovec  iov[10];
    int           n_iov = 0;
    int           connection_close = 0;
    dpl_dict_t    *headers_request = NULL;
    dpl_dict_t    *headers_reply = NULL;
    dpl_s3_req_mask_t req_mask = 0u;
    dpl_req_t           *req = NULL;
    char          subresource[  11 /* for 'partNumber=' */ + 18 /* max len for %lu */
                                + 10 /* for 'uploadId=' */   + strlen(uploadid)];


    snprintf(subresource, sizeof(subresource), "partNumber=%u&uploadId=%s", partnb, uploadid);

    req = dpl_req_new(ctx);
    if (NULL == req)
    {
        ret = DPL_ENOMEM;
        goto end;
    }

    dpl_req_set_method(req, DPL_METHOD_PUT);

    if (NULL == bucket)
    {
        ret = DPL_EINVAL;
        goto end;
    }

    ret = dpl_req_set_bucket(req, bucket);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_set_resource(req, resource);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_set_subresource(req, subresource);
    if (DPL_SUCCESS != ret)
        goto end;

    dpl_req_set_data(req, buf, len);
    dpl_req_add_behavior(req, DPL_BEHAVIOR_MD5);

    ret = dpl_s3_req_build(req, req_mask, &headers_request);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_try_connect(ctx, req, &conn);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_add_host_to_headers(req, headers_request);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_s3_add_authorization_to_headers(req, headers_request, NULL, NULL);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
    if (DPL_SUCCESS != ret)
        goto end;

    iov[n_iov].iov_base = header;
    iov[n_iov].iov_len = header_len;
    n_iov++;

    //final crlf
    iov[n_iov].iov_base = "\r\n";
    iov[n_iov].iov_len = 2;
    n_iov++;

    //buffer
    iov[n_iov].iov_base = (void *)buf;
    iov[n_iov].iov_len = len;
    n_iov++;

    ret = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
    if (DPL_SUCCESS != ret)
    {
        DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
        connection_close = 1;
        goto end;
    }

    ret = dpl_read_http_reply(conn, 1, NULL, NULL,
                              &headers_reply, &connection_close);
    if (DPL_SUCCESS != ret)
        goto end;

    if (etagp)
    {
        /*
         * Etag of the part is returned as a header, as a string value.
         * Thus, it should be surrounded by double-quotes that we need
         * to remove
         */
        dpl_dict_var_t *var = NULL;
        const char *ro_etag = NULL;
        const char *start_etag = NULL;
        const char *end_etag = NULL;

        ret = dpl_dict_get_lowered(headers_reply, "Etag", &var);
        if (ret != DPL_SUCCESS || var == NULL)
        {
            ret = DPL_FAILURE;
            goto end;
        }

        assert(var->val->type == DPL_VALUE_STRING);
        ro_etag = dpl_sbuf_get_str(var->val->string);
        start_etag = strchr(ro_etag, '"');
        if (start_etag == NULL)
        {
            start_etag = ro_etag;
            end_etag = start_etag + strlen(start_etag);
        }
        else
        {
            start_etag += 1; // go after the " char
            end_etag = strchr(start_etag, '"');
        }

        *etagp = strndup(start_etag, end_etag - start_etag);
        if (*etagp == NULL)
        {
            ret = DPL_ENOMEM;
            goto end;
        }
    }

    ret = DPL_SUCCESS;

end:
    if (NULL != conn)
    {
        if (1 == connection_close)
            dpl_conn_terminate(conn);
        else
            dpl_conn_release(conn);
    }

    if (NULL != headers_reply)
        dpl_dict_free(headers_reply);

    if (NULL != headers_request)
        dpl_dict_free(headers_request);

    if (NULL != req)
        dpl_req_free(req);

    return ret;
}
Beispiel #4
0
dpl_status_t
dpl_cdmi_delete(dpl_ctx_t *ctx,
                const char *bucket,
                const char *resource,
                const char *subresource,
                const dpl_option_t *option,
                dpl_ftype_t object_type,
                const dpl_condition_t *condition,
                char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_cdmi_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  if (option)
    {
      if (option->mask & DPL_OPTION_HTTP_COMPAT)
        req_mask |= DPL_CDMI_REQ_HTTP_COMPAT;
    }

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_DELETE);

  ret2 = dpl_cdmi_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

#if 0
  if (NULL != condition)
    {
      dpl_req_set_condition(req, condition);
    }
#endif

  //build request
  ret2 = dpl_cdmi_req_build(req, req_mask, &headers_request, NULL, NULL);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #5
0
dpl_status_t
dpl_s3_get(dpl_ctx_t *ctx,
           const char *bucket,
           const char *resource,
           const char *subresource,
           const dpl_option_t *option,
           dpl_ftype_t object_type,
           const dpl_condition_t *condition,
           const dpl_range_t *range,
           char **data_bufp,
           unsigned int *data_lenp,
           dpl_dict_t **metadatap,
           dpl_sysmd_t *sysmdp,
           char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  char          *data_buf = NULL;
  u_int         data_len;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_GET);

  if (NULL == bucket)
    {
      ret = DPL_EINVAL;
      goto end;
    }

  ret2 = dpl_req_set_bucket(req, bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (NULL != condition)
    {
      dpl_req_set_condition(req, condition);
    }

  if (range)
    {
      ret2 = dpl_req_add_range(req, range->start, range->end);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  //build request
  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, &data_buf, &data_len, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_s3_get_metadata_from_headers(headers_reply, metadatap, sysmdp);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != data_bufp)
    {
      *data_bufp = data_buf;
      data_buf = NULL; //consume it
    }

  if (NULL != data_lenp)
    *data_lenp = data_len;

  //dpl_dict_print(headers_reply);

  ret = DPL_SUCCESS;

 end:

  if (NULL != data_buf)
    free(data_buf);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #6
0
dpl_status_t
dpl_s3_stream_multipart_complete(dpl_ctx_t *ctx,
                                 const char *bucket,
                                 const char *resource,
                                 const char *uploadid,
                                 struct json_object *parts,
                                 unsigned int n_parts,
                                 const dpl_dict_t *metadata,
                                 const dpl_sysmd_t *sysmd)
{
    dpl_status_t  ret;
    dpl_conn_t    *conn = NULL;
    char          header[dpl_header_size];
    u_int         header_len;
    struct iovec  iov[10];
    int           n_iov = 0;
    int           connection_close = 0;
    dpl_dict_t    *headers_request = NULL;
    dpl_dict_t    *headers_reply = NULL;
    dpl_req_t     *req = NULL;
    char          subresource[strlen(uploadid) + 10 /* for 'uploadId=' */];
    char          *databuf = NULL;
    unsigned int  databuflen = 0;
    char          *replybuf = NULL;
    unsigned int  replybuflen = 0;

    snprintf(subresource, sizeof(subresource), "uploadId=%s", uploadid);

    req = dpl_req_new(ctx);
    if (NULL == req)
    {
        ret = DPL_ENOMEM;
        goto end;
    }

    dpl_req_set_method(req, DPL_METHOD_POST);

    if (NULL == bucket)
    {
        ret = DPL_EINVAL;
        goto end;
    }

    ret = dpl_req_set_bucket(req, bucket);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_set_resource(req, resource);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_set_subresource(req, subresource);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = multipart_complete_gen_body(ctx, parts, n_parts,
                                      &databuf, &databuflen);
    if (DPL_SUCCESS != ret)
        goto end;

    dpl_req_set_data(req, databuf, databuflen);
    dpl_req_add_behavior(req, DPL_BEHAVIOR_MD5);

    if (sysmd)
    {
        if (sysmd->mask & DPL_SYSMD_MASK_CANNED_ACL)
            dpl_req_set_canned_acl(req, sysmd->canned_acl);

        if (sysmd->mask & DPL_SYSMD_MASK_STORAGE_CLASS)
            dpl_req_set_storage_class(req, sysmd->storage_class);
    }

    if (NULL != metadata)
    {
        ret = dpl_req_add_metadata(req, metadata);
        if (DPL_SUCCESS != ret)
            goto end;
    }
    ret = dpl_s3_req_build(req, 0u, &headers_request);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_try_connect(ctx, req, &conn);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_add_host_to_headers(req, headers_request);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_s3_add_authorization_to_headers(req, headers_request, NULL, NULL);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_gen_http_request(ctx, req, headers_request, NULL,
                                   header, sizeof (header), &header_len);
    if (DPL_SUCCESS != ret)
        goto end;

    iov[n_iov].iov_base = header;
    iov[n_iov].iov_len = header_len;
    n_iov++;

    //final crlf
    iov[n_iov].iov_base = "\r\n";
    iov[n_iov].iov_len = 2;
    n_iov++;

    //buffer
    iov[n_iov].iov_base = (void *)databuf;
    iov[n_iov].iov_len = databuflen;
    n_iov++;

    ret = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
    if (DPL_SUCCESS != ret)
    {
        DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
        connection_close = 1;
        goto end;
    }

    ret = dpl_read_http_reply_ext(conn, 1, 0, &replybuf, &replybuflen,
                                  &headers_reply, &connection_close);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = _multipart_parse_complete(ctx, replybuf, replybuflen);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = DPL_SUCCESS;

end:
    free(databuf);
    if (NULL != conn)
    {
        if (1 == connection_close)
            dpl_conn_terminate(conn);
        else
            dpl_conn_release(conn);
    }

    if (NULL != headers_reply)
        dpl_dict_free(headers_reply);

    if (NULL != headers_request)
        dpl_dict_free(headers_request);

    if (NULL != req)
        dpl_req_free(req);

    return ret;
}
Beispiel #7
0
dpl_status_t
dpl_s3_delete_bucket(dpl_ctx_t *ctx,
                     const char *bucket,
                     char **locationp)
{
  int           ret, ret2;
  dpl_conn_t    *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_DELETE);

  if (NULL == bucket)
    {
      ret = DPL_EINVAL;
      goto end;
    }

  ret2 = dpl_req_set_bucket(req, bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_resource(req, "/");
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #8
0
dpl_status_t
dpl_sproxyd_get_id(dpl_ctx_t *ctx,
                   const char *bucket,
                   const char *resource,
                   const char *subresource,
                   const dpl_option_t *option,
                   dpl_ftype_t object_type,
                   const dpl_condition_t *condition,
                   const dpl_range_t *range,
                   char **data_bufp,
                   unsigned int *data_lenp,
                   dpl_dict_t **metadatap,
                   dpl_sysmd_t *sysmdp,
                   char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  char          *data_buf = NULL;
  u_int         data_len;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_sproxyd_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_GET);

  ret2 = dpl_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (NULL != condition)
    {
      dpl_req_set_condition(req, condition);
    }

  if (range)
    {
      ret2 = dpl_req_add_range(req, range->start, range->end);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (option)
    {
      if (option->mask & DPL_OPTION_CONSISTENT)
        req_mask |= DPL_SPROXYD_REQ_CONSISTENT;
    }

  dpl_req_set_object_type(req, object_type);

  //build request
  ret2 = dpl_sproxyd_req_build(req, req_mask, -1, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  if (option && option->mask & DPL_OPTION_NOALLOC)
    {
      data_buf = *data_bufp;
      data_len = *data_lenp;
    }

  ret2 = dpl_read_http_reply_ext(conn, 1,
                                 (option && option->mask & DPL_OPTION_NOALLOC) ? 1 : 0,
                                 &data_buf, &data_len, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_sproxyd_get_metadata_from_headers(headers_reply, metadatap, sysmdp);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != data_bufp)
    {
      *data_bufp = data_buf;
      data_buf = NULL; //consume it
    }

  if (NULL != data_lenp)
    *data_lenp = data_len;

  ret = DPL_SUCCESS;

 end:

  if ((option && !(option->mask & DPL_OPTION_NOALLOC)) && NULL != data_buf)
    free(data_buf);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #9
0
dpl_status_t
dpl_s3_list_bucket(dpl_ctx_t *ctx,
                   const char *bucket,
                   const char *prefix,
                   const char *delimiter,
                   const int max_keys,
                   dpl_vec_t **objectsp,
                   dpl_vec_t **common_prefixesp,
                   char **locationp)
{
  int           ret, ret2;
  dpl_conn_t    *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  char          *data_buf = NULL;
  u_int         data_len;
  dpl_vec_t     *objects = NULL;
  dpl_vec_t     *common_prefixes = NULL;
  dpl_dict_t    *query_params = NULL;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_GET);

  if (NULL == bucket)
    {
      ret = DPL_EINVAL;
      goto end;
    }

  ret2 = dpl_req_set_bucket(req, bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_resource(req, "/");
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  query_params = dpl_dict_new(13);
  if (NULL == query_params)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  if (NULL != prefix)
    {
      ret2 = dpl_dict_add(query_params, "prefix", prefix, 0);
      if (DPL_SUCCESS != ret2)
        {
          ret = DPL_ENOMEM;
          goto end;
        }
    }

  if (NULL != delimiter)
    {
      ret2 = dpl_dict_add(query_params, "delimiter", delimiter, 0);
      if (DPL_SUCCESS != ret2)
        {
          ret = DPL_ENOMEM;
          goto end;
        }
    }

  if (-1 != max_keys)
    {
      char tmp[32] = "";

      snprintf(tmp, sizeof tmp, "%d", max_keys);
      ret2 = dpl_dict_add(query_params, "max-keys", tmp, 0);
      if (DPL_SUCCESS != ret2)
        {
          ret = DPL_ENOMEM;
          goto end;
        }
    }

  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, query_params, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, &data_buf, &data_len, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  objects = dpl_vec_new(2, 2);
  if (NULL == objects)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  common_prefixes = dpl_vec_new(2, 2);
  if (NULL == common_prefixes)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  ret = dpl_s3_parse_list_bucket(ctx, data_buf, data_len, objects, common_prefixes);
  if (DPL_SUCCESS != ret)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != objectsp)
    {
      *objectsp = objects;
      objects = NULL; //consume it
    }

  if (NULL != common_prefixesp)
    {
      *common_prefixesp = common_prefixes;
      common_prefixes = NULL; //consume it
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != objects)
    dpl_vec_objects_free(objects);

  if (NULL != common_prefixes)
    dpl_vec_common_prefixes_free(common_prefixes);

  if (NULL != data_buf)
    free(data_buf);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != query_params)
    dpl_dict_free(query_params);

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #10
0
/**
 * make a bucket
 *
 * @param ctx
 * @param bucket
 * @param location_constraint
 * @param sysmd
 *
 * @return
 */
dpl_status_t
dpl_s3_make_bucket(dpl_ctx_t *ctx,
                   const char *bucket,
                   const dpl_sysmd_t *sysmd,
                   char **locationp)
{
  int           ret, ret2;
  dpl_conn_t    *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  char          *location_constraint_str;
  char          data_str[1024];
  u_int         data_len;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_PUT);

  if (NULL == bucket)
    {
      ret = DPL_EINVAL;
      goto end;
    }

  ret2 = dpl_req_set_bucket(req, bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_resource(req, "/");
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (sysmd)
    {
      if (sysmd->mask & DPL_SYSMD_MASK_LOCATION_CONSTRAINT)
        {
          if (DPL_LOCATION_CONSTRAINT_US_STANDARD == sysmd->location_constraint)
            {
              data_str[0] = 0;
              data_len = 0;
            }
          else
            {
              location_constraint_str = dpl_location_constraint_str(sysmd->location_constraint);
              if (NULL == location_constraint_str)
                {
                  ret = DPL_ENOMEM;
                  goto end;
                }

              snprintf(data_str, sizeof (data_str),
                       "<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
                       "<LocationConstraint>%s</LocationConstraint>\n"
                       "</CreateBucketConfiguration>\n",
                       location_constraint_str);

              data_len = strlen(data_str);
            }
        }
      else
        {
          data_str[0] = 0;
          data_len = 0;
        }
    }
  else
    {
      data_str[0] = 0;
      data_len = 0;
    }

  dpl_req_set_data(req, data_str, data_len);

  if (sysmd)
    {
      if (sysmd->mask & DPL_SYSMD_MASK_CANNED_ACL)
        dpl_req_set_canned_acl(req, sysmd->canned_acl);

      if (sysmd->mask & DPL_SYSMD_MASK_STORAGE_CLASS)
        dpl_req_set_storage_class(req, sysmd->storage_class);
    }

  //build request
  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  //buffer
  iov[n_iov].iov_base = data_str;
  iov[n_iov].iov_len = data_len;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #11
0
dpl_status_t
dpl_s3_copy(dpl_ctx_t *ctx,
            const char *src_bucket,
            const char *src_resource,
            const char *src_subresource,
            const char *dst_bucket,
            const char *dst_resource,
            const char *dst_subresource,
            const dpl_option_t *option,
            dpl_ftype_t object_type,
            dpl_copy_directive_t copy_directive,
            const dpl_dict_t *metadata,
            const dpl_sysmd_t *sysmd,
            const dpl_condition_t *condition,
            char **locationp)
{
  int           ret, ret2;
  dpl_conn_t    *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_PUT);

  req_mask |= DPL_S3_REQ_COPY;

  if (NULL != condition)
    dpl_req_set_copy_source_condition(req, condition);

  if (NULL == src_bucket || NULL == dst_bucket)
    {
      ret = DPL_EINVAL;
      goto end;
    }

  ret2 = dpl_req_set_bucket(req, dst_bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_resource(req, dst_resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != dst_subresource)
    {
      ret2 = dpl_req_set_subresource(req, dst_subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  ret2 = dpl_req_set_src_bucket(req, src_bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_src_resource(req, src_resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != src_subresource)
    {
      ret2 = dpl_req_set_src_subresource(req, src_subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  dpl_req_set_copy_directive(req, copy_directive);

  if (NULL != metadata)
    {
      ret2 = dpl_req_add_metadata(req, metadata);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (sysmd)
    {
      if (sysmd->mask & DPL_SYSMD_MASK_CANNED_ACL)
        dpl_req_set_canned_acl(req, sysmd->canned_acl);
    }

  //build request
  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = DPL_FAILURE;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #12
0
dpl_status_t
dpl_s3_list_all_my_buckets(dpl_ctx_t *ctx,
                           dpl_vec_t **vecp,
                           char **locationp)
{
  int           ret, ret2;
  dpl_conn_t    *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  char          *data_buf = NULL;
  u_int         data_len;
  dpl_vec_t     *vec = NULL;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_GET);

  ret2 = dpl_req_set_resource(req, "/");
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //build request
  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, &data_buf, &data_len, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  vec = dpl_vec_new(2, 2);
  if (NULL == vec)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  ret2 = dpl_s3_parse_list_all_my_buckets(ctx, data_buf, data_len, vec);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != vecp)
    {
      *vecp = vec;
      vec = NULL; //consume vec
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != vec)
    dpl_vec_buckets_free(vec);

  if (NULL != data_buf)
    free(data_buf);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #13
0
dpl_status_t
dpl_s3_get_buffered(dpl_ctx_t *ctx,
                    const char *bucket,
                    const char *resource,
                    const char *subresource,
                    const dpl_option_t *option,
                    dpl_ftype_t object_type,
                    const dpl_condition_t *condition,
                    const dpl_range_t *range,
                    dpl_metadatum_func_t metadatum_func,
                    dpl_dict_t **metadatap,
                    dpl_sysmd_t *sysmdp,
                    dpl_buffer_func_t buffer_func,
                    void *cb_arg,
                    char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_req_t     *req = NULL;
  struct get_conven gc;
  int http_status;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  memset(&gc, 0, sizeof (gc));
  gc.metadata = dpl_dict_new(13);
  if (NULL == gc.metadata)
    {
      ret = DPL_ENOMEM;
      goto end;
    }
  gc.sysmdp = sysmdp;
  gc.metadatum_func = metadatum_func;
  gc.buffer_func = buffer_func;
  gc.cb_arg = cb_arg;

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_GET);

  if (NULL == bucket)
    {
      ret = DPL_EINVAL;
      goto end;
    }

  ret2 = dpl_req_set_bucket(req, bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (NULL != condition)
    {
      dpl_req_set_condition(req, condition);
    }

  //build request
  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      gc.connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply_buffered(conn, 1, &http_status, cb_get_header, cb_get_buffer, &gc);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "read http answer failed");
      gc.connection_close = 1;
      ret = ret2;
      goto end;
    }

  if (!conn->ctx->keep_alive)
    gc.connection_close = 1;

  if (NULL != metadatap)
    {
      *metadatap = gc.metadata;
      gc.metadata = NULL;
    }
  
  //map http_status to relevant value
  ret = dpl_map_http_status(http_status);

  //caller is responsible for logging the event

 end:

  if (NULL != conn)
    {
      if (1 == gc.connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != gc.metadata)
    dpl_dict_free(gc.metadata);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #14
0
dpl_status_t
dpl_cdmi_get(dpl_ctx_t *ctx,
             const char *bucket,
             const char *resource,
             const char *subresource,
             const dpl_option_t *option,
             dpl_ftype_t object_type,
             const dpl_condition_t *condition,
             const dpl_range_t *range,
             char **data_bufp,
             unsigned int *data_lenp,
             dpl_dict_t **metadatap,
             dpl_sysmd_t *sysmdp,
             char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  char          *data_buf = NULL;
  u_int         data_len;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  int raw = 0;
  dpl_value_t *val = NULL;
  dpl_dict_var_t *var = NULL;
  dpl_dict_var_t *encoding = NULL;
  int value_len;
  int orig_len;
  char *orig_buf = NULL;
  dpl_cdmi_req_mask_t req_mask = 0u;
  char *location;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  if (option)
    {
      if (option->mask & DPL_OPTION_HTTP_COMPAT)
        req_mask |= DPL_CDMI_REQ_HTTP_COMPAT;

      if (option->mask & DPL_OPTION_RAW)
        raw = 1;
    }

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_GET);

  if (NULL != bucket)
    {
      ret2 = dpl_req_set_bucket(req, bucket);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  ret2 = dpl_cdmi_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL == subresource)
    {
      if (DPL_FTYPE_REG == object_type)
        {
          subresource = "valuetransferencoding";
        }
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (NULL != condition)
    {
      dpl_req_set_condition(req, condition);
    }

  if (range)
    {
      ret2 = dpl_cdmi_req_add_range(req, req_mask, range);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  dpl_req_set_object_type(req, object_type);

  //build request
  ret2 = dpl_cdmi_req_build(req, 0, &headers_request, NULL, NULL);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  if (option && option->mask & DPL_OPTION_NOALLOC)
    {
      data_buf = *data_bufp;
      data_len = *data_lenp;
    }

  ret2 = dpl_read_http_reply_ext(conn, 1, 
                                 (option && option->mask & DPL_OPTION_NOALLOC) ? 1 : 0,
                                 &data_buf, &data_len, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      if (DPL_EREDIRECT == ret2)
        {
          if (NULL != locationp)
            {
              location = dpl_location(headers_reply);
              if (NULL == location)
                {
                  DPL_TRACE(conn->ctx, DPL_TRACE_ERR,
                            "missing \"Location\" header in redirect HTTP response");
                  connection_close = 1;
                  ret = DPL_FAILURE;
                  goto end;
                }
              *locationp = strdup(location);
            }
        }
      ret = ret2;
      goto end;
    }

  if (req_mask & DPL_CDMI_REQ_HTTP_COMPAT)
    {
      //metadata are in headers
      ret2 = dpl_cdmi_get_metadata_from_headers(headers_reply, metadatap, sysmdp);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }
  else
    {
      if (!raw)
        {
          char *tmp;

          //extract data+metadata from json
          ret2 = dpl_cdmi_parse_json_buffer(ctx, data_buf, data_len, &val);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }
          
          if (DPL_VALUE_SUBDICT != val->type)
            {
              ret = DPL_EINVAL;
              goto end;
            }

          ret2 = dpl_cdmi_get_metadata_from_values(val->subdict, metadatap, sysmdp);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }
          
          //find the value object
          ret2 = dpl_dict_get_lowered(val->subdict, "value", &var);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }
          
          if (DPL_VALUE_STRING != var->val->type)
            {
              ret = DPL_EINVAL;
              goto end;
            }
          
          ret2 = dpl_dict_get_lowered(val->subdict, "valuetransferencoding",
                                      &encoding);
          if (DPL_SUCCESS != ret2)
            {
              ret = ret2;
              goto end;
            }

          if (DPL_VALUE_STRING != encoding->val->type)
            {
              ret = DPL_EINVAL;
              goto end;
            }

          value_len = var->val->string->len;

          if (0 == strcmp(dpl_sbuf_get_str(encoding->val->string), "base64"))
            {
              orig_len = DPL_BASE64_ORIG_LENGTH(value_len);
              orig_buf = malloc(orig_len);
              if (NULL == orig_buf)
                {
                  ret = DPL_ENOMEM;
                  goto end;
                }
          
              orig_len = dpl_base64_decode((u_char *) dpl_sbuf_get_str(var->val->string), value_len, (u_char *) orig_buf);
          
              //swap pointers
              tmp = data_buf;
              data_buf = orig_buf;
              orig_buf = tmp;
              data_len = orig_len;
            }
          else if (0 == strcmp(dpl_sbuf_get_str(encoding->val->string), "utf-8"))
            {
              orig_buf = data_buf;
              orig_len = data_len;
              data_buf = malloc(value_len + 1);
              if (NULL == data_buf)
                {
                  ret = DPL_ENOMEM;
                  goto end;
                }
              memcpy(data_buf, dpl_sbuf_get_str(var->val->string), value_len);
              data_buf[value_len] = '\0';
              data_len = value_len;
            }
          else
            {
              DPL_TRACE(conn->ctx, DPL_TRACE_ERR,
                        "unknown \"valuetransferencoding\" received: \"%s\"",
                        encoding->val->string);
              ret = DPL_EINVAL;
              goto end;
            }
        }
    }

  if (NULL != data_bufp)
    {
      *data_bufp = data_buf;
      data_buf = NULL; //consume it
    }

  if (NULL != data_lenp)
    *data_lenp = data_len;

  ret = DPL_SUCCESS;

 end:

  if (NULL != orig_buf)
    free(orig_buf);

  if (NULL != val)
    dpl_value_free(val);

  if ((option && !(option->mask & DPL_OPTION_NOALLOC)) && NULL != data_buf)
    free(data_buf);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #15
0
dpl_status_t
dpl_srws_put_internal(dpl_ctx_t *ctx,
                      const char *bucket,
                      const char *resource,
                      const char *subresource,
                      const dpl_option_t *option,
                      dpl_ftype_t object_type,
                      const dpl_condition_t *condition,
                      const dpl_range_t *range,
                      const dpl_dict_t *metadata,
                      const dpl_sysmd_t *sysmd,
                      const char *data_buf,
                      unsigned int data_len,
                      int mdonly,
                      char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_srws_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_PUT);

  ret2 = dpl_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  if (option)
    {
      if (option->mask & DPL_OPTION_LAZY)
        req_mask |= DPL_SRWS_REQ_LAZY;
    }

  dpl_req_set_object_type(req, object_type);

  if (option)
    {
      if (option->mask & DPL_OPTION_LAZY)
        req_mask |= DPL_SRWS_REQ_LAZY;
    }

  if (mdonly)
    {
      req_mask |= DPL_SRWS_REQ_MD_ONLY;
    }
  else
    {
      dpl_req_set_data(req, data_buf, data_len);
    }

  dpl_req_add_behavior(req, DPL_BEHAVIOR_MD5);

  if (option)
    {
      if (option->mask & DPL_OPTION_LAZY)
        req_mask |= DPL_SRWS_REQ_LAZY;
    }

  if (NULL != metadata)
    {
      ret2 = dpl_req_add_metadata(req, metadata);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  //build request
  ret2 = dpl_srws_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  //buffer
  iov[n_iov].iov_base = (void *)data_buf;
  iov[n_iov].iov_len = data_len;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #16
0
dpl_status_t
dpl_cdmi_list_bucket(dpl_ctx_t *ctx,
                     const char *bucket,
                     const char *prefix,
                     const char *delimiter,
                     const int max_keys,
                     dpl_vec_t **objectsp,
                     dpl_vec_t **common_prefixesp,
                     char **locationp)
{
  int           ret, ret2;
  dpl_conn_t   *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  char          *data_buf = NULL;
  u_int         data_len;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_vec_t     *common_prefixes = NULL;
  dpl_vec_t     *objects = NULL;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_GET);

  ret2 = dpl_cdmi_req_set_resource(req, NULL != prefix ? prefix : "/");
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  dpl_req_set_object_type(req, DPL_FTYPE_DIR);

  //build request
  ret2 = dpl_cdmi_req_build(req, 0, &headers_request, NULL, NULL);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  //contact default host
  dpl_req_rm_behavior(req, DPL_BEHAVIOR_VIRTUAL_HOSTING);

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, &data_buf, &data_len, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  objects = dpl_vec_new(2, 2);
  if (NULL == objects)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  common_prefixes = dpl_vec_new(2, 2);
  if (NULL == common_prefixes)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  ret2 = dpl_cdmi_parse_list_bucket(ctx, data_buf, data_len, prefix, objects, common_prefixes);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != objectsp)
    {
      *objectsp = objects;
      objects = NULL; //consume it
    }

  if (NULL != common_prefixesp)
    {
      *common_prefixesp = common_prefixes;
      common_prefixes = NULL; //consume it
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != objects)
    dpl_vec_objects_free(objects);

  if (NULL != common_prefixes)
    dpl_vec_common_prefixes_free(common_prefixes);

  if (NULL != data_buf)
    free(data_buf);

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #17
0
dpl_status_t
dpl_s3_stream_multipart_init(dpl_ctx_t *ctx,
                             const char *bucket,
                             const char *resource,
                             const char **uploadidp)
{
    dpl_status_t  ret;
    dpl_conn_t    *conn = NULL;
    char          header[dpl_header_size];
    u_int         header_len;
    struct iovec  iov[10];
    int           n_iov = 0;
    int           connection_close = 0;
    dpl_dict_t    *headers_request = NULL;
    dpl_dict_t    *headers_reply = NULL;
    dpl_req_t     *req = NULL;
    char          *replybuf = NULL;
    unsigned int  replybuflen = 0;

    req = dpl_req_new(ctx);
    if (NULL == req)
    {
        ret = DPL_ENOMEM;
        goto end;
    }

    dpl_req_set_method(req, DPL_METHOD_POST);

    if (NULL == bucket)
    {
        ret = DPL_EINVAL;
        goto end;
    }

    ret = dpl_req_set_bucket(req, bucket);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_set_resource(req, resource);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_set_subresource(req, "uploads");
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_s3_req_build(req, 0u, &headers_request);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_try_connect(ctx, req, &conn);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_add_host_to_headers(req, headers_request);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_s3_add_authorization_to_headers(req, headers_request, NULL, NULL);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = dpl_req_gen_http_request(ctx, req, headers_request, NULL,
                                   header, sizeof (header), &header_len);
    if (DPL_SUCCESS != ret)
        goto end;

    iov[n_iov].iov_base = header;
    iov[n_iov].iov_len = header_len;
    n_iov++;

    //final crlf
    iov[n_iov].iov_base = "\r\n";
    iov[n_iov].iov_len = 2;
    n_iov++;

    ret = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
    if (DPL_SUCCESS != ret)
    {
        DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
        connection_close = 1;
        goto end;
    }

    ret = dpl_read_http_reply_ext(conn, 1, 0, &replybuf, &replybuflen,
                                  &headers_reply, &connection_close);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = _multipart_parse_init(ctx, replybuf, replybuflen, uploadidp);
    if (DPL_SUCCESS != ret)
        goto end;

    ret = DPL_SUCCESS;

end:
    if (NULL != conn)
    {
        if (1 == connection_close)
            dpl_conn_terminate(conn);
        else
            dpl_conn_release(conn);
    }

    if (NULL != headers_reply)
        dpl_dict_free(headers_reply);

    if (NULL != headers_request)
        dpl_dict_free(headers_request);

    if (NULL != req)
        dpl_req_free(req);

    return ret;
}
Beispiel #18
0
dpl_status_t
dpl_s3_put(dpl_ctx_t *ctx,
           const char *bucket,
           const char *resource,
           const char *subresource,
           const dpl_option_t *option,
           dpl_ftype_t object_type,
           const dpl_condition_t *condition,
           const dpl_range_t *range,
           const dpl_dict_t *metadata,
           const dpl_sysmd_t *sysmd,
           const char *data_buf,
           unsigned int data_len,
           const dpl_dict_t *query_params,
           dpl_sysmd_t *returned_sysmdp,
           char **locationp)
{
  int           ret, ret2;
  dpl_conn_t    *conn = NULL;
  char          header[dpl_header_size];
  u_int         header_len;
  struct iovec  iov[10];
  int           n_iov = 0;
  int           connection_close = 0;
  dpl_dict_t    *headers_request = NULL;
  dpl_dict_t    *headers_reply = NULL;
  dpl_req_t     *req = NULL;
  dpl_s3_req_mask_t req_mask = 0u;

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "");

  req = dpl_req_new(ctx);
  if (NULL == req)
    {
      ret = DPL_ENOMEM;
      goto end;
    }

  dpl_req_set_method(req, DPL_METHOD_PUT);

  if (NULL == bucket)
    {
      ret = DPL_EINVAL;
      goto end;
    }

  ret2 = dpl_req_set_bucket(req, bucket);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_set_resource(req, resource);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  if (NULL != subresource)
    {
      ret2 = dpl_req_set_subresource(req, subresource);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  dpl_req_set_data(req, data_buf, data_len);

  dpl_req_add_behavior(req, DPL_BEHAVIOR_MD5);

  if (sysmd)
    {
      if (sysmd->mask & DPL_SYSMD_MASK_CANNED_ACL)
        dpl_req_set_canned_acl(req, sysmd->canned_acl);

      if (sysmd->mask & DPL_SYSMD_MASK_STORAGE_CLASS)
        dpl_req_set_storage_class(req, sysmd->storage_class);
    }

  if (NULL != metadata)
    {
      ret2 = dpl_req_add_metadata(req, metadata);
      if (DPL_SUCCESS != ret2)
        {
          ret = ret2;
          goto end;
        }
    }

  ret2 = dpl_s3_req_build(req, req_mask, &headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_try_connect(ctx, req, &conn);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_add_host_to_headers(req, headers_request);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_s3_add_authorization_to_headers(req, headers_request, NULL, NULL);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret2 = dpl_req_gen_http_request(ctx, req, headers_request, NULL, header, sizeof (header), &header_len);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len = header_len;
  n_iov++;

  //final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  //buffer
  iov[n_iov].iov_base = (void *)data_buf;
  iov[n_iov].iov_len = data_len;
  n_iov++;

  ret2 = dpl_conn_writev_all(conn, iov, n_iov, conn->ctx->write_timeout);
  if (DPL_SUCCESS != ret2)
    {
      DPL_TRACE(conn->ctx, DPL_TRACE_ERR, "writev failed");
      connection_close = 1;
      ret = ret2;
      goto end;
    }

  ret2 = dpl_read_http_reply(conn, 1, NULL, NULL, &headers_reply, &connection_close);
  if (DPL_SUCCESS != ret2)
    {
      ret = ret2;
      goto end;
    }

  ret = DPL_SUCCESS;

 end:

  if (NULL != conn)
    {
      if (1 == connection_close)
        dpl_conn_terminate(conn);
      else
        dpl_conn_release(conn);
    }

  if (NULL != headers_reply)
    dpl_dict_free(headers_reply);

  if (NULL != headers_request)
    dpl_dict_free(headers_request);

  if (NULL != req)
    dpl_req_free(req);

  DPL_TRACE(ctx, DPL_TRACE_BACKEND, "ret=%d", ret);

  return ret;
}
Beispiel #19
0
static dpl_status_t
delete_all(dpl_ctx_t *ctx, const char *bucket, dpl_locators_t *locators,
           struct dall_req *dctx, dpl_vec_t **objectsp)
{
  dpl_status_t  ret;
  struct iovec  iov[10];
  int           n_iov = 0;
  char          header[dpl_header_size];
  u_int         header_len;

  dctx->req = dpl_req_new(ctx);
  if (dctx->req == NULL)
    return DPL_ENOMEM;

  dpl_req_set_method(dctx->req, DPL_METHOD_POST);

  if (bucket == NULL)
    return DPL_EINVAL;

  ret = dpl_req_set_bucket(dctx->req, bucket);
  if (ret != DPL_SUCCESS)
    return ret;

  ret = dpl_req_set_resource(dctx->req, "/");
  if (ret != DPL_SUCCESS)
    return ret;

  ret = dpl_req_set_subresource(dctx->req, "delete");
  if (ret != DPL_SUCCESS)
    return ret;

  dctx->query_params = dpl_dict_new(1);
  if (dctx->query_params == NULL)
    return ENOMEM;

  /* Note: Juste for authentification signature calcul */
  ret = dpl_dict_add(dctx->query_params, "delete", "", 0);
  if (DPL_SUCCESS != ret)
    return ret;

  dctx->data = dpl_sbuf_new(4096);
  if (dctx->data == NULL)
    return DPL_ENOMEM;

  ret = get_delete_data_content(ctx, locators, dctx->data);
  if (ret != DPL_SUCCESS)
    return ret;

  dpl_req_set_data(dctx->req, dctx->data->buf, dctx->data->len);
  dpl_req_add_behavior(dctx->req, DPL_BEHAVIOR_MD5);

  ret = dpl_s3_req_build(dctx->req, 0, &dctx->headers);
  if (ret != DPL_SUCCESS)
    return ret;

  ret = dpl_try_connect(ctx, dctx->req, &dctx->conn);
  if (ret != DPL_SUCCESS)
    return ret;

  ret = dpl_add_host_to_headers(dctx->req, dctx->headers);
  if (ret != DPL_SUCCESS)
    return ret;

  ret = dpl_s3_add_authorization_to_headers(dctx->req, dctx->headers, dctx->query_params, NULL);
  if (ret != DPL_SUCCESS)
    return ret;

  ret = dpl_req_gen_http_request(ctx, dctx->req, dctx->headers, NULL,
                                 header, sizeof(header), &header_len);
  if (ret != DPL_SUCCESS)
    return ret;

  // Headers
  iov[n_iov].iov_base = header;
  iov[n_iov].iov_len  = header_len;
  n_iov++;

  // Final crlf
  iov[n_iov].iov_base = "\r\n";
  iov[n_iov].iov_len = 2;
  n_iov++;

  // Data
  iov[n_iov].iov_base = (void *) dctx->data->buf;
  iov[n_iov].iov_len = dctx->data->len;
  n_iov++;

  ret = dpl_conn_writev_all(dctx->conn, iov, n_iov, ctx->write_timeout);
  if (ret != DPL_SUCCESS) {
    dctx->connection_close = 1;
    return ret;
  }

  ret = dpl_read_http_reply(dctx->conn, 1,
                            &dctx->answer_data, &dctx->answer_len,
                            NULL, &dctx->connection_close);

  if (dctx->answer_len > 0) {
    dpl_status_t        parse_ret;

    dctx->objects = dpl_vec_new(2, 2);
    if (dctx->objects == NULL)
      return DPL_ENOMEM;

    parse_ret = dpl_s3_parse_delete_all(ctx, dctx->answer_data, dctx->answer_len, dctx->objects);
    if (parse_ret == DPL_SUCCESS && objectsp != NULL) {
      *objectsp = dctx->objects;
      dctx->objects = NULL;
    }

    if (ret == DPL_SUCCESS)
      ret = parse_ret;
  }

  return ret;
}