void http_connection_get_container_meta (HttpConnection *con,
HttpConnection_container_meta_cb container_meta_cb, gpointer ctx)
{
gchar *req_path;
gboolean res;
ContainerMeta *meta;
LOG_debug (CON_CONT, "Getting container meta for: %s", application_get_container_name (con->app));
// acquire HTTP client
http_connection_acquire (con);
meta = g_new0 (ContainerMeta, 1);
meta->ctx = ctx;
meta->container_meta_cb = container_meta_cb;
req_path = g_strdup_printf ("/%s", application_get_container_name (con->app));
res = http_connection_make_request_to_storage_url (con,
req_path, "HEAD", NULL,
http_connection_on_container_meta_cb,
meta
);
g_free (req_path);
if (!res) {
LOG_err (CON_CONT, "Failed to create HTTP request !");
container_meta_cb (ctx, FALSE);
http_connection_release (con);
return;
}
}
static void fileio_simple_download_on_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIOSimpleDownload *fsim = (FileIOSimpleDownload *) ctx;
gboolean res;
LOG_debug (FIO_LOG, CON_H"Downloading data.", con);
http_connection_acquire (con);
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
res = http_connection_make_request (con,
fsim->fname, "GET", NULL, TRUE, NULL,
fileio_simple_download_on_sent_cb,
fsim
);
if (!res) {
LOG_err (FIO_LOG, CON_H"Failed to create HTTP request !", con);
http_connection_release (con);
fsim->on_download_cb (fsim->ctx, FALSE, NULL, 0);
fileio_simple_download_destroy (fsim);
return;
}
}
// got HttpConnection object
static void fileio_write_on_multipart_init_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileWriteData *wdata = (FileWriteData *) ctx;
gboolean res;
gchar *path;
http_connection_acquire (con);
path = g_strdup_printf ("%s?uploads", wdata->fop->fname);
// send storage class with the init request
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
res = http_connection_make_request (con,
path, "POST", NULL, TRUE, NULL,
fileio_write_on_multipart_init_cb,
wdata
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (wdata->ino), con);
http_connection_release (con);
wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0);
g_free (wdata);
return;
}
}
// got HttpConnection object
static void fileio_write_on_send_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileWriteData *wdata = (FileWriteData *) ctx;
gchar *path;
gboolean res;
FileIOPart *part;
size_t buf_len;
const gchar *buf;
http_connection_acquire (con);
// add part information to the list
part = g_new0 (FileIOPart, 1);
part->part_number = wdata->fop->part_number;
buf_len = evbuffer_get_length (wdata->fop->write_buf);
buf = (const gchar *) evbuffer_pullup (wdata->fop->write_buf, buf_len);
// XXX: move to separate thread
// 1. calculate MD5 of a part.
get_md5_sum (buf, buf_len, &part->md5str, &part->md5b);
// 2. calculate MD5 of multiple message blocks
MD5_Update (&wdata->fop->md5, buf, buf_len);
wdata->fop->l_parts = g_list_append (wdata->fop->l_parts, part);
path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s",
wdata->fop->fname, wdata->fop->part_number, wdata->fop->uploadid);
// increase part number
wdata->fop->part_number++;
// XXX: check that part_number does not exceeds 10000
// add output headers
http_connection_add_output_header (con, "Content-MD5", part->md5b);
res = http_connection_make_request (con,
path, "PUT", wdata->fop->write_buf, TRUE, NULL,
fileio_write_on_send_cb,
wdata
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (wdata->ino), con);
http_connection_release (con);
wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0);
g_free (wdata);
return;
}
}
// got HttpConnection object
static void fileio_release_on_update_headers_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIO *fop = (FileIO *) ctx;
gchar *path;
gchar *cpy_path;
gboolean res;
unsigned char digest[16];
gchar *md5str;
size_t i;
LOG_debug (FIO_LOG, INO_CON_H"Updating object's headers..", INO_T (fop->ino), con);
http_connection_acquire (con);
if (fop->content_type)
http_connection_add_output_header (con, "Content-Type", fop->content_type);
http_connection_add_output_header (con, "x-amz-metadata-directive", "REPLACE");
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
MD5_Final (digest, &fop->md5);
md5str = g_malloc (33);
for (i = 0; i < 16; ++i)
sprintf(&md5str[i*2], "%02x", (unsigned int)digest[i]);
http_connection_add_output_header (con, "x-amz-meta-md5", md5str);
g_free (md5str);
cpy_path = g_strdup_printf ("%s%s", conf_get_string (application_get_conf (fop->app), "s3.bucket_name"), fop->fname);
http_connection_add_output_header (con, "x-amz-copy-source", cpy_path);
g_free (cpy_path);
path = g_strdup_printf ("%s", fop->fname);
res = http_connection_make_request (con,
path, "PUT", NULL, TRUE, NULL,
fileio_release_on_update_header_cb,
fop
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con);
http_connection_release (con);
fileio_destroy (fop);
return;
}
}
// got HttpConnection object
static void fileio_read_on_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileReadData *rdata = (FileReadData *) ctx;
gboolean res;
guint64 part_size;
http_connection_acquire (con);
part_size = conf_get_uint (application_get_conf (rdata->fop->app), "s3.part_size");
// small file - get the whole file at once
if (rdata->fop->file_size < part_size)
rdata->request_offset = 0;
// calculate offset
else {
gchar *range_hdr;
if (part_size < rdata->size)
part_size = rdata->size;
rdata->request_offset = rdata->off;
range_hdr = g_strdup_printf ("bytes=%"G_GUINT64_FORMAT"-%"G_GUINT64_FORMAT,
(gint64)rdata->request_offset, (gint64)(rdata->request_offset + part_size));
http_connection_add_output_header (con, "Range", range_hdr);
g_free (range_hdr);
}
res = http_connection_make_request (con,
rdata->fop->fname, "GET", NULL, TRUE, NULL,
fileio_read_on_get_cb,
rdata
);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (rdata->ino), con);
http_connection_release (con);
rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0);
g_free (rdata);
return;
}
}
// got HttpConnection object
static void fileio_release_on_complete_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIO *fop = (FileIO *) ctx;
gchar *path;
gboolean res;
struct evbuffer *xml_buf;
GList *l;
xml_buf = evbuffer_new ();
evbuffer_add_printf (xml_buf, "%s", "<CompleteMultipartUpload>");
for (l = g_list_first (fop->l_parts); l; l = g_list_next (l)) {
FileIOPart *part = (FileIOPart *) l->data;
evbuffer_add_printf (xml_buf,
"<Part><PartNumber>%u</PartNumber><ETag>\"%s\"</ETag></Part>",
part->part_number, part->md5str);
}
evbuffer_add_printf (xml_buf, "%s", "</CompleteMultipartUpload>");
LOG_debug (FIO_LOG, INO_CON_H"Sending Multipart Final part..", INO_T (fop->ino), con);
http_connection_acquire (con);
path = g_strdup_printf ("%s?uploadId=%s",
fop->fname, fop->uploadid);
res = http_connection_make_request (con,
path, "POST", xml_buf, TRUE, NULL,
fileio_release_on_complete_cb,
fop
);
g_free (path);
evbuffer_free (xml_buf);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con);
http_connection_release (con);
fileio_destroy (fop);
return;
}
}
// got HttpConnection object
static void fileio_read_on_head_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileReadData *rdata = (FileReadData *) ctx;
gboolean res;
http_connection_acquire (con);
res = http_connection_make_request (con,
rdata->fop->fname, "HEAD", NULL, FALSE, NULL,
fileio_read_on_head_cb,
rdata
);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (rdata->ino), con);
http_connection_release (con);
rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0);
g_free (rdata);
return;
}
}
static void fileio_simple_upload_on_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIOSimpleUpload *fsim = (FileIOSimpleUpload *) ctx;
time_t t;
gchar str[10];
char time_str[50];
gboolean res;
LOG_debug (FIO_LOG, CON_H"Uploading data. Size: %zu", con, evbuffer_get_length (fsim->write_buf));
http_connection_acquire (con);
snprintf (str, sizeof (str), "%d", fsim->mode);
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
http_connection_add_output_header (con, "x-amz-meta-mode", str);
t = time (NULL);
if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) {
http_connection_add_output_header (con, "x-amz-meta-date", time_str);
}
res = http_connection_make_request (con,
fsim->fname, "PUT", fsim->write_buf, TRUE, NULL,
fileio_simple_upload_on_sent_cb,
fsim
);
if (!res) {
LOG_err (FIO_LOG, CON_H"Failed to create HTTP request !", con);
http_connection_release (con);
fsim->on_upload_cb (fsim->ctx, FALSE);
fileio_simple_upload_destroy (fsim);
return;
}
}
// got HttpConnection object
static void fileio_release_on_part_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIO *fop = (FileIO *) ctx;
gchar *path;
gboolean res;
FileIOPart *part;
size_t buf_len;
const gchar *buf;
LOG_debug (FIO_LOG, INO_CON_H"Releasing fop. Size: %zu", INO_T (fop->ino), con, evbuffer_get_length (fop->write_buf));
// add part information to the list
part = g_new0 (FileIOPart, 1);
part->part_number = fop->part_number;
buf_len = evbuffer_get_length (fop->write_buf);
buf = (const gchar *)evbuffer_pullup (fop->write_buf, buf_len);
// XXX: move to separate thread
// 1. calculate MD5 of a part.
get_md5_sum (buf, buf_len, &part->md5str, &part->md5b);
// 2. calculate MD5 of multiple message blocks
MD5_Update (&fop->md5, buf, buf_len);
fop->l_parts = g_list_append (fop->l_parts, part);
// if this is a multipart
if (fop->multipart_initiated) {
if (!fop->uploadid) {
LOG_err (FIO_LOG, INO_CON_H"UploadID is not set, aborting operation !", INO_T (fop->ino), con);
fileio_destroy (fop);
return;
}
path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s",
fop->fname, fop->part_number, fop->uploadid);
fop->part_number++;
} else {
path = g_strdup (fop->fname);
}
#ifdef MAGIC_ENABLED
// guess MIME type
gchar *mime_type = magic_buffer (application_get_magic_ctx (fop->app), buf, buf_len);
if (mime_type) {
LOG_debug (FIO_LOG, "Guessed MIME type of %s as %s", path, mime_type);
fop->content_type = g_strdup (mime_type);
} else {
LOG_err (FIO_LOG, "Failed to guess MIME type of %s !", path);
}
#endif
http_connection_acquire (con);
// add output headers
http_connection_add_output_header (con, "Content-MD5", part->md5b);
if (fop->content_type)
http_connection_add_output_header (con, "Content-Type", fop->content_type);
// if this is the full file
if (!fop->multipart_initiated) {
time_t t;
gchar time_str[50];
// Add current time
t = time (NULL);
if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) {
http_connection_add_output_header (con, "x-amz-meta-date", time_str);
}
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
}
res = http_connection_make_request (con,
path, "PUT", fop->write_buf, TRUE, NULL,
fileio_release_on_part_sent_cb,
fop
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con);
http_connection_release (con);
fileio_destroy (fop);
return;
}
}
