// buffer is sent static void fileio_write_on_send_cb (HttpConnection *con, void *ctx, gboolean success, G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers) { FileWriteData *wdata = (FileWriteData *) ctx; const char *versioning_header; http_connection_release (con); if (!success) { LOG_err (FIO_LOG, INO_CON_H"Failed to send bufer to server !", INO_T (wdata->ino), con); wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0); g_free (wdata); return; } versioning_header = http_find_header (headers, "x-amz-version-id"); if (versioning_header) { cache_mng_update_version_id (application_get_cache_mng (wdata->fop->app), wdata->ino, versioning_header); } // empty part buffer evbuffer_drain (wdata->fop->write_buf, -1); // done sending part wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, TRUE, wdata->buf_size); g_free (wdata); }
void http_connection_get_container_meta (HttpConnection *con, HttpConnection_container_meta_cb container_meta_cb, gpointer ctx) { gchar *req_path; gboolean res; ContainerMeta *meta; LOG_debug (CON_CONT, "Getting container meta for: %s", application_get_container_name (con->app)); // acquire HTTP client http_connection_acquire (con); meta = g_new0 (ContainerMeta, 1); meta->ctx = ctx; meta->container_meta_cb = container_meta_cb; req_path = g_strdup_printf ("/%s", application_get_container_name (con->app)); res = http_connection_make_request_to_storage_url (con, req_path, "HEAD", NULL, http_connection_on_container_meta_cb, meta ); g_free (req_path); if (!res) { LOG_err (CON_CONT, "Failed to create HTTP request !"); container_meta_cb (ctx, FALSE); http_connection_release (con); return; } }
// file is sent static void fileio_release_on_part_sent_cb (HttpConnection *con, void *ctx, gboolean success, G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers) { FileIO *fop = (FileIO *) ctx; const gchar *versioning_header; http_connection_release (con); if (!success) { LOG_err (FIO_LOG, INO_CON_H"Failed to send bufer to server !", INO_T (fop->ino), con); fileio_destroy (fop); return; } versioning_header = http_find_header (headers, "x-amz-version-id"); if (versioning_header) { cache_mng_update_version_id (application_get_cache_mng (fop->app), fop->ino, versioning_header); } // if it's a multi part upload - Complete Multipart Upload if (fop->multipart_initiated) { fileio_release_complete_multipart (fop); // or we are done } else { fileio_release_update_headers (fop); //fileio_destroy (fop); } }
static void fileio_write_on_multipart_init_cb (HttpConnection *con, void *ctx, gboolean success, const gchar *buf, size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers) { FileWriteData *wdata = (FileWriteData *) ctx; gchar *uploadid; http_connection_release (con); wdata->fop->multipart_initiated = TRUE; if (!success || !buf_len) { LOG_err (FIO_LOG, INO_CON_H"Failed to get multipart init data from the server !", INO_T (wdata->ino), con); wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0); g_free (wdata); return; } uploadid = get_uploadid (buf, buf_len); if (!uploadid) { LOG_err (FIO_LOG, INO_CON_H"Failed to parse multipart init data!", INO_T (wdata->ino), con); wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0); g_free (wdata); return; } wdata->fop->uploadid = g_strdup (uploadid); xmlFree (uploadid); // done, resume uploading part wdata->fop->part_number = 1; fileio_write_send_part (wdata); }
// got HttpConnection object static void fileio_write_on_multipart_init_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileWriteData *wdata = (FileWriteData *) ctx; gboolean res; gchar *path; http_connection_acquire (con); path = g_strdup_printf ("%s?uploads", wdata->fop->fname); // send storage class with the init request http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type")); res = http_connection_make_request (con, path, "POST", NULL, TRUE, NULL, fileio_write_on_multipart_init_cb, wdata ); g_free (path); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (wdata->ino), con); http_connection_release (con); wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0); g_free (wdata); return; } }
// multipart is sent static void fileio_release_on_complete_cb (HttpConnection *con, void *ctx, gboolean success, G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers) { FileIO *fop = (FileIO *) ctx; const gchar *versioning_header; http_connection_release (con); if (!success) { LOG_err (FIO_LOG, INO_CON_H"Failed to send Multipart data to the server !", INO_T (fop->ino), con); fileio_destroy (fop); return; } versioning_header = http_find_header (headers, "x-amz-version-id"); if (versioning_header) { cache_mng_update_version_id (application_get_cache_mng (fop->app), fop->ino, versioning_header); } // done LOG_debug (FIO_LOG, INO_CON_H"Multipart Upload is done !", INO_T (fop->ino), con); // fileio_destroy (fop); fileio_release_update_headers (fop); }
/*{{{ GET request */ static void fileio_read_on_get_cb (HttpConnection *con, void *ctx, gboolean success, const gchar *buf, size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers) { FileReadData *rdata = (FileReadData *) ctx; const char *versioning_header = NULL; // release HttpConnection http_connection_release (con); if (!success) { LOG_err (FIO_LOG, INO_CON_H"Failed to get file from server !", INO_T (rdata->ino), con); rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0); g_free (rdata); return; } // store it in the local cache cache_mng_store_file_buf (application_get_cache_mng (rdata->fop->app), rdata->ino, buf_len, rdata->request_offset, (unsigned char *) buf, NULL, NULL); // update version ID versioning_header = http_find_header (headers, "x-amz-version-id"); if (versioning_header) { cache_mng_update_version_id (application_get_cache_mng (rdata->fop->app), rdata->ino, versioning_header); } LOG_debug (FIO_LOG, INO_H"Storing [%"G_GUINT64_FORMAT" %zu]", INO_T(rdata->ino), rdata->request_offset, buf_len); // and read it fileio_read_get_buf (rdata); }
static void fileio_simple_download_on_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileIOSimpleDownload *fsim = (FileIOSimpleDownload *) ctx; gboolean res; LOG_debug (FIO_LOG, CON_H"Downloading data.", con); http_connection_acquire (con); http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type")); res = http_connection_make_request (con, fsim->fname, "GET", NULL, TRUE, NULL, fileio_simple_download_on_sent_cb, fsim ); if (!res) { LOG_err (FIO_LOG, CON_H"Failed to create HTTP request !", con); http_connection_release (con); fsim->on_download_cb (fsim->ctx, FALSE, NULL, 0); fileio_simple_download_destroy (fsim); return; } }
static void fileio_simple_download_on_sent_cb (HttpConnection *con, void *ctx, gboolean success, G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers) { FileIOSimpleDownload *fsim = (FileIOSimpleDownload *) ctx; http_connection_release (con); fsim->on_download_cb (fsim->ctx, success, buf, buf_len); fileio_simple_download_destroy (fsim); }
// got HttpConnection object static void fileio_write_on_send_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileWriteData *wdata = (FileWriteData *) ctx; gchar *path; gboolean res; FileIOPart *part; size_t buf_len; const gchar *buf; http_connection_acquire (con); // add part information to the list part = g_new0 (FileIOPart, 1); part->part_number = wdata->fop->part_number; buf_len = evbuffer_get_length (wdata->fop->write_buf); buf = (const gchar *) evbuffer_pullup (wdata->fop->write_buf, buf_len); // XXX: move to separate thread // 1. calculate MD5 of a part. get_md5_sum (buf, buf_len, &part->md5str, &part->md5b); // 2. calculate MD5 of multiple message blocks MD5_Update (&wdata->fop->md5, buf, buf_len); wdata->fop->l_parts = g_list_append (wdata->fop->l_parts, part); path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s", wdata->fop->fname, wdata->fop->part_number, wdata->fop->uploadid); // increase part number wdata->fop->part_number++; // XXX: check that part_number does not exceeds 10000 // add output headers http_connection_add_output_header (con, "Content-MD5", part->md5b); res = http_connection_make_request (con, path, "PUT", wdata->fop->write_buf, TRUE, NULL, fileio_write_on_send_cb, wdata ); g_free (path); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (wdata->ino), con); http_connection_release (con); wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0); g_free (wdata); return; } }
// Directory read callback function static void http_connection_on_container_meta_cb (HttpConnection *con, void *ctx, G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers, gboolean success) { ContainerMeta *meta = (ContainerMeta *) ctx; http_connection_release (con); if (!success) LOG_err (CON_CONT, "Failed to get container info !"); meta->container_meta_cb (meta->ctx, success); g_free (meta); }
// got HttpConnection object static void fileio_release_on_update_headers_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileIO *fop = (FileIO *) ctx; gchar *path; gchar *cpy_path; gboolean res; unsigned char digest[16]; gchar *md5str; size_t i; LOG_debug (FIO_LOG, INO_CON_H"Updating object's headers..", INO_T (fop->ino), con); http_connection_acquire (con); if (fop->content_type) http_connection_add_output_header (con, "Content-Type", fop->content_type); http_connection_add_output_header (con, "x-amz-metadata-directive", "REPLACE"); http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type")); MD5_Final (digest, &fop->md5); md5str = g_malloc (33); for (i = 0; i < 16; ++i) sprintf(&md5str[i*2], "%02x", (unsigned int)digest[i]); http_connection_add_output_header (con, "x-amz-meta-md5", md5str); g_free (md5str); cpy_path = g_strdup_printf ("%s%s", conf_get_string (application_get_conf (fop->app), "s3.bucket_name"), fop->fname); http_connection_add_output_header (con, "x-amz-copy-source", cpy_path); g_free (cpy_path); path = g_strdup_printf ("%s", fop->fname); res = http_connection_make_request (con, path, "PUT", NULL, TRUE, NULL, fileio_release_on_update_header_cb, fop ); g_free (path); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con); http_connection_release (con); fileio_destroy (fop); return; } }
/*{{{ update headers on uploaded object */ static void fileio_release_on_update_header_cb (HttpConnection *con, void *ctx, gboolean success, G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len, G_GNUC_UNUSED struct evkeyvalq *headers) { FileIO *fop = (FileIO *) ctx; http_connection_release (con); if (!success) { LOG_err (FIO_LOG, INO_CON_H"Failed to update headers on the server !", INO_T (fop->ino), con); fileio_destroy (fop); return; } // done LOG_debug (FIO_LOG, INO_CON_H"Headers are updated !", INO_T (fop->ino), con); fileio_destroy (fop); }
// got HttpConnection object static void fileio_read_on_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileReadData *rdata = (FileReadData *) ctx; gboolean res; guint64 part_size; http_connection_acquire (con); part_size = conf_get_uint (application_get_conf (rdata->fop->app), "s3.part_size"); // small file - get the whole file at once if (rdata->fop->file_size < part_size) rdata->request_offset = 0; // calculate offset else { gchar *range_hdr; if (part_size < rdata->size) part_size = rdata->size; rdata->request_offset = rdata->off; range_hdr = g_strdup_printf ("bytes=%"G_GUINT64_FORMAT"-%"G_GUINT64_FORMAT, (gint64)rdata->request_offset, (gint64)(rdata->request_offset + part_size)); http_connection_add_output_header (con, "Range", range_hdr); g_free (range_hdr); } res = http_connection_make_request (con, rdata->fop->fname, "GET", NULL, TRUE, NULL, fileio_read_on_get_cb, rdata ); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (rdata->ino), con); http_connection_release (con); rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0); g_free (rdata); return; } }
// got HttpConnection object static void fileio_release_on_complete_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileIO *fop = (FileIO *) ctx; gchar *path; gboolean res; struct evbuffer *xml_buf; GList *l; xml_buf = evbuffer_new (); evbuffer_add_printf (xml_buf, "%s", "<CompleteMultipartUpload>"); for (l = g_list_first (fop->l_parts); l; l = g_list_next (l)) { FileIOPart *part = (FileIOPart *) l->data; evbuffer_add_printf (xml_buf, "<Part><PartNumber>%u</PartNumber><ETag>\"%s\"</ETag></Part>", part->part_number, part->md5str); } evbuffer_add_printf (xml_buf, "%s", "</CompleteMultipartUpload>"); LOG_debug (FIO_LOG, INO_CON_H"Sending Multipart Final part..", INO_T (fop->ino), con); http_connection_acquire (con); path = g_strdup_printf ("%s?uploadId=%s", fop->fname, fop->uploadid); res = http_connection_make_request (con, path, "POST", xml_buf, TRUE, NULL, fileio_release_on_complete_cb, fop ); g_free (path); evbuffer_free (xml_buf); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con); http_connection_release (con); fileio_destroy (fop); return; } }
// got HttpConnection object static void fileio_read_on_head_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileReadData *rdata = (FileReadData *) ctx; gboolean res; http_connection_acquire (con); res = http_connection_make_request (con, rdata->fop->fname, "HEAD", NULL, FALSE, NULL, fileio_read_on_head_cb, rdata ); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (rdata->ino), con); http_connection_release (con); rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0); g_free (rdata); return; } }
static void fileio_simple_upload_on_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileIOSimpleUpload *fsim = (FileIOSimpleUpload *) ctx; time_t t; gchar str[10]; char time_str[50]; gboolean res; LOG_debug (FIO_LOG, CON_H"Uploading data. Size: %zu", con, evbuffer_get_length (fsim->write_buf)); http_connection_acquire (con); snprintf (str, sizeof (str), "%d", fsim->mode); http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type")); http_connection_add_output_header (con, "x-amz-meta-mode", str); t = time (NULL); if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) { http_connection_add_output_header (con, "x-amz-meta-date", time_str); } res = http_connection_make_request (con, fsim->fname, "PUT", fsim->write_buf, TRUE, NULL, fileio_simple_upload_on_sent_cb, fsim ); if (!res) { LOG_err (FIO_LOG, CON_H"Failed to create HTTP request !", con); http_connection_release (con); fsim->on_upload_cb (fsim->ctx, FALSE); fileio_simple_upload_destroy (fsim); return; } }
static void fileio_read_on_head_cb (HttpConnection *con, void *ctx, gboolean success, G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len, struct evkeyvalq *headers) { FileReadData *rdata = (FileReadData *) ctx; const char *content_len_header; DirTree *dtree; // release HttpConnection http_connection_release (con); if (!success) { LOG_err (FIO_LOG, INO_CON_H"Failed to get HEAD from server !", INO_T (rdata->ino), con); rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0); g_free (rdata); return; } rdata->fop->head_req_sent = TRUE; // update DirTree dtree = application_get_dir_tree (rdata->fop->app); dir_tree_set_entry_exist (dtree, rdata->ino); // consistency checking: // 1. check local and remote file sizes content_len_header = http_find_header (headers, "Content-Length"); if (content_len_header) { guint64 local_size = 0; gint64 size = 0; size = strtoll ((char *)content_len_header, NULL, 10); if (size < 0) { LOG_err (FIO_LOG, INO_CON_H"Header contains incorrect file size!", INO_T (rdata->ino), con); size = 0; } rdata->fop->file_size = size; LOG_debug (FIO_LOG, INO_H"Remote file size: %"G_GUINT64_FORMAT, INO_T (rdata->ino), rdata->fop->file_size); local_size = cache_mng_get_file_length (application_get_cache_mng (rdata->fop->app), rdata->ino); if (local_size != rdata->fop->file_size) { LOG_debug (FIO_LOG, INO_H"Local and remote file sizes do not match, invalidating local cached file!", INO_T (rdata->ino)); cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino); } } // 2. use one of the following ways to check that local and remote files are identical // if versioning is enabled: compare version IDs // if bucket has versioning disabled: compare MD5 sums if (conf_get_boolean (application_get_conf (rdata->fop->app), "s3.versioning")) { const char *versioning_header = http_find_header (headers, "x-amz-version-id"); if (versioning_header) { const gchar *local_version_id = cache_mng_get_version_id (application_get_cache_mng (rdata->fop->app), rdata->ino); if (local_version_id && !strcmp (local_version_id, versioning_header)) { LOG_debug (FIO_LOG, INO_H"Both version IDs match, using local cached file!", INO_T (rdata->ino)); } else { LOG_debug (FIO_LOG, INO_H"Version IDs do not match, invalidating local cached file!: %s %s", INO_T (rdata->ino), local_version_id, versioning_header); cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino); } // header was not found } else { LOG_debug (FIO_LOG, INO_H"Versioning header was not found, invalidating local cached file!", INO_T (rdata->ino)); cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino); } //check for MD5 } else { const char *md5_header = http_find_header (headers, "x-amz-meta-md5"); if (md5_header) { gchar *md5str = NULL; // at this point we have both remote and local MD5 sums if (cache_mng_get_md5 (application_get_cache_mng (rdata->fop->app), rdata->ino, &md5str)) { if (!strncmp (md5_header, md5str, 32)) { LOG_debug (FIO_LOG, INO_H"MD5 sums match, using local cached file!", INO_T (rdata->ino)); } else { LOG_debug (FIO_LOG, INO_H"MD5 sums do not match, invalidating local cached file!", INO_T (rdata->ino)); cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino); } } else { LOG_debug (FIO_LOG, INO_H"Failed to get local MD5 sum, invalidating local cached file!", INO_T (rdata->ino)); cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino); } if (md5str) g_free (md5str); // header was not found } else { LOG_debug (FIO_LOG, INO_H"MD5 sum header was not found, invalidating local cached file!", INO_T (rdata->ino)); cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino); } } // resume downloading file fileio_read_get_buf (rdata); }
// got HttpConnection object static void fileio_release_on_part_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileIO *fop = (FileIO *) ctx; gchar *path; gboolean res; FileIOPart *part; size_t buf_len; const gchar *buf; LOG_debug (FIO_LOG, INO_CON_H"Releasing fop. Size: %zu", INO_T (fop->ino), con, evbuffer_get_length (fop->write_buf)); // add part information to the list part = g_new0 (FileIOPart, 1); part->part_number = fop->part_number; buf_len = evbuffer_get_length (fop->write_buf); buf = (const gchar *)evbuffer_pullup (fop->write_buf, buf_len); // XXX: move to separate thread // 1. calculate MD5 of a part. get_md5_sum (buf, buf_len, &part->md5str, &part->md5b); // 2. calculate MD5 of multiple message blocks MD5_Update (&fop->md5, buf, buf_len); fop->l_parts = g_list_append (fop->l_parts, part); // if this is a multipart if (fop->multipart_initiated) { if (!fop->uploadid) { LOG_err (FIO_LOG, INO_CON_H"UploadID is not set, aborting operation !", INO_T (fop->ino), con); fileio_destroy (fop); return; } path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s", fop->fname, fop->part_number, fop->uploadid); fop->part_number++; } else { path = g_strdup (fop->fname); } #ifdef MAGIC_ENABLED // guess MIME type gchar *mime_type = magic_buffer (application_get_magic_ctx (fop->app), buf, buf_len); if (mime_type) { LOG_debug (FIO_LOG, "Guessed MIME type of %s as %s", path, mime_type); fop->content_type = g_strdup (mime_type); } else { LOG_err (FIO_LOG, "Failed to guess MIME type of %s !", path); } #endif http_connection_acquire (con); // add output headers http_connection_add_output_header (con, "Content-MD5", part->md5b); if (fop->content_type) http_connection_add_output_header (con, "Content-Type", fop->content_type); // if this is the full file if (!fop->multipart_initiated) { time_t t; gchar time_str[50]; // Add current time t = time (NULL); if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) { http_connection_add_output_header (con, "x-amz-meta-date", time_str); } http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type")); } res = http_connection_make_request (con, path, "PUT", fop->write_buf, TRUE, NULL, fileio_release_on_part_sent_cb, fop ); g_free (path); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con); http_connection_release (con); fileio_destroy (fop); return; } }