// got HttpConnection object static void fileio_write_on_send_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileWriteData *wdata = (FileWriteData *) ctx; gchar *path; gboolean res; FileIOPart *part; size_t buf_len; const gchar *buf; http_connection_acquire (con); // add part information to the list part = g_new0 (FileIOPart, 1); part->part_number = wdata->fop->part_number; buf_len = evbuffer_get_length (wdata->fop->write_buf); buf = (const gchar *) evbuffer_pullup (wdata->fop->write_buf, buf_len); // XXX: move to separate thread // 1. calculate MD5 of a part. get_md5_sum (buf, buf_len, &part->md5str, &part->md5b); // 2. calculate MD5 of multiple message blocks MD5_Update (&wdata->fop->md5, buf, buf_len); wdata->fop->l_parts = g_list_append (wdata->fop->l_parts, part); path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s", wdata->fop->fname, wdata->fop->part_number, wdata->fop->uploadid); // increase part number wdata->fop->part_number++; // XXX: check that part_number does not exceeds 10000 // add output headers http_connection_add_output_header (con, "Content-MD5", part->md5b); res = http_connection_make_request (con, path, "PUT", wdata->fop->write_buf, TRUE, NULL, fileio_write_on_send_cb, wdata ); g_free (path); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (wdata->ino), con); http_connection_release (con); wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0); g_free (wdata); return; } }
// create max_files and fill with random data // return list of {file name, content md5} static GList *populate_file_list (gint max_files, GList *l_files, gchar *in_dir) { gint i; gchar *out_dir; GError *error = NULL; FileData *fdata; gchar *name; FILE *f; out_dir = g_dir_make_tmp (NULL, &error); g_assert (out_dir); LOG_debug (POOL_TEST, "In dir: %s Out dir: %s", in_dir, out_dir); for (i = 0; i < max_files; i++) { char *bytes; size_t bytes_len; fdata = g_new0 (FileData, 1); fdata->checked = FALSE; bytes_len = g_random_int_range (100000, 1000000); bytes = g_malloc (bytes_len + 1); RAND_pseudo_bytes ((unsigned char *)bytes, bytes_len); *(bytes + bytes_len) = '\0'; name = get_random_string (15, TRUE); fdata->in_name = g_strdup_printf ("%s/%s", in_dir, name); f = fopen (fdata->in_name, "w"); fwrite (bytes, 1, bytes_len + 1, f); fclose (f); fdata->out_name = g_strdup_printf ("%s/%s", out_dir, name); get_md5_sum (bytes, bytes_len + 1, &fdata->md5, NULL); fdata->fout = fopen (fdata->out_name, "w"); g_assert (fdata->fout); fdata->url = g_strdup_printf ("http://127.0.0.1:8011/%s", name); g_assert (fdata->url); LOG_debug (POOL_TEST, "%s -> %s, size: %u", fdata->in_name, fdata->md5, bytes_len); l_files = g_list_append (l_files, fdata); } return l_files; }
// got HttpConnection object static void fileio_release_on_part_con_cb (gpointer client, gpointer ctx) { HttpConnection *con = (HttpConnection *) client; FileIO *fop = (FileIO *) ctx; gchar *path; gboolean res; FileIOPart *part; size_t buf_len; const gchar *buf; LOG_debug (FIO_LOG, INO_CON_H"Releasing fop. Size: %zu", INO_T (fop->ino), con, evbuffer_get_length (fop->write_buf)); // add part information to the list part = g_new0 (FileIOPart, 1); part->part_number = fop->part_number; buf_len = evbuffer_get_length (fop->write_buf); buf = (const gchar *)evbuffer_pullup (fop->write_buf, buf_len); // XXX: move to separate thread // 1. calculate MD5 of a part. get_md5_sum (buf, buf_len, &part->md5str, &part->md5b); // 2. calculate MD5 of multiple message blocks MD5_Update (&fop->md5, buf, buf_len); fop->l_parts = g_list_append (fop->l_parts, part); // if this is a multipart if (fop->multipart_initiated) { if (!fop->uploadid) { LOG_err (FIO_LOG, INO_CON_H"UploadID is not set, aborting operation !", INO_T (fop->ino), con); fileio_destroy (fop); return; } path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s", fop->fname, fop->part_number, fop->uploadid); fop->part_number++; } else { path = g_strdup (fop->fname); } #ifdef MAGIC_ENABLED // guess MIME type gchar *mime_type = magic_buffer (application_get_magic_ctx (fop->app), buf, buf_len); if (mime_type) { LOG_debug (FIO_LOG, "Guessed MIME type of %s as %s", path, mime_type); fop->content_type = g_strdup (mime_type); } else { LOG_err (FIO_LOG, "Failed to guess MIME type of %s !", path); } #endif http_connection_acquire (con); // add output headers http_connection_add_output_header (con, "Content-MD5", part->md5b); if (fop->content_type) http_connection_add_output_header (con, "Content-Type", fop->content_type); // if this is the full file if (!fop->multipart_initiated) { time_t t; gchar time_str[50]; // Add current time t = time (NULL); if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) { http_connection_add_output_header (con, "x-amz-meta-date", time_str); } http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type")); } res = http_connection_make_request (con, path, "PUT", fop->write_buf, TRUE, NULL, fileio_release_on_part_sent_cb, fop ); g_free (path); if (!res) { LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con); http_connection_release (con); fileio_destroy (fop); return; } }