// got HttpConnection object
static void fileio_write_on_send_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileWriteData *wdata = (FileWriteData *) ctx;
gchar *path;
gboolean res;
FileIOPart *part;
size_t buf_len;
const gchar *buf;
http_connection_acquire (con);
// add part information to the list
part = g_new0 (FileIOPart, 1);
part->part_number = wdata->fop->part_number;
buf_len = evbuffer_get_length (wdata->fop->write_buf);
buf = (const gchar *) evbuffer_pullup (wdata->fop->write_buf, buf_len);
// XXX: move to separate thread
// 1. calculate MD5 of a part.
get_md5_sum (buf, buf_len, &part->md5str, &part->md5b);
// 2. calculate MD5 of multiple message blocks
MD5_Update (&wdata->fop->md5, buf, buf_len);
wdata->fop->l_parts = g_list_append (wdata->fop->l_parts, part);
path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s",
wdata->fop->fname, wdata->fop->part_number, wdata->fop->uploadid);
// increase part number
wdata->fop->part_number++;
// XXX: check that part_number does not exceeds 10000
// add output headers
http_connection_add_output_header (con, "Content-MD5", part->md5b);
res = http_connection_make_request (con,
path, "PUT", wdata->fop->write_buf, TRUE, NULL,
fileio_write_on_send_cb,
wdata
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (wdata->ino), con);
http_connection_release (con);
wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0);
g_free (wdata);
return;
}
}
// create max_files and fill with random data
// return list of {file name, content md5}
static GList *populate_file_list (gint max_files, GList *l_files, gchar *in_dir)
{
gint i;
gchar *out_dir;
GError *error = NULL;
FileData *fdata;
gchar *name;
FILE *f;
out_dir = g_dir_make_tmp (NULL, &error);
g_assert (out_dir);
LOG_debug (POOL_TEST, "In dir: %s Out dir: %s", in_dir, out_dir);
for (i = 0; i < max_files; i++) {
char *bytes;
size_t bytes_len;
fdata = g_new0 (FileData, 1);
fdata->checked = FALSE;
bytes_len = g_random_int_range (100000, 1000000);
bytes = g_malloc (bytes_len + 1);
RAND_pseudo_bytes ((unsigned char *)bytes, bytes_len);
*(bytes + bytes_len) = '\0';
name = get_random_string (15, TRUE);
fdata->in_name = g_strdup_printf ("%s/%s", in_dir, name);
f = fopen (fdata->in_name, "w");
fwrite (bytes, 1, bytes_len + 1, f);
fclose (f);
fdata->out_name = g_strdup_printf ("%s/%s", out_dir, name);
get_md5_sum (bytes, bytes_len + 1, &fdata->md5, NULL);
fdata->fout = fopen (fdata->out_name, "w");
g_assert (fdata->fout);
fdata->url = g_strdup_printf ("http://127.0.0.1:8011/%s", name);
g_assert (fdata->url);
LOG_debug (POOL_TEST, "%s -> %s, size: %u", fdata->in_name, fdata->md5, bytes_len);
l_files = g_list_append (l_files, fdata);
}
return l_files;
}
// got HttpConnection object
static void fileio_release_on_part_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIO *fop = (FileIO *) ctx;
gchar *path;
gboolean res;
FileIOPart *part;
size_t buf_len;
const gchar *buf;
LOG_debug (FIO_LOG, INO_CON_H"Releasing fop. Size: %zu", INO_T (fop->ino), con, evbuffer_get_length (fop->write_buf));
// add part information to the list
part = g_new0 (FileIOPart, 1);
part->part_number = fop->part_number;
buf_len = evbuffer_get_length (fop->write_buf);
buf = (const gchar *)evbuffer_pullup (fop->write_buf, buf_len);
// XXX: move to separate thread
// 1. calculate MD5 of a part.
get_md5_sum (buf, buf_len, &part->md5str, &part->md5b);
// 2. calculate MD5 of multiple message blocks
MD5_Update (&fop->md5, buf, buf_len);
fop->l_parts = g_list_append (fop->l_parts, part);
// if this is a multipart
if (fop->multipart_initiated) {
if (!fop->uploadid) {
LOG_err (FIO_LOG, INO_CON_H"UploadID is not set, aborting operation !", INO_T (fop->ino), con);
fileio_destroy (fop);
return;
}
path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s",
fop->fname, fop->part_number, fop->uploadid);
fop->part_number++;
} else {
path = g_strdup (fop->fname);
}
#ifdef MAGIC_ENABLED
// guess MIME type
gchar *mime_type = magic_buffer (application_get_magic_ctx (fop->app), buf, buf_len);
if (mime_type) {
LOG_debug (FIO_LOG, "Guessed MIME type of %s as %s", path, mime_type);
fop->content_type = g_strdup (mime_type);
} else {
LOG_err (FIO_LOG, "Failed to guess MIME type of %s !", path);
}
#endif
http_connection_acquire (con);
// add output headers
http_connection_add_output_header (con, "Content-MD5", part->md5b);
if (fop->content_type)
http_connection_add_output_header (con, "Content-Type", fop->content_type);
// if this is the full file
if (!fop->multipart_initiated) {
time_t t;
gchar time_str[50];
// Add current time
t = time (NULL);
if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) {
http_connection_add_output_header (con, "x-amz-meta-date", time_str);
}
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
}
res = http_connection_make_request (con,
path, "PUT", fop->write_buf, TRUE, NULL,
fileio_release_on_part_sent_cb,
fop
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con);
http_connection_release (con);
fileio_destroy (fop);
return;
}
}
