/*{{{ create / destroy */
CacheMng *cache_mng_create (Application *app)
{
CacheMng *cmng;
cmng = g_new0 (CacheMng, 1);
cmng->app = app;
cmng->h_entries = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, cache_entry_destroy);
cmng->q_lru = g_queue_new ();
cmng->size = 0;
cmng->check_time = time (NULL);
cmng->max_size = conf_get_uint (application_get_conf (cmng->app), "filesystem.cache_dir_max_size");
cmng->cache_dir = g_strdup_printf ("%s/%s",
conf_get_string (application_get_conf (cmng->app), "filesystem.cache_dir"), CACHE_MNGR_DIR);
cmng->cache_hits = 0;
cmng->cache_miss = 0;
cache_mng_rm_cache_dir (cmng);
if (g_mkdir_with_parents (cmng->cache_dir, 0700) != 0) {
LOG_err (CMNG_LOG, "Failed to remove directory: %s", cmng->cache_dir);
cache_mng_destroy (cmng);
return NULL;
}
return cmng;
}
void stats_srv_add_op_history (StatSrv *stat_srv, const gchar *str)
{
// stats server is disabled
if (!conf_get_boolean (application_get_conf (stat_srv->app), "statistics.enabled"))
return;
// maintain queue size
while (g_queue_get_length (stat_srv->q_op_history) + 1 >= conf_get_uint (application_get_conf (stat_srv->app), "statistics.history_size")) {
gchar *tmp = g_queue_pop_tail (stat_srv->q_op_history);
g_free (tmp);
}
g_queue_push_head (stat_srv->q_op_history, g_strdup (str));
}
// got HttpConnection object
static void fileio_write_on_multipart_init_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileWriteData *wdata = (FileWriteData *) ctx;
gboolean res;
gchar *path;
http_connection_acquire (con);
path = g_strdup_printf ("%s?uploads", wdata->fop->fname);
// send storage class with the init request
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
res = http_connection_make_request (con,
path, "POST", NULL, TRUE, NULL,
fileio_write_on_multipart_init_cb,
wdata
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (wdata->ino), con);
http_connection_release (con);
wdata->on_buffer_written_cb (wdata->fop, wdata->ctx, FALSE, 0);
g_free (wdata);
return;
}
}
static void fileio_simple_download_on_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIOSimpleDownload *fsim = (FileIOSimpleDownload *) ctx;
gboolean res;
LOG_debug (FIO_LOG, CON_H"Downloading data.", con);
http_connection_acquire (con);
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
res = http_connection_make_request (con,
fsim->fname, "GET", NULL, TRUE, NULL,
fileio_simple_download_on_sent_cb,
fsim
);
if (!res) {
LOG_err (FIO_LOG, CON_H"Failed to create HTTP request !", con);
http_connection_release (con);
fsim->on_download_cb (fsim->ctx, FALSE, NULL, 0);
fileio_simple_download_destroy (fsim);
return;
}
}
// got HttpConnection object
static void fileio_release_on_update_headers_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIO *fop = (FileIO *) ctx;
gchar *path;
gchar *cpy_path;
gboolean res;
unsigned char digest[16];
gchar *md5str;
size_t i;
LOG_debug (FIO_LOG, INO_CON_H"Updating object's headers..", INO_T (fop->ino), con);
http_connection_acquire (con);
if (fop->content_type)
http_connection_add_output_header (con, "Content-Type", fop->content_type);
http_connection_add_output_header (con, "x-amz-metadata-directive", "REPLACE");
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
MD5_Final (digest, &fop->md5);
md5str = g_malloc (33);
for (i = 0; i < 16; ++i)
sprintf(&md5str[i*2], "%02x", (unsigned int)digest[i]);
http_connection_add_output_header (con, "x-amz-meta-md5", md5str);
g_free (md5str);
cpy_path = g_strdup_printf ("%s%s", conf_get_string (application_get_conf (fop->app), "s3.bucket_name"), fop->fname);
http_connection_add_output_header (con, "x-amz-copy-source", cpy_path);
g_free (cpy_path);
path = g_strdup_printf ("%s", fop->fname);
res = http_connection_make_request (con,
path, "PUT", NULL, TRUE, NULL,
fileio_release_on_update_header_cb,
fop
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con);
http_connection_release (con);
fileio_destroy (fop);
return;
}
}
void fileio_write_buffer (FileIO *fop,
const char *buf, size_t buf_size, off_t off, fuse_ino_t ino,
FileIO_on_buffer_written_cb on_buffer_written_cb, gpointer ctx)
{
FileWriteData *wdata;
// XXX: allow only sequentially write
// current written bytes should be always match offset
if (off >= 0 && fop->current_size != (guint64)off) {
LOG_err (FIO_LOG, INO_H"Write call with offset %"OFF_FMT" is not allowed !", INO_T (ino), off);
on_buffer_written_cb (fop, ctx, FALSE, 0);
return;
}
// add data to output buffer
evbuffer_add (fop->write_buf, buf, buf_size);
fop->current_size += buf_size;
LOG_debug (FIO_LOG, INO_H"Write buf size: %zd", INO_T (ino), evbuffer_get_length (fop->write_buf));
// CacheMng
cache_mng_store_file_buf (application_get_cache_mng (fop->app),
ino, buf_size, off, (unsigned char *) buf,
NULL, NULL);
// if current write buffer exceeds "part_size" - this is a multipart upload
if (evbuffer_get_length (fop->write_buf) >= conf_get_uint (application_get_conf (fop->app), "s3.part_size")) {
// init helper struct
wdata = g_new0 (FileWriteData, 1);
wdata->fop = fop;
wdata->buf_size = buf_size;
wdata->off = off;
wdata->ino = ino;
wdata->on_buffer_written_cb = on_buffer_written_cb;
wdata->ctx = ctx;
// init multipart upload
if (!fop->multipart_initiated) {
fileio_write_init_multipart (wdata);
// else send the current part
} else {
fileio_write_send_part (wdata);
}
// or just notify client that we are ready for more data
} else {
on_buffer_written_cb (fop, ctx, TRUE, buf_size);
}
}
// create S3HttpConnection object
// establish HTTP connections to S3
gpointer s3http_connection_create (Application *app)
{
S3HttpConnection *con;
int port;
AppConf *conf;
con = g_new0 (S3HttpConnection, 1);
if (!con) {
LOG_err (CON_LOG, "Failed to create S3HttpConnection !");
return NULL;
}
conf = application_get_conf (app);
con->app = app;
con->bucket_name = g_strdup (application_get_bucket_name (app));
con->is_acquired = FALSE;
port = application_get_port (app);
// if no port is specified, libevent returns -1
if (port == -1) {
port = conf->http_port;
}
LOG_debug (CON_LOG, "Connecting to %s:%d",
application_get_host (app),
port
);
// XXX: implement SSL
con->evcon = evhttp_connection_base_new (
application_get_evbase (app),
application_get_dnsbase (app),
application_get_host (app),
port
);
if (!con->evcon) {
LOG_err (CON_LOG, "Failed to create evhttp_connection !");
return NULL;
}
evhttp_connection_set_timeout (con->evcon, conf->timeout);
evhttp_connection_set_retries (con->evcon, conf->retries);
evhttp_connection_set_closecb (con->evcon, s3http_connection_on_close, con);
return (gpointer)con;
}
static void fileio_release_update_headers (FileIO *fop)
{
// update MD5 headers only if versioning is disabled
if (conf_get_boolean (application_get_conf (fop->app), "s3.versioning")) {
LOG_debug (FIO_LOG, INO_H"File uploaded !", INO_T (fop->ino));
fileio_destroy (fop);
} else {
if (!client_pool_get_client (application_get_write_client_pool (fop->app),
fileio_release_on_update_headers_con_cb, fop)) {
LOG_err (FIO_LOG, INO_H"Failed to get HTTP client !", INO_T (fop->ino));
fileio_destroy (fop);
return;
}
}
}
// prepare HTTP request
static void dir_tree_file_read_prepare_request (DirTreeFileOpData *op_data, S3HttpClient *http, off_t off, size_t size)
{
gchar *auth_str;
char time_str[100];
time_t t = time (NULL);
gchar auth_key[300];
gchar *url;
gchar range[300];
AppConf *conf;
s3http_client_request_reset (http);
s3http_client_set_cb_ctx (http, op_data);
// s3http_client_set_on_chunk_cb (http, dir_tree_file_read_on_chunk_cb);
s3http_client_set_on_last_chunk_cb (http, dir_tree_file_read_on_last_chunk_cb);
s3http_client_set_output_length (http, 0);
strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t));
auth_str = (gchar *)s3http_connection_get_auth_string (op_data->dtree->app, "GET", "", op_data->en->fullpath, time_str);
snprintf (auth_key, sizeof (auth_key), "AWS %s:%s", application_get_access_key_id (op_data->dtree->app), auth_str);
g_free (auth_str);
snprintf (range, sizeof (range), "bytes=%"OFF_FMT"-%"OFF_FMT, off, off+size - 1);
LOG_debug (DIR_TREE_LOG, "range: %s", range);
s3http_client_add_output_header (http, "Authorization", auth_key);
s3http_client_add_output_header (http, "Date", time_str);
s3http_client_add_output_header (http, "Range", range);
s3http_client_add_output_header (http, "Host", application_get_host_header (op_data->dtree->app));
// XXX: HTTPS
conf = application_get_conf (op_data->dtree->app);
if (conf->path_style) {
url = g_strdup_printf ("http://%s:%d/%s%s", application_get_host (op_data->dtree->app),
application_get_port (op_data->dtree->app),
application_get_bucket_name (op_data->dtree->app),
op_data->en->fullpath);
} else {
url = g_strdup_printf ("http://%s%d%s", application_get_host (op_data->dtree->app),
application_get_port (op_data->dtree->app),
op_data->en->fullpath);
}
s3http_client_start_request (http, S3Method_get, url);
g_free (auth_str);
g_free (url);
}
// connect to the remote server
static void s3http_client_connect (S3HttpClient *http)
{
int port;
AppConf *conf;
if (http->connection_state == S3C_connecting)
return;
conf = application_get_conf (http->app);
if (http->bev)
bufferevent_free (http->bev);
http->bev = bufferevent_socket_new (http->evbase, -1, 0);
if (!http->bev) {
LOG_err (HTTP_LOG, "Failed to create HTTP object!");
}
// XXX:
// bufferevent_set_timeouts (http->bev,
port = evhttp_uri_get_port (http->http_uri);
// if no port is specified, libevent returns -1
if (port == -1) {
port = conf->http_port;
}
LOG_debug (HTTP_LOG, "Connecting to %s:%d .. %p",
evhttp_uri_get_host (http->http_uri),
port, http
);
http->connection_state = S3C_connecting;
bufferevent_enable (http->bev, EV_WRITE);
bufferevent_setcb (http->bev,
NULL, NULL, s3http_client_connection_event_cb,
http
);
bufferevent_socket_connect_hostname (http->bev, http->dns_base,
AF_UNSPEC,
evhttp_uri_get_host (http->http_uri),
port
);
}
// got HttpConnection object
static void fileio_read_on_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileReadData *rdata = (FileReadData *) ctx;
gboolean res;
guint64 part_size;
http_connection_acquire (con);
part_size = conf_get_uint (application_get_conf (rdata->fop->app), "s3.part_size");
// small file - get the whole file at once
if (rdata->fop->file_size < part_size)
rdata->request_offset = 0;
// calculate offset
else {
gchar *range_hdr;
if (part_size < rdata->size)
part_size = rdata->size;
rdata->request_offset = rdata->off;
range_hdr = g_strdup_printf ("bytes=%"G_GUINT64_FORMAT"-%"G_GUINT64_FORMAT,
(gint64)rdata->request_offset, (gint64)(rdata->request_offset + part_size));
http_connection_add_output_header (con, "Range", range_hdr);
g_free (range_hdr);
}
res = http_connection_make_request (con,
rdata->fop->fname, "GET", NULL, TRUE, NULL,
fileio_read_on_get_cb,
rdata
);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (rdata->ino), con);
http_connection_release (con);
rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0);
g_free (rdata);
return;
}
}
DirTree *dir_tree_create (Application *app)
{
DirTree *dtree;
AppConf *conf;
conf = application_get_conf (app);
dtree = g_new0 (DirTree, 1);
dtree->app = app;
// children entries are destroyed by parent directory entries
dtree->h_inodes = g_hash_table_new (g_direct_hash, g_direct_equal);
dtree->max_ino = FUSE_ROOT_ID;
dtree->current_age = 0;
dtree->dir_cache_max_time = conf->dir_cache_max_time; //XXX
dtree->current_write_ops = 0;
dtree->root = dir_tree_add_entry (dtree, "/", DIR_DEFAULT_MODE, DET_dir, 0, 0, time (NULL));
LOG_debug (DIR_TREE_LOG, "DirTree created");
return dtree;
}
static void fileio_simple_upload_on_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIOSimpleUpload *fsim = (FileIOSimpleUpload *) ctx;
time_t t;
gchar str[10];
char time_str[50];
gboolean res;
LOG_debug (FIO_LOG, CON_H"Uploading data. Size: %zu", con, evbuffer_get_length (fsim->write_buf));
http_connection_acquire (con);
snprintf (str, sizeof (str), "%d", fsim->mode);
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
http_connection_add_output_header (con, "x-amz-meta-mode", str);
t = time (NULL);
if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) {
http_connection_add_output_header (con, "x-amz-meta-date", time_str);
}
res = http_connection_make_request (con,
fsim->fname, "PUT", fsim->write_buf, TRUE, NULL,
fileio_simple_upload_on_sent_cb,
fsim
);
if (!res) {
LOG_err (FIO_LOG, CON_H"Failed to create HTTP request !", con);
http_connection_release (con);
fsim->on_upload_cb (fsim->ctx, FALSE);
fileio_simple_upload_destroy (fsim);
return;
}
}
// creates connection pool object
// create client_count clients
// return NULL if error
S3ClientPool *s3client_pool_create (Application *app,
gint client_count,
S3ClientPool_client_create client_create,
S3ClientPool_client_destroy client_destroy,
S3ClientPool_client_set_on_released_cb client_set_on_released_cb,
S3ClientPool_client_check_rediness client_check_rediness)
{
S3ClientPool *pool;
gint i;
PoolClient *pc;
AppConf *conf;
conf = application_get_conf (app);
pool = g_new0 (S3ClientPool, 1);
pool->app = app;
pool->evbase = application_get_evbase (app);
pool->dns_base = application_get_dnsbase (app);
pool->l_clients = NULL;
pool->q_requests = g_queue_new ();
pool->max_requests = conf->max_requests_per_pool;
for (i = 0; i < client_count; i++) {
pc = g_new0 (PoolClient, 1);
pc->pool = pool;
pc->client = client_create (app);
pc->client_check_rediness = client_check_rediness;
pc->client_destroy = client_destroy;
// add to the list
pool->l_clients = g_list_append (pool->l_clients, pc);
// add callback
client_set_on_released_cb (pc->client, s3client_pool_on_client_released, pc);
}
return pool;
}
StatSrv *stat_srv_create (Application *app)
{
StatSrv *stat_srv;
stat_srv = g_new0 (StatSrv, 1);
stat_srv->app = app;
stat_srv->q_op_history = g_queue_new ();
stat_srv->boot_time = time (NULL);
// stats server is disabled
if (!conf_get_boolean (application_get_conf (stat_srv->app), "statistics.enabled")) {
return stat_srv;
}
stat_srv->http = evhttp_new (application_get_evbase (app));
if (!stat_srv->http) {
LOG_err (STAT_LOG, "Failed to create statistics server !");
return NULL;
}
// bind
if (evhttp_bind_socket (stat_srv->http,
conf_get_string (application_get_conf (stat_srv->app), "statistics.host"),
conf_get_int (application_get_conf (stat_srv->app), "statistics.port")) == -1) {
LOG_err (STAT_LOG, "Failed to bind statistics server to %s:%d",
conf_get_string (application_get_conf (stat_srv->app), "statistics.host"),
conf_get_int (application_get_conf (stat_srv->app), "statistics.port")
);
return NULL;
}
// install handlers
evhttp_set_cb (stat_srv->http, conf_get_string (application_get_conf (stat_srv->app), "statistics.stats_path"), stat_srv_on_stats_cb, stat_srv);
evhttp_set_gencb (stat_srv->http, stat_srv_on_gen_cb, stat_srv);
return stat_srv;
}
gboolean s3http_connection_make_request (S3HttpConnection *con,
const gchar *resource_path, const gchar *request_str,
const gchar *http_cmd,
struct evbuffer *out_buffer,
S3HttpConnection_responce_cb responce_cb,
S3HttpConnection_error_cb error_cb,
gpointer ctx)
{
gchar *auth_str;
struct evhttp_request *req;
gchar auth_key[300];
time_t t;
char time_str[50];
RequestData *data;
int res;
enum evhttp_cmd_type cmd_type;
AppConf *conf;
data = g_new0 (RequestData, 1);
data->responce_cb = responce_cb;
data->error_cb = error_cb;
data->ctx = ctx;
data->con = con;
if (!strcasecmp (http_cmd, "GET")) {
cmd_type = EVHTTP_REQ_GET;
} else if (!strcasecmp (http_cmd, "PUT")) {
cmd_type = EVHTTP_REQ_PUT;
} else if (!strcasecmp (http_cmd, "DELETE")) {
cmd_type = EVHTTP_REQ_DELETE;
} else if (!strcasecmp (http_cmd, "HEAD")) {
cmd_type = EVHTTP_REQ_HEAD;
} else {
LOG_err (CON_LOG, "Unsupported HTTP method: %s", http_cmd);
return FALSE;
}
t = time (NULL);
strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t));
auth_str = s3http_connection_get_auth_string (con->app, http_cmd, "", resource_path, time_str);
snprintf (auth_key, sizeof (auth_key), "AWS %s:%s", application_get_access_key_id (con->app), auth_str);
g_free (auth_str);
req = evhttp_request_new (s3http_connection_on_responce_cb, data);
if (!req) {
LOG_err (CON_LOG, "Failed to create HTTP request object !");
return FALSE;
}
evhttp_add_header (req->output_headers, "Authorization", auth_key);
evhttp_add_header (req->output_headers, "Host", application_get_host_header (con->app));
evhttp_add_header (req->output_headers, "Date", time_str);
if (out_buffer) {
evbuffer_add_buffer (req->output_buffer, out_buffer);
}
conf = application_get_conf (con->app);
if (conf->path_style) {
request_str = g_strdup_printf("/%s%s", application_get_bucket_name (con->app), request_str);
}
LOG_debug (CON_LOG, "[%p] bucket: %s path: %s", con, application_get_bucket_name (con->app), request_str);
res = evhttp_make_request (s3http_connection_get_evcon (con), req, cmd_type, request_str);
if (res < 0)
return FALSE;
else
return TRUE;
}
// got HttpConnection object
static void fileio_release_on_part_con_cb (gpointer client, gpointer ctx)
{
HttpConnection *con = (HttpConnection *) client;
FileIO *fop = (FileIO *) ctx;
gchar *path;
gboolean res;
FileIOPart *part;
size_t buf_len;
const gchar *buf;
LOG_debug (FIO_LOG, INO_CON_H"Releasing fop. Size: %zu", INO_T (fop->ino), con, evbuffer_get_length (fop->write_buf));
// add part information to the list
part = g_new0 (FileIOPart, 1);
part->part_number = fop->part_number;
buf_len = evbuffer_get_length (fop->write_buf);
buf = (const gchar *)evbuffer_pullup (fop->write_buf, buf_len);
// XXX: move to separate thread
// 1. calculate MD5 of a part.
get_md5_sum (buf, buf_len, &part->md5str, &part->md5b);
// 2. calculate MD5 of multiple message blocks
MD5_Update (&fop->md5, buf, buf_len);
fop->l_parts = g_list_append (fop->l_parts, part);
// if this is a multipart
if (fop->multipart_initiated) {
if (!fop->uploadid) {
LOG_err (FIO_LOG, INO_CON_H"UploadID is not set, aborting operation !", INO_T (fop->ino), con);
fileio_destroy (fop);
return;
}
path = g_strdup_printf ("%s?partNumber=%u&uploadId=%s",
fop->fname, fop->part_number, fop->uploadid);
fop->part_number++;
} else {
path = g_strdup (fop->fname);
}
#ifdef MAGIC_ENABLED
// guess MIME type
gchar *mime_type = magic_buffer (application_get_magic_ctx (fop->app), buf, buf_len);
if (mime_type) {
LOG_debug (FIO_LOG, "Guessed MIME type of %s as %s", path, mime_type);
fop->content_type = g_strdup (mime_type);
} else {
LOG_err (FIO_LOG, "Failed to guess MIME type of %s !", path);
}
#endif
http_connection_acquire (con);
// add output headers
http_connection_add_output_header (con, "Content-MD5", part->md5b);
if (fop->content_type)
http_connection_add_output_header (con, "Content-Type", fop->content_type);
// if this is the full file
if (!fop->multipart_initiated) {
time_t t;
gchar time_str[50];
// Add current time
t = time (NULL);
if (strftime (time_str, sizeof (time_str), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t))) {
http_connection_add_output_header (con, "x-amz-meta-date", time_str);
}
http_connection_add_output_header (con, "x-amz-storage-class", conf_get_string (application_get_conf (con->app), "s3.storage_type"));
}
res = http_connection_make_request (con,
path, "PUT", fop->write_buf, TRUE, NULL,
fileio_release_on_part_sent_cb,
fop
);
g_free (path);
if (!res) {
LOG_err (FIO_LOG, INO_CON_H"Failed to create HTTP request !", INO_T (fop->ino), con);
http_connection_release (con);
fileio_destroy (fop);
return;
}
}
static void stat_srv_on_stats_cb (struct evhttp_request *req, void *ctx)
{
StatSrv *stat_srv = (StatSrv *) ctx;
struct evbuffer *evb = NULL;
gint ref = 0;
GString *str;
struct evhttp_uri *uri;
guint32 total_inodes, file_num, dir_num;
guint64 read_ops, write_ops, readdir_ops, lookup_ops;
guint32 cache_entries;
guint64 total_cache_size, cache_hits, cache_miss;
struct tm *cur_p;
struct tm cur;
time_t now;
char ts[50];
uri = evhttp_uri_parse (evhttp_request_get_uri (req));
LOG_debug (STAT_LOG, "Incoming request: %s from %s:%d",
evhttp_request_get_uri (req), req->remote_host, req->remote_port);
if (uri) {
const gchar *query;
query = evhttp_uri_get_query (uri);
if (query) {
const gchar *refresh = NULL;
struct evkeyvalq q_params;
TAILQ_INIT (&q_params);
evhttp_parse_query_str (query, &q_params);
refresh = http_find_header (&q_params, "refresh");
if (refresh)
ref = atoi (refresh);
evhttp_clear_headers (&q_params);
}
evhttp_uri_free (uri);
}
str = g_string_new (NULL);
now = time (NULL);
localtime_r (&now, &cur);
cur_p = &cur;
if (!strftime (ts, sizeof (ts), "%H:%M:%S", cur_p))
ts[0] = '\0';
g_string_append_printf (str, "RioFS version: %s Uptime: %u sec, Now: %s, Log level: %d, Dir cache time: %u sec<BR>",
VERSION, (guint32)(now - stat_srv->boot_time), ts, log_level,
conf_get_uint (application_get_conf (stat_srv->app), "filesystem.dir_cache_max_time"));
// DirTree
dir_tree_get_stats (application_get_dir_tree (stat_srv->app), &total_inodes, &file_num, &dir_num);
g_string_append_printf (str, "<BR>DirTree: <BR>-Total inodes: %u, Total files: %u, Total directories: %u<BR>",
total_inodes, file_num, dir_num);
// Fuse
rfuse_get_stats (application_get_rfuse (stat_srv->app), &read_ops, &write_ops, &readdir_ops, &lookup_ops);
g_string_append_printf (str, "<BR>Fuse: <BR>-Read ops: %"G_GUINT64_FORMAT", Write ops: %"G_GUINT64_FORMAT
", Readdir ops: %"G_GUINT64_FORMAT", Lookup ops: %"G_GUINT64_FORMAT"<BR>",
read_ops, write_ops, readdir_ops, lookup_ops);
// CacheMng
cache_mng_get_stats (application_get_cache_mng (stat_srv->app), &cache_entries, &total_cache_size, &cache_hits, &cache_miss);
g_string_append_printf (str, "<BR>CacheMng: <BR>-Total entries: %"G_GUINT32_FORMAT", Total cache size: %"G_GUINT64_FORMAT
" bytes, Cache hits: %"G_GUINT64_FORMAT", Cache misses: %"G_GUINT64_FORMAT" <BR>",
cache_entries, total_cache_size, cache_hits, cache_miss);
g_string_append_printf (str, "<BR>Read workers (%d): <BR>",
client_pool_get_client_count (application_get_read_client_pool (stat_srv->app)));
client_pool_get_client_stats_info (application_get_read_client_pool (stat_srv->app), str, &print_format_http);
g_string_append_printf (str, "<BR>Write workers (%d): <BR>",
client_pool_get_client_count (application_get_write_client_pool (stat_srv->app)));
client_pool_get_client_stats_info (application_get_write_client_pool (stat_srv->app), str, &print_format_http);
g_string_append_printf (str, "<BR>Op workers (%d): <BR>",
client_pool_get_client_count (application_get_ops_client_pool (stat_srv->app)));
client_pool_get_client_stats_info (application_get_ops_client_pool (stat_srv->app), str, &print_format_http);
g_string_append_printf (str, "<BR><BR>Operation history (%u max items): <BR>",
conf_get_uint (application_get_conf (stat_srv->app), "statistics.history_size"));
g_queue_foreach (stat_srv->q_op_history, (GFunc) stat_srv_print_history_item, str);
evb = evbuffer_new ();
evbuffer_add_printf (evb, "<HTTP>");
if (ref) {
evbuffer_add_printf (evb, "<HEAD><meta http-equiv=\"refresh\" content=\"%d\"></HEAD>", ref);
}
evbuffer_add_printf (evb, "<BODY>");
evbuffer_add (evb, str->str, str->len);
evbuffer_add_printf (evb, "</BODY></HTTP>");
evhttp_send_reply (req, HTTP_OK, "OK", evb);
evbuffer_free (evb);
g_string_free (str, TRUE);
}
static void fileio_read_on_head_cb (HttpConnection *con, void *ctx, gboolean success,
G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len,
struct evkeyvalq *headers)
{
FileReadData *rdata = (FileReadData *) ctx;
const char *content_len_header;
DirTree *dtree;
// release HttpConnection
http_connection_release (con);
if (!success) {
LOG_err (FIO_LOG, INO_CON_H"Failed to get HEAD from server !", INO_T (rdata->ino), con);
rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0);
g_free (rdata);
return;
}
rdata->fop->head_req_sent = TRUE;
// update DirTree
dtree = application_get_dir_tree (rdata->fop->app);
dir_tree_set_entry_exist (dtree, rdata->ino);
// consistency checking:
// 1. check local and remote file sizes
content_len_header = http_find_header (headers, "Content-Length");
if (content_len_header) {
guint64 local_size = 0;
gint64 size = 0;
size = strtoll ((char *)content_len_header, NULL, 10);
if (size < 0) {
LOG_err (FIO_LOG, INO_CON_H"Header contains incorrect file size!", INO_T (rdata->ino), con);
size = 0;
}
rdata->fop->file_size = size;
LOG_debug (FIO_LOG, INO_H"Remote file size: %"G_GUINT64_FORMAT, INO_T (rdata->ino), rdata->fop->file_size);
local_size = cache_mng_get_file_length (application_get_cache_mng (rdata->fop->app), rdata->ino);
if (local_size != rdata->fop->file_size) {
LOG_debug (FIO_LOG, INO_H"Local and remote file sizes do not match, invalidating local cached file!",
INO_T (rdata->ino));
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}
}
// 2. use one of the following ways to check that local and remote files are identical
// if versioning is enabled: compare version IDs
// if bucket has versioning disabled: compare MD5 sums
if (conf_get_boolean (application_get_conf (rdata->fop->app), "s3.versioning")) {
const char *versioning_header = http_find_header (headers, "x-amz-version-id");
if (versioning_header) {
const gchar *local_version_id = cache_mng_get_version_id (application_get_cache_mng (rdata->fop->app), rdata->ino);
if (local_version_id && !strcmp (local_version_id, versioning_header)) {
LOG_debug (FIO_LOG, INO_H"Both version IDs match, using local cached file!", INO_T (rdata->ino));
} else {
LOG_debug (FIO_LOG, INO_H"Version IDs do not match, invalidating local cached file!: %s %s",
INO_T (rdata->ino), local_version_id, versioning_header);
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}
// header was not found
} else {
LOG_debug (FIO_LOG, INO_H"Versioning header was not found, invalidating local cached file!", INO_T (rdata->ino));
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}
//check for MD5
} else {
const char *md5_header = http_find_header (headers, "x-amz-meta-md5");
if (md5_header) {
gchar *md5str = NULL;
// at this point we have both remote and local MD5 sums
if (cache_mng_get_md5 (application_get_cache_mng (rdata->fop->app), rdata->ino, &md5str)) {
if (!strncmp (md5_header, md5str, 32)) {
LOG_debug (FIO_LOG, INO_H"MD5 sums match, using local cached file!", INO_T (rdata->ino));
} else {
LOG_debug (FIO_LOG, INO_H"MD5 sums do not match, invalidating local cached file!", INO_T (rdata->ino));
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}
} else {
LOG_debug (FIO_LOG, INO_H"Failed to get local MD5 sum, invalidating local cached file!", INO_T (rdata->ino));
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}
if (md5str)
g_free (md5str);
// header was not found
} else {
LOG_debug (FIO_LOG, INO_H"MD5 sum header was not found, invalidating local cached file!", INO_T (rdata->ino));
cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
}
}
// resume downloading file
fileio_read_get_buf (rdata);
}
