Ejemplo n.º 1
0
// store file buffer into local storage
// if success == TRUE then "buf" successfuly stored on disc
void cache_mng_store_file_buf (CacheMng *cmng, fuse_ino_t ino, size_t size, off_t off, unsigned char *buf,
    cache_mng_on_store_file_buf_cb on_store_file_buf_cb, void *ctx)
{
    struct _CacheContext *context;
    struct _CacheEntry *entry;
    ssize_t res;
    int fd;
    char path[PATH_MAX];
    guint64 old_length, new_length;
    guint64 range_size;
    time_t now;

    range_size = (guint64)(off + size);

    // limit the number of cache checks
    now = time (NULL);
    if (cmng->check_time < now && now - cmng->check_time >= 10) {
        // remove data until we have at least size bytes of max_size left
        while (cmng->max_size < cmng->size + size && g_queue_peek_tail (cmng->q_lru)) {
            entry = (struct _CacheEntry *) g_queue_peek_tail (cmng->q_lru);

            cache_mng_remove_file (cmng, entry->ino);
        }
        cmng->check_time = now;
    }

    context = cache_context_create (size, ctx);
    context->cb.store_cb = on_store_file_buf_cb;

    cache_mng_file_name (cmng, path, sizeof (path), ino);
    fd = open (path, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
    if (fd < 0) {
        LOG_err (CMNG_LOG, INO_H"Failed to create / open file for writing! Path: %s", INO_T (ino), path);
        if (context->cb.store_cb)
            context->cb.store_cb (FALSE, context->user_ctx);
        cache_context_destroy (context);
        return;
    }
    res = pwrite(fd, buf, size, off);
    close (fd);

    entry = g_hash_table_lookup (cmng->h_entries, GUINT_TO_POINTER (ino));

    if (!entry) {
        entry = cache_entry_create (ino);
        g_queue_push_head (cmng->q_lru, entry);
        entry->ll_lru = g_queue_peek_head_link (cmng->q_lru);
        g_hash_table_insert (cmng->h_entries, GUINT_TO_POINTER (ino), entry);
    }

    old_length = range_length (entry->avail_range);
    range_add (entry->avail_range, off, range_size);
    new_length = range_length (entry->avail_range);
    if (new_length >= old_length)
        cmng->size += new_length - old_length;
    else {
        LOG_err (CMNG_LOG, INO_H"New length is less than the old length !: %"G_GUINT64_FORMAT" <= %"G_GUINT64_FORMAT, 
            INO_T (ino), new_length, old_length);
    }
    
    // update modification time
    entry->modification_time = time (NULL);
   
    context->success = (res == (ssize_t) size);

    LOG_debug (CMNG_LOG, INO_H"Written [%"OFF_FMT":%zu] bytes, result: %s", 
        INO_T (ino), off, size, context->success ? "OK" : "Failed");

    context->ev = event_new (application_get_evbase (cmng->app), -1,  0,
                    cache_write_cb, context);
    // fire this event at once
    event_active (context->ev, 0, 0);
    event_add (context->ev, NULL);
}
Ejemplo n.º 2
0
static void fileio_read_on_head_cb (HttpConnection *con, void *ctx, gboolean success,
    G_GNUC_UNUSED const gchar *buf, G_GNUC_UNUSED size_t buf_len,
    struct evkeyvalq *headers)
{
    FileReadData *rdata = (FileReadData *) ctx;
    const char *content_len_header;
    DirTree *dtree;

    // release HttpConnection
    http_connection_release (con);

    if (!success) {
        LOG_err (FIO_LOG, INO_CON_H"Failed to get HEAD from server !", INO_T (rdata->ino), con);
        rdata->on_buffer_read_cb (rdata->ctx, FALSE, NULL, 0);
        g_free (rdata);
        return;
    }

    rdata->fop->head_req_sent = TRUE;
    // update DirTree
    dtree = application_get_dir_tree (rdata->fop->app);
    dir_tree_set_entry_exist (dtree, rdata->ino);

    // consistency checking:

    // 1. check local and remote file sizes
    content_len_header = http_find_header (headers, "Content-Length");
    if (content_len_header) {
        guint64 local_size = 0;
        gint64 size = 0;

        size = strtoll ((char *)content_len_header, NULL, 10);
        if (size < 0) {
            LOG_err (FIO_LOG, INO_CON_H"Header contains incorrect file size!", INO_T (rdata->ino), con);
            size = 0;
        }

        rdata->fop->file_size = size;
        LOG_debug (FIO_LOG, INO_H"Remote file size: %"G_GUINT64_FORMAT, INO_T (rdata->ino), rdata->fop->file_size);

        local_size = cache_mng_get_file_length (application_get_cache_mng (rdata->fop->app), rdata->ino);
        if (local_size != rdata->fop->file_size) {
            LOG_debug (FIO_LOG, INO_H"Local and remote file sizes do not match, invalidating local cached file!",
                INO_T (rdata->ino));
            cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
        }
    }

    // 2. use one of the following ways to check that local and remote files are identical
    // if versioning is enabled: compare version IDs
    // if bucket has versioning disabled: compare MD5 sums
    if (conf_get_boolean (application_get_conf (rdata->fop->app), "s3.versioning")) {
        const char *versioning_header = http_find_header (headers, "x-amz-version-id");
        if (versioning_header) {
            const gchar *local_version_id = cache_mng_get_version_id (application_get_cache_mng (rdata->fop->app), rdata->ino);
            if (local_version_id && !strcmp (local_version_id, versioning_header)) {
                LOG_debug (FIO_LOG, INO_H"Both version IDs match, using local cached file!", INO_T (rdata->ino));
            } else {
                LOG_debug (FIO_LOG, INO_H"Version IDs do not match, invalidating local cached file!: %s %s",
                    INO_T (rdata->ino), local_version_id, versioning_header);
                cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
            }

        // header was not found
        } else {
            LOG_debug (FIO_LOG, INO_H"Versioning header was not found, invalidating local cached file!", INO_T (rdata->ino));
            cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
        }

    //check for MD5
    } else  {
        const char *md5_header = http_find_header (headers, "x-amz-meta-md5");
        if (md5_header) {
            gchar *md5str = NULL;

            // at this point we have both remote and local MD5 sums
            if (cache_mng_get_md5 (application_get_cache_mng (rdata->fop->app), rdata->ino, &md5str)) {
                if (!strncmp (md5_header, md5str, 32)) {
                    LOG_debug (FIO_LOG, INO_H"MD5 sums match, using local cached file!", INO_T (rdata->ino));
                } else {
                    LOG_debug (FIO_LOG, INO_H"MD5 sums do not match, invalidating local cached file!", INO_T (rdata->ino));
                    cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
                }
            } else {
                LOG_debug (FIO_LOG, INO_H"Failed to get local MD5 sum, invalidating local cached file!", INO_T (rdata->ino));
                cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
            }

            if (md5str)
                g_free (md5str);

        // header was not found
        } else {
            LOG_debug (FIO_LOG, INO_H"MD5 sum header was not found, invalidating local cached file!", INO_T (rdata->ino));
            cache_mng_remove_file (application_get_cache_mng (rdata->fop->app), rdata->ino);
        }
    }

    // resume downloading file
    fileio_read_get_buf (rdata);
}