Пример #1
0
void file_update_block(const char *blockdata, unsigned long long blocknr,
                       unsigned int offsetblock, unsigned long long size, 
                       unsigned long long inode, unsigned char *chksum, 
                       off_t offset)
{
    DBT *data = NULL;
    DBT *decrypted = NULL;
    DBT *cachedata;
    unsigned char *dbdata;
    INOBNO inobno;
    compr *uncompdata;
    BLKCACHE *blk;
    unsigned char *dtiger;
    INUSE *inuse;
    bool compressed = 1;
    QDTA *dta;
#ifndef SHA3
    word64 res[3];
#endif

    FUNC;
    LDEBUG
        ("file_update_block : inode %llu blocknr %llu offsetblock %llu, size %llu",
         inode, blocknr, (unsigned long long) offsetblock,
         (unsigned long long) size);
    inobno.inode = inode;
    inobno.blocknr = blocknr;

    dbdata = (unsigned char *) s_malloc(BLKSIZE);
    memset(dbdata, 0, BLKSIZE);
    data = try_block_cache(inode, blocknr, 0);
    if (NULL != data) {
        LDEBUG("try_block_cache : HIT");
        memcpy(dbdata, data->data, data->size);
        memcpy(dbdata + offsetblock, blockdata, size);
        add_blk_to_cache(inode, blocknr, offsetblock + size, dbdata, offset);
        update_filesize(inode, size, offsetblock, blocknr, 0, 0, 0);
        free(dbdata);
        DBTfree(data);
        return;
    } else
        LDEBUG("%s: block not found in cache.", __FUNCTION__);

// We don't need the old blockdata when we overwrite it completely anyway.
    if (offsetblock > 0 || size < BLKSIZE) {
        get_moddb_lock();
// First read the cache
        decrypted = search_memhash(dbdtaq, chksum, config->hashlen);
        if (NULL == decrypted) {
            LDEBUG("%s: Not in dbdtaq", __FUNCTION__);
            data = file_tgr_read_data(chksum);
            if (NULL == data) {
                LDEBUG("%s: Not found", __FUNCTION__);
                cachedata =
                    search_memhash(blkcache, &inobno.inode,
                                   sizeof(unsigned long long));
                if (NULL != cachedata) {
                    blk = (BLKCACHE *) cachedata->data;
#ifdef SHA3
                    dtiger=sha_binhash(blk->blockdata, BLKSIZE);
#else
                    binhash(blk->blockdata, BLKSIZE, res);
                    dtiger=(unsigned char *)&res;
#endif
                    if (0 == memcmp(dtiger, chksum, config->hashlen)) {
                        LDEBUG("%s: data alloc here", __FUNCTION__);
                        data = s_malloc(sizeof(DBT));
                        data->data = s_malloc(BLKSIZE);
                        data->size = BLKSIZE;
                        memcpy(data->data, blk->blockdata, BLKSIZE);
                        DBTfree(cachedata);
                        compressed = 0;
                    } else {
                        LDEBUG
                            ("updateBlock : Not in dbcache, out of luck.");
                        loghash("updateBlock : No data found to read ",
                                chksum);
                        die_dataerr
                            ("file_update_block : No data found to read - this should never happen: inode :%llu: blocknr :%llu",
                             inode, blocknr);
                    }
#ifdef SHA3
                    free(dtiger);
#endif
                } else {
                    log_fatal_hash("file_update_block : No data found to read ",
                            chksum);
                    die_dataerr
                        ("file_update_block : No data found to read, this should never happen: inode :%llu: blocknr :%llu",
                         inode, blocknr);
                }
            }
        } else {
            data = s_malloc(sizeof(DBT));
            dta = (QDTA *) decrypted->data;
            data->data = s_malloc(dta->size);
            memcpy(data->data, dta->data, dta->size);
            data->size = dta->size;
            DBTfree(decrypted);
            LDEBUG("data->size = %lu", data->size);
        }
        release_moddb_lock();
        if (compressed && data->size < BLKSIZE) {
#ifdef LZO
            uncompdata = lzo_decompress(data->data, data->size);
#else
            uncompdata = clz_decompress(data->data, data->size);
#endif
            memcpy(dbdata, uncompdata->data, uncompdata->size);
            comprfree(uncompdata);
        } else {
            memcpy(dbdata, data->data, data->size);
        }
        DBTfree(data);
    }
    memcpy(dbdata + offsetblock, blockdata, size);
    add_blk_to_cache(inode, blocknr, offsetblock + size, dbdata, offset);
    inuse = file_get_inuse(chksum);
    if (NULL == inuse)
        die_dataerr("file_update_block : hash not found");
    if (inuse->inuse <= 1) {
        file_delete_data_cache(chksum, &inobno);
        put_on_freelist(inuse);
        delete_inuse(chksum);
    } else {
        inuse->inuse--;
        file_update_inuse(chksum, inuse);
    }
    free(inuse);
    update_filesize(inode, size, offsetblock, blocknr, 0, 0, 0);
    free(dbdata);
    EFUNC;
    return;
}
Пример #2
0
int mongo_write(const char *path, const char *buf, size_t size,
                off_t offset, struct fuse_file_info *fi)
{
    struct inode * e;
    int res;
    size_t reallen;
    int32_t realend = size, blk_offset = 0;
    const off_t write_end = size + offset;
    char * lock;
    bson doc, cond;
    mongo * conn = get_conn();
    uint8_t hash[20];
    time_t now = time(NULL);

    e = (struct inode*)fi->fh;
    if((res = get_cached_inode(path, e)) != 0)
        return res;

    if(e->mode & S_IFDIR)
        return -EISDIR;

    /* Uncomment this for incredibly slow length calculations.
    for(;realend >= 0 && buf[realend] == '\0'; realend--);
    realend++;
    for(blk_offset = 0; blk_offset < realend && buf[blk_offset] == 0; blk_offset++);
    blk_offset -= blk_offset > 0 ? 1 : 0;
    * The code below uses SSE4 instructions to find the first/last
    * zero bytes by doing 16-byte comparisons at a time. This should give
    * a ~16 speed boost on blocks with lots of zero bytes over the dumb
    * method above.
    */
    
    if(size >= 16) {
        __m128i zero = _mm_setzero_si128();
        lock = (char*)buf + size - 16;
        while(lock >= buf) {
            __m128i x = _mm_loadu_si128((__m128i*)lock);
            res = _mm_movemask_epi8(_mm_cmpeq_epi8(zero, x));
            if(res == 0xffff) {
                lock -= 16;
                continue;
            }
            realend = lock - buf + fls(res ^ 0xffff);
            break;
        }
        if(lock <= buf)
            realend = 0;

        lock = (char*)buf;
        while(lock - buf < realend) {
            __m128i x = _mm_loadu_si128((__m128i*)lock);
            res = _mm_movemask_epi8(_mm_cmpeq_epi8(zero, x));
            if(res == 0xffff) {
                lock += 16;
                continue;
            }
            blk_offset = lock - buf + ffs(res ^ 0xffff) - 1;
            break;
        }
    }

    reallen = realend - blk_offset;
    if(reallen == 0) {
        pthread_mutex_lock(&e->wr_lock);
        res = insert_empty(&e->wr_extent, offset, size);
        goto end;
    }

#ifdef __APPLE__
    CC_SHA1(buf, size, hash);
#else
    SHA1(buf, size, hash);
#endif

    bson_init(&cond);
    bson_append_binary(&cond, "_id", 0, (char*)hash, sizeof(hash));
    bson_finish(&cond);

    bson_init(&doc);
    bson_append_start_object(&doc, "$setOnInsert");
    char * comp_out = get_compress_buf();
    size_t comp_size = snappy_max_compressed_length(reallen);
    if((res = snappy_compress(buf + blk_offset, reallen,
        comp_out, &comp_size)) != SNAPPY_OK) {
        fprintf(stderr, "Error compressing input: %d\n", res);
        return -EIO;
    }
    bson_append_binary(&doc, "data", 0, comp_out, comp_size);
    bson_append_int(&doc, "offset", blk_offset);
    bson_append_int(&doc, "size", size);
    bson_append_time_t(&doc, "created", now);
    bson_append_finish_object(&doc);
    bson_finish(&doc);

    res = mongo_update(conn, blocks_name, &cond, &doc,
        MONGO_UPDATE_UPSERT, NULL);
    bson_destroy(&doc);
    bson_destroy(&cond);

    if(res != MONGO_OK) {
        fprintf(stderr, "Error committing block %s\n", conn->lasterrstr);
        return -EIO;
    }

    pthread_mutex_lock(&e->wr_lock);
    res = insert_hash(&e->wr_extent, offset, size, hash);

end:
    if(write_end > e->size)
        e->size = write_end;

    if(now - e->wr_age > 3) {
        res = serialize_extent(e, e->wr_extent);
        if(res != 0) {
            pthread_mutex_unlock(&e->wr_lock);
            return res;    
        }
        e->wr_age = now;
    }
    pthread_mutex_unlock(&e->wr_lock);
    if(res != 0)
        return res;

    res = update_filesize(e, write_end);

    if(res != 0)
        return res;
    return size;
}
Пример #3
0
void add_file_block(BLKDTA * blkdta)
{
    INOBNO inobno;
    DBT *cachedata = NULL;
    INUSE *inuse;
	OFFHASH *offhash;

    inobno.inode = blkdta->inode;
    inobno.blocknr = blkdta->blocknr;

    FUNC;
    LDEBUG("add_file_block : inode %llu - %llu", inobno.inode,
           inobno.blocknr);
    if (blkdta->bsize + blkdta->offsetblock < BLKSIZE) {
// Flush the blockcache before overwriting.
        cachedata = try_block_cache(blkdta->inode, blkdta->blocknr, 0);
        if (cachedata)
            DBTfree(cachedata);
        add_blk_to_cache(blkdta->inode, blkdta->blocknr, 
			blkdta->bsize + blkdta->offsetblock, blkdta->blockfiller, 
			blkdta->offsetfile);
        LDEBUG
            ("add_file_block : wrote with add_blk_to_cache  : inode %llu - %llu size %i",
             inobno.inode, inobno.blocknr, blkdta->bsize);
        update_filesize(blkdta->inode, blkdta->bsize, blkdta->offsetblock,
                        blkdta->blocknr, blkdta->sparse, 0, 0);
        return;
    }
    inuse = file_get_inuse(blkdta->stiger);
    if (inuse == NULL) {
        if (NULL == blkdta->compressed) {
#ifdef LZO
            blkdta->compressed =
                lzo_compress(blkdta->blockfiller, BLKSIZE);
#else
            blkdta->compressed =
                clz_compress(blkdta->blockfiller, BLKSIZE);
#endif
        }
        LDEBUG("Compressed %i bytes to %lu bytes", BLKSIZE,
               blkdta->compressed->size);
        loghash("add_file_block call qdta for hash :", blkdta->stiger);
        inuse = s_malloc(sizeof(INUSE));
        inuse->inuse = 0;
        inuse->offset = get_offset(blkdta->compressed->size);
        LDEBUG("add to offset %llu", inuse->offset);
        inuse->size = blkdta->compressed->size;
        file_qdta(&inobno, blkdta->stiger, blkdta->compressed->data,
                  blkdta->compressed->size, inuse->offset);
        loghash("add_file_block queued with qdta", blkdta->stiger);
        update_filesize(blkdta->inode, blkdta->bsize, blkdta->offsetblock,
                        blkdta->blocknr, blkdta->sparse,
                        blkdta->compressed->size, 0);
    } else {
        update_filesize(blkdta->inode, blkdta->bsize, blkdta->offsetblock,
                        blkdta->blocknr, blkdta->sparse, 0, 1);
    }
    if (NULL != blkdta->compressed)
        comprfree(blkdta->compressed);
    inuse->inuse = inuse->inuse + 1;
    file_update_inuse(blkdta->stiger, inuse);
	offhash = s_malloc(sizeof(OFFHASH));
	offhash->offset = blkdta->offsetfile;
	memcpy(offhash->stiger, blkdta->stiger, config->hashlen);
    write_dbb_to_cache(&inobno, offhash);
	free(offhash);
    free(inuse);
    EFUNC;
    return;
}