static void
ngx_http_tfs_remote_block_cache_get_handler(ngx_http_tair_key_value_t *kv,
ngx_int_t rc, void *data)
{
u_char *p, *q;
uint32_t ds_count;
ngx_http_tfs_t *t;
ngx_http_tfs_inet_t *addr;
ngx_http_tfs_segment_data_t *segment_data;
ngx_http_tfs_block_cache_key_t key;
ngx_http_tfs_block_cache_value_t value;
ngx_http_tfs_remote_block_cache_ctx_t *ctx = data;
t = ctx->data;
segment_data = &t->file.segment_data[t->file.segment_index];
if (rc == NGX_HTTP_ETAIR_SUCCESS) {
q = kv->key.data;
p = kv->value->data;
if (p != NULL
&& (kv->value->len
> NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE))
{
key.ns_addr = *(uint64_t *)q;
q += sizeof(uint64_t);
key.block_id = *(uint32_t *)q;
ds_count = *(uint32_t *)p;
p += sizeof(uint32_t);
if (ds_count > 0) {
segment_data->block_info.ds_count = ds_count;
segment_data->block_info.ds_addrs = ngx_pcalloc(t->pool,
sizeof(ngx_http_tfs_inet_t) * ds_count);
if (segment_data->block_info.ds_addrs == NULL) {
ngx_http_tfs_finalize_request(t->data, t,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
ngx_memcpy(segment_data->block_info.ds_addrs, p,
ds_count * sizeof(ngx_http_tfs_inet_t));
/* insert local block cache */
if (t->block_cache_ctx.use_cache
& NGX_HTTP_TFS_LOCAL_BLOCK_CACHE)
{
value.ds_count = ds_count;
value.ds_addrs =
(uint64_t *)segment_data->block_info.ds_addrs;
ngx_http_tfs_local_block_cache_insert(
t->block_cache_ctx.local_ctx,
t->log, &key, &value);
}
/* skip GET_BLK_INFO state */
t->state += 1;
segment_data->cache_hit = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE;
/* select data server */
addr = ngx_http_tfs_select_data_server(t, segment_data);
ngx_http_tfs_peer_set_addr(t->pool,
&t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER],
addr);
} else {
/* remote block cache invalid, need remove it */
ngx_http_tfs_remote_block_cache_remove(ctx, t->pool, t->log,
&key);
}
}
} else {
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, t->log, 0,
"lookup remote block cache, "
"ns addr: %V, block id: %uD not found",
&t->name_server_addr_text,
segment_data->segment_info.block_id);
}
ngx_http_tfs_finalize_state(t, NGX_OK);
}
static void
ngx_http_tfs_dedup_callback(ngx_http_tfs_dedup_ctx_t *ctx, ngx_int_t rc)
{
ngx_http_tfs_t *t;
ngx_http_tfs_rcs_info_t *rc_info;
t = ctx->data;
rc_info = t->rc_info_node;
switch (t->r_ctx.action.code) {
case NGX_HTTP_TFS_ACTION_REMOVE_FILE:
switch(t->state) {
case NGX_HTTP_TFS_STATE_REMOVE_READ_META_SEGMENT:
/* exist in tair */
if (rc == NGX_OK) {
/* check file name */
rc = ngx_http_tfs_dedup_check_filename(&ctx->dup_file_name,
&t->r_ctx.fsname);
if (rc == NGX_OK) {
/* file name match, modify ref count and save tair */
if (t->r_ctx.unlink_type == NGX_HTTP_TFS_UNLINK_DELETE) {
if (--ctx->file_ref_count <= 0) {
/* if ref count is 0,
* remove key from tair then unlink file
*/
t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
t->tfs_peer->body_buffer = ctx->save_body_buffer;
ctx->file_data = t->meta_segment_data;
rc = ngx_http_tfs_dedup_remove(ctx, t->pool,
t->log);
/* do not care delete tair fail,
* go on unlinking file
*/
if (rc == NGX_ERROR) {
ngx_http_tfs_finalize_state(t, NGX_OK);
}
return;
}
/* file_ref_count > 0, just save tair */
t->state = NGX_HTTP_TFS_STATE_REMOVE_DONE;
ctx->file_data = t->meta_segment_data;
rc = ngx_http_tfs_dedup_set(ctx, t->pool, t->log);
/* do not care save tair fail, return success */
if (rc == NGX_ERROR) {
ngx_http_tfs_finalize_state(t, NGX_DONE);
}
return;
}
}
/* file name not match, unlink file */
t->tfs_peer->body_buffer = ctx->save_body_buffer;
t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
ngx_http_tfs_finalize_state(t, NGX_OK);
return;
}
/* not exist in tair, unlink file */
t->tfs_peer->body_buffer = ctx->save_body_buffer;
t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
ngx_http_tfs_finalize_state(t, NGX_OK);
return;
case NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO:
case NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA:
ngx_http_tfs_finalize_state(t, NGX_OK);
return;
case NGX_HTTP_TFS_STATE_REMOVE_DONE:
ngx_http_tfs_finalize_state(t, NGX_DONE);
return;
}
break;
case NGX_HTTP_TFS_ACTION_WRITE_FILE:
switch(t->state) {
case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS:
case NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO:
/* exist in tair */
if (rc == NGX_OK) {
/* check suffix */
rc = ngx_http_tfs_dedup_check_suffix(&ctx->dup_file_name,
&t->r_ctx.file_suffix);
if (rc == NGX_OK) {
/* suffix match, need to stat file */
rc = ngx_http_tfs_raw_fsname_parse(&ctx->dup_file_name,
&ctx->dup_file_suffix,
&t->r_ctx.fsname);
if (rc == NGX_OK) {
t->file.cluster_id = t->r_ctx.fsname.cluster_id;
t->is_stat_dup_file = NGX_HTTP_TFS_YES;
t->state = NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO;
}
} else {
/* suffix not match, need save new tfs file,
* do not save tair
*/
t->use_dedup = NGX_HTTP_TFS_NO;
}
} /* not exist in tair need save new tfs file and tair */
/* need reset meta segment */
rc = ngx_http_tfs_get_meta_segment(t);
if (rc != NGX_OK) {
ngx_log_error(NGX_LOG_ERR, t->log, 0,
"tfs get meta segment failed");
ngx_http_tfs_finalize_request(t->data, t,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
/* lookup block cache */
if (t->is_stat_dup_file) {
/* dedup write may need to stat file */
if (rc_info->use_remote_block_cache) {
rc = ngx_http_tfs_get_remote_block_cache_instance(
&t->block_cache_ctx.remote_ctx,
&rc_info->remote_block_cache_info);
if (rc == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, t->log, 0,
"get remote block cache instance failed.");
} else {
t->block_cache_ctx.use_cache |=
NGX_HTTP_TFS_REMOTE_BLOCK_CACHE;
}
}
ngx_http_tfs_lookup_block_cache(t);
return;
}
ngx_http_tfs_finalize_state(t, NGX_OK);
break;
case NGX_HTTP_TFS_STATE_WRITE_STAT_DUP_FILE:
if (rc == NGX_OK) {
t->state = NGX_HTTP_TFS_STATE_WRITE_DONE;
ngx_http_tfs_finalize_state(t, NGX_DONE);
} else {
/* save tair(add ref count) failed,
* need save new tfs file, do not save tair
*/
t->state = NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
t->use_dedup = NGX_HTTP_TFS_NO;
/* need reset output buf */
t->out_bufs = NULL;
/* need reset block id and file id */
t->file.segment_data[0].segment_info.block_id = 0;
t->file.segment_data[0].segment_info.file_id = 0;
ngx_http_tfs_finalize_state(t, NGX_OK);
}
break;
case NGX_HTTP_TFS_STATE_WRITE_DONE:
ngx_http_tfs_finalize_state(t, NGX_DONE);
break;
}
}
return;
}
static void
ngx_http_tfs_remote_block_cache_mget_handler(ngx_array_t *kvs, ngx_int_t rc,
void *data)
{
u_char *p, *q;
uint32_t ds_count, block_count;
ngx_uint_t i, j, hit_count;
ngx_http_tfs_t *t;
ngx_http_tair_key_value_t *kv;
ngx_http_tfs_segment_data_t *segment_data;
ngx_http_tfs_block_cache_key_t key;
ngx_http_tfs_block_cache_value_t value;
ngx_http_tfs_remote_block_cache_ctx_t *ctx = data;
t = ctx->data;
segment_data = &t->file.segment_data[t->file.segment_index];
block_count = t->file.segment_count - t->file.segment_index;
if (block_count > NGX_HTTP_TFS_MAX_BATCH_COUNT) {
block_count = NGX_HTTP_TFS_MAX_BATCH_COUNT;
}
if (rc == NGX_OK) {
kv = kvs->elts;
hit_count = 0;
for (i = 0; i < kvs->nelts; i++, kv++) {
if (kv->rc != NGX_HTTP_ETAIR_SUCCESS) {
continue;
}
q = kv->key.data;
p = kv->value->data;
if (p != NULL
&& (kv->value->len
> NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE))
{
key.ns_addr = *(uint64_t *)q;
q += sizeof(uint64_t);
key.block_id = *(uint32_t *)q;
ds_count = *(uint32_t *)p;
p += sizeof(uint32_t);
if (ds_count > 0) {
/* find out segment */
for (j = 0; j < block_count; j++) {
if(segment_data[j].segment_info.block_id == key.block_id
&& segment_data[j].block_info.ds_addrs == NULL)
{
break;
}
}
/* not found, some error happen */
if (j == block_count) {
continue;
}
segment_data[j].block_info.ds_count = ds_count;
segment_data[j].block_info.ds_addrs = ngx_pcalloc(t->pool,
ds_count * sizeof(ngx_http_tfs_inet_t));
if (segment_data[j].block_info.ds_addrs == NULL) {
ngx_http_tfs_finalize_request(t->data, t,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
ngx_memcpy(segment_data[j].block_info.ds_addrs, p,
ds_count * sizeof(ngx_http_tfs_inet_t));
if (t->block_cache_ctx.use_cache
& NGX_HTTP_TFS_LOCAL_BLOCK_CACHE)
{
value.ds_count = ds_count;
value.ds_addrs =
(uint64_t *)segment_data[j].block_info.ds_addrs;
ngx_http_tfs_local_block_cache_insert(
t->block_cache_ctx.local_ctx, t->log, &key, &value);
}
hit_count++;
segment_data[j].cache_hit = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE;
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, t->log, 0,
"remote block cache hit, "
"ns addr: %V, block id: %uD",
&t->name_server_addr_text,
segment_data[j].segment_info.block_id);
} else {
/* remote block cache invalid, need remove it */
ngx_http_tfs_remote_block_cache_remove(ctx, t->pool, t->log,
&key);
}
}
}
if (hit_count > 0) {
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, t->log, 0,
"batch lookup remote block cache, hit_count: %ui",
hit_count);
/* remote block cache hit count */
t->file.curr_batch_count += hit_count;
if (hit_count == kvs->nelts) {
/* all cache hit, start batch process */
t->decline_handler = ngx_http_tfs_batch_process_start;
rc = NGX_DECLINED;
}
}
} else {
rc = NGX_OK;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, t->log, 0,
"remote block cache miss");
}
ngx_http_tfs_finalize_state(t, rc);
}
