static void ngx_http_tfs_remote_block_cache_get_handler(ngx_http_tair_key_value_t *kv, ngx_int_t rc, void *data) { u_char *p, *q; uint32_t ds_count; ngx_http_tfs_t *t; ngx_http_tfs_inet_t *addr; ngx_http_tfs_segment_data_t *segment_data; ngx_http_tfs_block_cache_key_t key; ngx_http_tfs_block_cache_value_t value; ngx_http_tfs_remote_block_cache_ctx_t *ctx = data; t = ctx->data; segment_data = &t->file.segment_data[t->file.segment_index]; if (rc == NGX_HTTP_ETAIR_SUCCESS) { q = kv->key.data; p = kv->value->data; if (p != NULL && (kv->value->len > NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE)) { key.ns_addr = *(uint64_t *)q; q += sizeof(uint64_t); key.block_id = *(uint32_t *)q; ds_count = *(uint32_t *)p; p += sizeof(uint32_t); if (ds_count > 0) { segment_data->block_info.ds_count = ds_count; segment_data->block_info.ds_addrs = ngx_pcalloc(t->pool, sizeof(ngx_http_tfs_inet_t) * ds_count); if (segment_data->block_info.ds_addrs == NULL) { ngx_http_tfs_finalize_request(t->data, t, NGX_HTTP_INTERNAL_SERVER_ERROR); return; } ngx_memcpy(segment_data->block_info.ds_addrs, p, ds_count * sizeof(ngx_http_tfs_inet_t)); /* insert local block cache */ if (t->block_cache_ctx.use_cache & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) { value.ds_count = ds_count; value.ds_addrs = (uint64_t *)segment_data->block_info.ds_addrs; ngx_http_tfs_local_block_cache_insert( t->block_cache_ctx.local_ctx, t->log, &key, &value); } /* skip GET_BLK_INFO state */ t->state += 1; segment_data->cache_hit = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; /* select data server */ addr = ngx_http_tfs_select_data_server(t, segment_data); ngx_http_tfs_peer_set_addr(t->pool, &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER], addr); } else { /* remote block cache invalid, need remove it */ ngx_http_tfs_remote_block_cache_remove(ctx, t->pool, t->log, &key); } } } else { ngx_log_debug2(NGX_LOG_DEBUG_HTTP, t->log, 0, "lookup remote block cache, " "ns addr: %V, block id: %uD not found", &t->name_server_addr_text, segment_data->segment_info.block_id); } ngx_http_tfs_finalize_state(t, NGX_OK); }
static void ngx_http_tfs_dedup_callback(ngx_http_tfs_dedup_ctx_t *ctx, ngx_int_t rc) { ngx_http_tfs_t *t; ngx_http_tfs_rcs_info_t *rc_info; t = ctx->data; rc_info = t->rc_info_node; switch (t->r_ctx.action.code) { case NGX_HTTP_TFS_ACTION_REMOVE_FILE: switch(t->state) { case NGX_HTTP_TFS_STATE_REMOVE_READ_META_SEGMENT: /* exist in tair */ if (rc == NGX_OK) { /* check file name */ rc = ngx_http_tfs_dedup_check_filename(&ctx->dup_file_name, &t->r_ctx.fsname); if (rc == NGX_OK) { /* file name match, modify ref count and save tair */ if (t->r_ctx.unlink_type == NGX_HTTP_TFS_UNLINK_DELETE) { if (--ctx->file_ref_count <= 0) { /* if ref count is 0, * remove key from tair then unlink file */ t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO; t->is_stat_dup_file = NGX_HTTP_TFS_NO; t->tfs_peer->body_buffer = ctx->save_body_buffer; ctx->file_data = t->meta_segment_data; rc = ngx_http_tfs_dedup_remove(ctx, t->pool, t->log); /* do not care delete tair fail, * go on unlinking file */ if (rc == NGX_ERROR) { ngx_http_tfs_finalize_state(t, NGX_OK); } return; } /* file_ref_count > 0, just save tair */ t->state = NGX_HTTP_TFS_STATE_REMOVE_DONE; ctx->file_data = t->meta_segment_data; rc = ngx_http_tfs_dedup_set(ctx, t->pool, t->log); /* do not care save tair fail, return success */ if (rc == NGX_ERROR) { ngx_http_tfs_finalize_state(t, NGX_DONE); } return; } } /* file name not match, unlink file */ t->tfs_peer->body_buffer = ctx->save_body_buffer; t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO; t->is_stat_dup_file = NGX_HTTP_TFS_NO; ngx_http_tfs_finalize_state(t, NGX_OK); return; } /* not exist in tair, unlink file */ t->tfs_peer->body_buffer = ctx->save_body_buffer; t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO; t->is_stat_dup_file = NGX_HTTP_TFS_NO; ngx_http_tfs_finalize_state(t, NGX_OK); return; case NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO: case NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA: ngx_http_tfs_finalize_state(t, NGX_OK); return; case NGX_HTTP_TFS_STATE_REMOVE_DONE: ngx_http_tfs_finalize_state(t, NGX_DONE); return; } break; case NGX_HTTP_TFS_ACTION_WRITE_FILE: switch(t->state) { case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS: case NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO: /* exist in tair */ if (rc == NGX_OK) { /* check suffix */ rc = ngx_http_tfs_dedup_check_suffix(&ctx->dup_file_name, &t->r_ctx.file_suffix); if (rc == NGX_OK) { /* suffix match, need to stat file */ rc = ngx_http_tfs_raw_fsname_parse(&ctx->dup_file_name, &ctx->dup_file_suffix, &t->r_ctx.fsname); if (rc == NGX_OK) { t->file.cluster_id = t->r_ctx.fsname.cluster_id; t->is_stat_dup_file = NGX_HTTP_TFS_YES; t->state = NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO; } } else { /* suffix not match, need save new tfs file, * do not save tair */ t->use_dedup = NGX_HTTP_TFS_NO; } } /* not exist in tair need save new tfs file and tair */ /* need reset meta segment */ rc = ngx_http_tfs_get_meta_segment(t); if (rc != NGX_OK) { ngx_log_error(NGX_LOG_ERR, t->log, 0, "tfs get meta segment failed"); ngx_http_tfs_finalize_request(t->data, t, NGX_HTTP_INTERNAL_SERVER_ERROR); return; } /* lookup block cache */ if (t->is_stat_dup_file) { /* dedup write may need to stat file */ if (rc_info->use_remote_block_cache) { rc = ngx_http_tfs_get_remote_block_cache_instance( &t->block_cache_ctx.remote_ctx, &rc_info->remote_block_cache_info); if (rc == NGX_ERROR) { ngx_log_error(NGX_LOG_ERR, t->log, 0, "get remote block cache instance failed."); } else { t->block_cache_ctx.use_cache |= NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; } } ngx_http_tfs_lookup_block_cache(t); return; } ngx_http_tfs_finalize_state(t, NGX_OK); break; case NGX_HTTP_TFS_STATE_WRITE_STAT_DUP_FILE: if (rc == NGX_OK) { t->state = NGX_HTTP_TFS_STATE_WRITE_DONE; ngx_http_tfs_finalize_state(t, NGX_DONE); } else { /* save tair(add ref count) failed, * need save new tfs file, do not save tair */ t->state = NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS; t->is_stat_dup_file = NGX_HTTP_TFS_NO; t->use_dedup = NGX_HTTP_TFS_NO; /* need reset output buf */ t->out_bufs = NULL; /* need reset block id and file id */ t->file.segment_data[0].segment_info.block_id = 0; t->file.segment_data[0].segment_info.file_id = 0; ngx_http_tfs_finalize_state(t, NGX_OK); } break; case NGX_HTTP_TFS_STATE_WRITE_DONE: ngx_http_tfs_finalize_state(t, NGX_DONE); break; } } return; }
static void ngx_http_tfs_remote_block_cache_mget_handler(ngx_array_t *kvs, ngx_int_t rc, void *data) { u_char *p, *q; uint32_t ds_count, block_count; ngx_uint_t i, j, hit_count; ngx_http_tfs_t *t; ngx_http_tair_key_value_t *kv; ngx_http_tfs_segment_data_t *segment_data; ngx_http_tfs_block_cache_key_t key; ngx_http_tfs_block_cache_value_t value; ngx_http_tfs_remote_block_cache_ctx_t *ctx = data; t = ctx->data; segment_data = &t->file.segment_data[t->file.segment_index]; block_count = t->file.segment_count - t->file.segment_index; if (block_count > NGX_HTTP_TFS_MAX_BATCH_COUNT) { block_count = NGX_HTTP_TFS_MAX_BATCH_COUNT; } if (rc == NGX_OK) { kv = kvs->elts; hit_count = 0; for (i = 0; i < kvs->nelts; i++, kv++) { if (kv->rc != NGX_HTTP_ETAIR_SUCCESS) { continue; } q = kv->key.data; p = kv->value->data; if (p != NULL && (kv->value->len > NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE)) { key.ns_addr = *(uint64_t *)q; q += sizeof(uint64_t); key.block_id = *(uint32_t *)q; ds_count = *(uint32_t *)p; p += sizeof(uint32_t); if (ds_count > 0) { /* find out segment */ for (j = 0; j < block_count; j++) { if(segment_data[j].segment_info.block_id == key.block_id && segment_data[j].block_info.ds_addrs == NULL) { break; } } /* not found, some error happen */ if (j == block_count) { continue; } segment_data[j].block_info.ds_count = ds_count; segment_data[j].block_info.ds_addrs = ngx_pcalloc(t->pool, ds_count * sizeof(ngx_http_tfs_inet_t)); if (segment_data[j].block_info.ds_addrs == NULL) { ngx_http_tfs_finalize_request(t->data, t, NGX_HTTP_INTERNAL_SERVER_ERROR); return; } ngx_memcpy(segment_data[j].block_info.ds_addrs, p, ds_count * sizeof(ngx_http_tfs_inet_t)); if (t->block_cache_ctx.use_cache & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) { value.ds_count = ds_count; value.ds_addrs = (uint64_t *)segment_data[j].block_info.ds_addrs; ngx_http_tfs_local_block_cache_insert( t->block_cache_ctx.local_ctx, t->log, &key, &value); } hit_count++; segment_data[j].cache_hit = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; ngx_log_debug2(NGX_LOG_DEBUG_HTTP, t->log, 0, "remote block cache hit, " "ns addr: %V, block id: %uD", &t->name_server_addr_text, segment_data[j].segment_info.block_id); } else { /* remote block cache invalid, need remove it */ ngx_http_tfs_remote_block_cache_remove(ctx, t->pool, t->log, &key); } } } if (hit_count > 0) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, t->log, 0, "batch lookup remote block cache, hit_count: %ui", hit_count); /* remote block cache hit count */ t->file.curr_batch_count += hit_count; if (hit_count == kvs->nelts) { /* all cache hit, start batch process */ t->decline_handler = ngx_http_tfs_batch_process_start; rc = NGX_DECLINED; } } } else { rc = NGX_OK; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, t->log, 0, "remote block cache miss"); } ngx_http_tfs_finalize_state(t, rc); }