ngx_int_t
ngx_http_tfs_process_ns(ngx_http_tfs_t *t)
{
uint32_t cluster_id;
ngx_buf_t *b;
ngx_int_t rc;
ngx_str_t *cluster_id_text;
ngx_http_tfs_inet_t *addr;
ngx_http_tfs_header_t *header;
ngx_http_tfs_rcs_info_t *rc_info;
ngx_http_tfs_peer_connection_t *tp;
ngx_http_tfs_logical_cluster_t *logical_cluster;
ngx_http_tfs_physical_cluster_t *physical_cluster;
header = (ngx_http_tfs_header_t *) t->header;
tp = t->tfs_peer;
b = &tp->body_buffer;
if (ngx_buf_size(b) < header->len) {
return NGX_AGAIN;
}
rc = ngx_http_tfs_name_server_parse_message(t);
ngx_http_tfs_clear_buf(b);
if (rc == NGX_ERROR) {
return rc;
}
if (rc <= NGX_HTTP_TFS_EXIT_GENERAL_ERROR) {
return NGX_HTTP_TFS_AGAIN;
}
switch (t->r_ctx.action.code) {
case NGX_HTTP_TFS_ACTION_STAT_FILE:
t->state = NGX_HTTP_TFS_STATE_STAT_STAT_FILE;
break;
case NGX_HTTP_TFS_ACTION_READ_FILE:
if (!t->parent
&& (t->r_ctx.version == 2
|| (t->is_large_file && !t->is_process_meta_seg)))
{
t->decline_handler = ngx_http_tfs_batch_process_start;
return NGX_DECLINED;
}
t->state = NGX_HTTP_TFS_STATE_READ_READ_DATA;
break;
case NGX_HTTP_TFS_ACTION_WRITE_FILE:
switch(t->state) {
case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS:
/* save cluster id */
if (t->loc_conf->upstream->enable_rcs) {
rc_info = t->rc_info_node;
logical_cluster =
&rc_info->logical_clusters[t->logical_cluster_index];
physical_cluster =
&logical_cluster->rw_clusters[t->rw_cluster_index];
/* check ns cluster id with rc configure */
cluster_id_text = &physical_cluster->cluster_id_text;
cluster_id = ngx_http_tfs_get_cluster_id(cluster_id_text->data);
if (t->file.cluster_id != cluster_id) {
ngx_log_error(NGX_LOG_ERR,
t->log, 0,
"error, cluster id conflict: "
"%uD(ns) <> %uD(rcs)",
t->file.cluster_id,
cluster_id);
return NGX_ERROR;
}
physical_cluster->cluster_id = t->file.cluster_id;
} else {
t->main_conf->cluster_id = t->file.cluster_id;
}
t->state = NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO;
return rc;
case NGX_HTTP_TFS_STATE_WRITE_GET_GROUP_COUNT:
if (t->group_count != 1) {
t->state = NGX_HTTP_TFS_STATE_WRITE_GET_GROUP_SEQ;
return rc;
}
/* group_count == 1, maybe able to make choice */
t->group_seq = 0;
case NGX_HTTP_TFS_STATE_WRITE_GET_GROUP_SEQ:
rc_info = t->rc_info_node;
ngx_http_tfs_rcs_set_group_info_by_addr(rc_info,
t->group_count,
t->group_seq,
t->name_server_addr);
rc = ngx_http_tfs_select_name_server(t, rc_info,
&t->name_server_addr,
&t->name_server_addr_text);
if (rc == NGX_ERROR) {
return NGX_HTTP_TFS_EXIT_SERVER_OBJECT_NOT_FOUND;
}
tp->peer.free(&tp->peer, tp->peer.data, 0);
ngx_http_tfs_peer_set_addr(t->pool,
&t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER],
&t->name_server_addr);
return rc;
case NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO:
if (t->is_stat_dup_file) {
t->state = NGX_HTTP_TFS_STATE_WRITE_STAT_DUP_FILE;
} else if (t->is_rolling_back) {
t->state = NGX_HTTP_TFS_STATE_WRITE_DELETE_DATA;
} else {
if (!t->parent
&& (t->r_ctx.version == 2
|| (t->is_large_file && !t->is_process_meta_seg)))
{
t->decline_handler = ngx_http_tfs_batch_process_start;
return NGX_DECLINED;
}
t->state = NGX_HTTP_TFS_STATE_WRITE_CREATE_FILE_NAME;
}
break;
}
break;
case NGX_HTTP_TFS_ACTION_REMOVE_FILE:
switch (t->state) {
case NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_COUNT:
/* maybe able to make choice */
if (t->group_count != 1) {
t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_SEQ;
}
/* group_count == 1, maybe able to make choice */
t->group_seq = 0;
case NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_SEQ:
rc_info = t->rc_info_node;
ngx_http_tfs_rcs_set_group_info_by_addr(rc_info,
t->group_count,
t->group_seq,
t->name_server_addr);
rc = ngx_http_tfs_select_name_server(t, rc_info,
&t->name_server_addr,
&t->name_server_addr_text);
if (rc == NGX_ERROR) {
/* in order to return 404 */
return NGX_HTTP_TFS_EXIT_SERVER_OBJECT_NOT_FOUND;
}
tp->peer.free(&tp->peer, tp->peer.data, 0);
ngx_http_tfs_peer_set_addr(t->pool,
&t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER],
&t->name_server_addr);
return rc;
case NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO:
if (t->is_large_file
&& t->r_ctx.unlink_type == NGX_HTTP_TFS_UNLINK_DELETE
&& t->meta_segment_data == NULL)
{
t->state = NGX_HTTP_TFS_STATE_REMOVE_READ_META_SEGMENT;
} else if (t->is_stat_dup_file) {
t->state = NGX_HTTP_TFS_STATE_REMOVE_STAT_FILE;
} else {
t->state = NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA;
}
}
break;
}
addr = ngx_http_tfs_select_data_server(t,
&t->file.segment_data[t->file.segment_index]);
if (addr == NULL) {
return NGX_ERROR;
}
ngx_http_tfs_peer_set_addr(t->pool,
&t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER],
addr);
return rc;
}
static void
ngx_http_tfs_remote_block_cache_get_handler(ngx_http_tair_key_value_t *kv,
ngx_int_t rc, void *data)
{
u_char *p, *q;
uint32_t ds_count;
ngx_http_tfs_t *t;
ngx_http_tfs_inet_t *addr;
ngx_http_tfs_segment_data_t *segment_data;
ngx_http_tfs_block_cache_key_t key;
ngx_http_tfs_block_cache_value_t value;
ngx_http_tfs_remote_block_cache_ctx_t *ctx = data;
t = ctx->data;
segment_data = &t->file.segment_data[t->file.segment_index];
if (rc == NGX_HTTP_ETAIR_SUCCESS) {
q = kv->key.data;
p = kv->value->data;
if (p != NULL
&& (kv->value->len
> NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE))
{
key.ns_addr = *(uint64_t *)q;
q += sizeof(uint64_t);
key.block_id = *(uint32_t *)q;
ds_count = *(uint32_t *)p;
p += sizeof(uint32_t);
if (ds_count > 0) {
segment_data->block_info.ds_count = ds_count;
segment_data->block_info.ds_addrs = ngx_pcalloc(t->pool,
sizeof(ngx_http_tfs_inet_t) * ds_count);
if (segment_data->block_info.ds_addrs == NULL) {
ngx_http_tfs_finalize_request(t->data, t,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
ngx_memcpy(segment_data->block_info.ds_addrs, p,
ds_count * sizeof(ngx_http_tfs_inet_t));
/* insert local block cache */
if (t->block_cache_ctx.use_cache
& NGX_HTTP_TFS_LOCAL_BLOCK_CACHE)
{
value.ds_count = ds_count;
value.ds_addrs =
(uint64_t *)segment_data->block_info.ds_addrs;
ngx_http_tfs_local_block_cache_insert(
t->block_cache_ctx.local_ctx,
t->log, &key, &value);
}
/* skip GET_BLK_INFO state */
t->state += 1;
segment_data->cache_hit = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE;
/* select data server */
addr = ngx_http_tfs_select_data_server(t, segment_data);
ngx_http_tfs_peer_set_addr(t->pool,
&t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER],
addr);
} else {
/* remote block cache invalid, need remove it */
ngx_http_tfs_remote_block_cache_remove(ctx, t->pool, t->log,
&key);
}
}
} else {
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, t->log, 0,
"lookup remote block cache, "
"ns addr: %V, block id: %uD not found",
&t->name_server_addr_text,
segment_data->segment_info.block_id);
}
ngx_http_tfs_finalize_state(t, NGX_OK);
}
ngx_int_t
ngx_http_tfs_retry_ds(ngx_http_tfs_t *t)
{
ngx_int_t rc;
ngx_http_tfs_inet_t *addr;
ngx_http_tfs_segment_data_t *segment_data;
ngx_http_tfs_peer_connection_t *tp;
tp = t->tfs_peer;
tp->peer.free(&tp->peer, tp->peer.data, 0);
segment_data = &t->file.segment_data[t->file.segment_index];
addr = ngx_http_tfs_select_data_server(t, segment_data);
if (addr == NULL) {
switch(t->r_ctx.action.code) {
case NGX_HTTP_TFS_ACTION_STAT_FILE:
t->state = NGX_HTTP_TFS_STATE_STAT_GET_BLK_INFO;
break;
case NGX_HTTP_TFS_ACTION_READ_FILE:
t->state = NGX_HTTP_TFS_STATE_READ_GET_BLK_INFO;
break;
case NGX_HTTP_TFS_ACTION_REMOVE_FILE:
if (t->is_large_file && t->is_process_meta_seg) {
return NGX_HTTP_TFS_EXIT_SERVER_OBJECT_NOT_FOUND;
}
/* TODO: dedup */
return NGX_ERROR;
case NGX_HTTP_TFS_ACTION_WRITE_FILE:
/* update not allow retry */
if (t->r_ctx.is_raw_update) {
return NGX_ERROR;
}
/* stat retry_ds failed, do not dedup,
* save new tfs file and do not save tair
*/
if (t->is_stat_dup_file) {
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
t->use_dedup = NGX_HTTP_TFS_NO;
t->state = NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS;
t->file.segment_data[0].segment_info.block_id = 0;
t->file.segment_data[0].segment_info.file_id = 0;
t->retry_curr_ns = NGX_HTTP_TFS_YES;
} else {
/* allow retry other writable clusters */
if (++t->retry_count <= NGX_HTTP_TFS_MAX_RETRY_COUNT) {
t->retry_curr_ns = NGX_HTTP_TFS_YES;
}
t->state = NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO;
segment_data->segment_info.block_id = 0;
segment_data->segment_info.file_id = 0;
segment_data->write_file_number = 0;
segment_data->segment_info.crc = 0;
/* reset all write data from orig_data */
segment_data->data = NULL;
rc = ngx_chain_add_copy_with_buf(t->pool,
&segment_data->data, segment_data->orig_data);
if (rc == NGX_ERROR) {
return NGX_ERROR;
}
t->file.left_length = segment_data->segment_info.size;
segment_data->oper_offset = 0;
segment_data->oper_size = ngx_min(t->file.left_length,
NGX_HTTP_TFS_MAX_FRAGMENT_SIZE);
}
break;
default:
return NGX_ERROR;
}
t->tfs_peer = ngx_http_tfs_select_peer(t);
if (t->tfs_peer == NULL) {
return NGX_ERROR;
}
t->recv_chain->buf = &t->header_buffer;
t->recv_chain->next->buf = &t->tfs_peer->body_buffer;
/* reset ds retry count */
segment_data->ds_retry = 0;
if (t->retry_handler == NULL) {
return NGX_ERROR;
}
return t->retry_handler(t);
}
ngx_http_tfs_peer_set_addr(t->pool,
&t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER],
addr);
if (ngx_http_tfs_reinit(t->data, t) != NGX_OK) {
return NGX_ERROR;
}
ngx_http_tfs_connect(t);
return NGX_OK;
}
