ngx_int_t
ngx_http_tfs_duplicate_check_filename(ngx_str_t *dup_file_name, ngx_http_tfs_raw_fsname_t* fsname)
{
ngx_int_t rc;
ngx_str_t dup_file_suffix = ngx_null_string;
ngx_http_tfs_raw_fsname_t dup_fsname;
rc = ngx_http_tfs_raw_fsname_parse(dup_file_name, &dup_file_suffix, &dup_fsname);
if (rc == NGX_OK) {
if (fsname->cluster_id == dup_fsname.cluster_id
&& fsname->file.block_id == dup_fsname.file.block_id
&& fsname->file.seq_id == dup_fsname.file.seq_id
&& fsname->file.suffix == dup_fsname.file.suffix)
{
return NGX_OK;
}
}
return NGX_ERROR;
}
static ngx_int_t
ngx_http_restful_parse_action_raw(ngx_http_request_t *r,
ngx_http_tfs_restful_ctx_t *ctx)
{
ngx_int_t rc;
ngx_str_t arg_value;
switch(r->method) {
case NGX_HTTP_GET:
if (ngx_http_arg(r, (u_char *) "suffix", 6, &arg_value) == NGX_OK) {
ctx->file_suffix = arg_value;
}
rc = ngx_http_tfs_raw_fsname_parse(&ctx->file_path_s, &ctx->file_suffix,
&ctx->fsname);
if (rc != NGX_OK) {
return NGX_HTTP_BAD_REQUEST;
}
if (ngx_http_arg(r, (u_char *) "type", 4, &arg_value) == NGX_OK) {
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->read_stat_type = ngx_atoi(arg_value.data, arg_value.len);
/* normal_read/stat(0) or force_read/stat(1) */
if (ctx->read_stat_type == NGX_ERROR
|| (ctx->read_stat_type != NGX_HTTP_TFS_READ_STAT_NORMAL
&& ctx->read_stat_type != NGX_HTTP_TFS_READ_STAT_FORCE))
{
return NGX_HTTP_BAD_REQUEST;
}
}
if (ctx->meta) {
ctx->action.code = NGX_HTTP_TFS_ACTION_STAT_FILE;
ngx_str_set(&ctx->action.msg, "stat_file");
} else {
ctx->action.code = NGX_HTTP_TFS_ACTION_READ_FILE;
ngx_str_set(&ctx->action.msg, "read_file");
if (ngx_http_arg(r, (u_char *) "offset", 6, &arg_value) == NGX_OK) {
ctx->offset = ngx_http_tfs_atoll(arg_value.data, arg_value.len);
if (ctx->offset == NGX_ERROR) {
return NGX_HTTP_BAD_REQUEST;
}
}
if (ngx_http_arg(r, (u_char *) "size", 4, &arg_value) == NGX_OK) {
rc = ngx_http_tfs_atoull(arg_value.data,
arg_value.len,
(unsigned long long *)&ctx->size);
if (rc == NGX_ERROR) {
return NGX_HTTP_BAD_REQUEST;
}
if (ctx->size == 0) {
return NGX_HTTP_BAD_REQUEST;
}
return NGX_OK;
}
ctx->size = NGX_HTTP_TFS_MAX_SIZE;
}
break;
case NGX_HTTP_POST:
ctx->action.code = NGX_HTTP_TFS_ACTION_WRITE_FILE;
if (ngx_http_arg(r, (u_char *) "suffix", 6, &arg_value) == NGX_OK) {
ctx->file_suffix = arg_value;
}
if (ngx_http_arg(r, (u_char *) "simple_name", 11, &arg_value)
== NGX_OK)
{
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->simple_name = ngx_atoi(arg_value.data, arg_value.len);
if (ctx->simple_name == NGX_ERROR
|| (ctx->simple_name != NGX_HTTP_TFS_NO
&& ctx->simple_name != NGX_HTTP_TFS_YES))
{
return NGX_HTTP_BAD_REQUEST;
}
}
if (ngx_http_arg(r, (u_char *) "large_file", 10, &arg_value)
== NGX_OK)
{
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->large_file = ngx_atoi(arg_value.data, arg_value.len);
if (ctx->large_file == NGX_ERROR
|| (ctx->large_file != NGX_HTTP_TFS_NO
&& ctx->large_file != NGX_HTTP_TFS_YES))
{
return NGX_HTTP_BAD_REQUEST;
}
}
if (ngx_http_arg(r, (u_char *) "meta_segment", 12, &arg_value)
== NGX_OK)
{
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->write_meta_segment = ngx_atoi(arg_value.data, arg_value.len);
if (ctx->write_meta_segment == NGX_ERROR
|| (ctx->write_meta_segment != NGX_HTTP_TFS_NO
&& ctx->write_meta_segment != NGX_HTTP_TFS_YES))
{
return NGX_HTTP_BAD_REQUEST;
}
}
if (ngx_http_arg(r, (u_char *) "no_dedup", 8, &arg_value) == NGX_OK) {
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->no_dedup = ngx_atoi(arg_value.data, arg_value.len);
if (ctx->no_dedup == NGX_ERROR
|| (ctx->no_dedup != NGX_HTTP_TFS_NO
&& ctx->no_dedup != NGX_HTTP_TFS_YES))
{
return NGX_HTTP_BAD_REQUEST;
}
}
ngx_str_set(&ctx->action.msg, "write_file");
break;
case NGX_HTTP_DELETE:
ctx->action.code = NGX_HTTP_TFS_ACTION_REMOVE_FILE;
ngx_str_set(&ctx->action.msg, "remove_file");
/* for outer user use */
if (ngx_http_arg(r, (u_char *) "hide", 4, &arg_value) == NGX_OK) {
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->unlink_type = ngx_atoi(arg_value.data, arg_value.len);
/* hide(1) or reveal(0)*/
if (ctx->unlink_type == NGX_ERROR
|| (ctx->unlink_type != 0 && ctx->unlink_type != 1))
{
return NGX_HTTP_BAD_REQUEST;
}
/* convert to actual type */
if (ctx->unlink_type == 1) {
ctx->unlink_type = NGX_HTTP_TFS_UNLINK_CONCEAL;
} else {
ctx->unlink_type = NGX_HTTP_TFS_UNLINK_REVEAL;
}
}
if (ngx_http_arg(r, (u_char *) "type", 4, &arg_value) == NGX_OK) {
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->unlink_type = ngx_atoi(arg_value.data, arg_value.len);
/* del(0) or undel(2) or hide(4) or reveal(6)*/
if (ctx->unlink_type == NGX_ERROR
|| (ctx->unlink_type != NGX_HTTP_TFS_UNLINK_DELETE
&& ctx->unlink_type != NGX_HTTP_TFS_UNLINK_UNDELETE
&& ctx->unlink_type != NGX_HTTP_TFS_UNLINK_CONCEAL
&& ctx->unlink_type != NGX_HTTP_TFS_UNLINK_REVEAL))
{
return NGX_HTTP_BAD_REQUEST;
}
}
if (ngx_http_arg(r, (u_char *) "suffix", 6, &arg_value) == NGX_OK) {
ctx->file_suffix = arg_value;
}
rc = ngx_http_tfs_raw_fsname_parse(&ctx->file_path_s, &ctx->file_suffix,
&ctx->fsname);
if (rc != NGX_OK) {
return NGX_HTTP_BAD_REQUEST;
}
/* large file not support UNDELETE */
if ((ctx->fsname.file_type == NGX_HTTP_TFS_LARGE_FILE_TYPE)
&& ctx->unlink_type == NGX_HTTP_TFS_UNLINK_UNDELETE)
{
return NGX_HTTP_BAD_REQUEST;
}
break;
case NGX_HTTP_HEAD:
if (ngx_http_arg(r, (u_char *) "suffix", 6, &arg_value) == NGX_OK) {
ctx->file_suffix = arg_value;
}
rc = ngx_http_tfs_raw_fsname_parse(&ctx->file_path_s, &ctx->file_suffix,
&ctx->fsname);
if (rc != NGX_OK) {
return NGX_HTTP_BAD_REQUEST;
}
if (ngx_http_arg(r, (u_char *) "type", 4, &arg_value) == NGX_OK) {
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->read_stat_type = ngx_atoi(arg_value.data, arg_value.len);
/* normal_read/stat(0) or force_read/stat(1) */
if (ctx->read_stat_type == NGX_ERROR
|| (ctx->read_stat_type != NGX_HTTP_TFS_READ_STAT_NORMAL
&& ctx->read_stat_type != NGX_HTTP_TFS_READ_STAT_FORCE))
{
return NGX_HTTP_BAD_REQUEST;
}
}
ctx->action.code = NGX_HTTP_TFS_ACTION_STAT_FILE;
ngx_str_set(&ctx->action.msg, "stat_file");
ctx->chk_exist = NGX_HTTP_TFS_YES;
break;
case NGX_HTTP_PUT:
ctx->action.code = NGX_HTTP_TFS_ACTION_WRITE_FILE;
if (ngx_http_arg(r, (u_char *) "suffix", 6, &arg_value) == NGX_OK) {
ctx->file_suffix = arg_value;
}
if (ngx_http_arg(r, (u_char *) "simple_name", 11, &arg_value)
== NGX_OK)
{
if (arg_value.len != 1) {
return NGX_HTTP_BAD_REQUEST;
}
ctx->simple_name = ngx_atoi(arg_value.data, arg_value.len);
if (ctx->simple_name == NGX_ERROR
|| (ctx->simple_name != NGX_HTTP_TFS_NO
&& ctx->simple_name != NGX_HTTP_TFS_YES))
{
return NGX_HTTP_BAD_REQUEST;
}
}
if (ctx->file_path_s.data == NULL) {
return NGX_HTTP_BAD_REQUEST;
}
/* large file not support update */
if (ngx_http_arg(r, (u_char *) "large_file", 10, &arg_value) == NGX_OK) {
return NGX_HTTP_BAD_REQUEST;
}
rc = ngx_http_tfs_raw_fsname_parse(&ctx->file_path_s, &ctx->file_suffix,
&ctx->fsname);
/* large file not support update */
if (rc != NGX_OK
|| (ctx->fsname.file_type == NGX_HTTP_TFS_LARGE_FILE_TYPE))
{
return NGX_HTTP_BAD_REQUEST;
}
ctx->is_raw_update = NGX_HTTP_TFS_YES;
ngx_str_set(&ctx->action.msg, "write_file");
break;
default:
return NGX_HTTP_BAD_REQUEST;
}
return NGX_OK;
}
static void
ngx_http_tfs_dedup_callback(ngx_http_tfs_dedup_ctx_t *ctx, ngx_int_t rc)
{
ngx_http_tfs_t *t;
ngx_http_tfs_rcs_info_t *rc_info;
t = ctx->data;
rc_info = t->rc_info_node;
switch (t->r_ctx.action.code) {
case NGX_HTTP_TFS_ACTION_REMOVE_FILE:
switch(t->state) {
case NGX_HTTP_TFS_STATE_REMOVE_READ_META_SEGMENT:
/* exist in tair */
if (rc == NGX_OK) {
/* check file name */
rc = ngx_http_tfs_dedup_check_filename(&ctx->dup_file_name,
&t->r_ctx.fsname);
if (rc == NGX_OK) {
/* file name match, modify ref count and save tair */
if (t->r_ctx.unlink_type == NGX_HTTP_TFS_UNLINK_DELETE) {
if (--ctx->file_ref_count <= 0) {
/* if ref count is 0,
* remove key from tair then unlink file
*/
t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
t->tfs_peer->body_buffer = ctx->save_body_buffer;
ctx->file_data = t->meta_segment_data;
rc = ngx_http_tfs_dedup_remove(ctx, t->pool,
t->log);
/* do not care delete tair fail,
* go on unlinking file
*/
if (rc == NGX_ERROR) {
ngx_http_tfs_finalize_state(t, NGX_OK);
}
return;
}
/* file_ref_count > 0, just save tair */
t->state = NGX_HTTP_TFS_STATE_REMOVE_DONE;
ctx->file_data = t->meta_segment_data;
rc = ngx_http_tfs_dedup_set(ctx, t->pool, t->log);
/* do not care save tair fail, return success */
if (rc == NGX_ERROR) {
ngx_http_tfs_finalize_state(t, NGX_DONE);
}
return;
}
}
/* file name not match, unlink file */
t->tfs_peer->body_buffer = ctx->save_body_buffer;
t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
ngx_http_tfs_finalize_state(t, NGX_OK);
return;
}
/* not exist in tair, unlink file */
t->tfs_peer->body_buffer = ctx->save_body_buffer;
t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
ngx_http_tfs_finalize_state(t, NGX_OK);
return;
case NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO:
case NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA:
ngx_http_tfs_finalize_state(t, NGX_OK);
return;
case NGX_HTTP_TFS_STATE_REMOVE_DONE:
ngx_http_tfs_finalize_state(t, NGX_DONE);
return;
}
break;
case NGX_HTTP_TFS_ACTION_WRITE_FILE:
switch(t->state) {
case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS:
case NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO:
/* exist in tair */
if (rc == NGX_OK) {
/* check suffix */
rc = ngx_http_tfs_dedup_check_suffix(&ctx->dup_file_name,
&t->r_ctx.file_suffix);
if (rc == NGX_OK) {
/* suffix match, need to stat file */
rc = ngx_http_tfs_raw_fsname_parse(&ctx->dup_file_name,
&ctx->dup_file_suffix,
&t->r_ctx.fsname);
if (rc == NGX_OK) {
t->file.cluster_id = t->r_ctx.fsname.cluster_id;
t->is_stat_dup_file = NGX_HTTP_TFS_YES;
t->state = NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO;
}
} else {
/* suffix not match, need save new tfs file,
* do not save tair
*/
t->use_dedup = NGX_HTTP_TFS_NO;
}
} /* not exist in tair need save new tfs file and tair */
/* need reset meta segment */
rc = ngx_http_tfs_get_meta_segment(t);
if (rc != NGX_OK) {
ngx_log_error(NGX_LOG_ERR, t->log, 0,
"tfs get meta segment failed");
ngx_http_tfs_finalize_request(t->data, t,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
/* lookup block cache */
if (t->is_stat_dup_file) {
/* dedup write may need to stat file */
if (rc_info->use_remote_block_cache) {
rc = ngx_http_tfs_get_remote_block_cache_instance(
&t->block_cache_ctx.remote_ctx,
&rc_info->remote_block_cache_info);
if (rc == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, t->log, 0,
"get remote block cache instance failed.");
} else {
t->block_cache_ctx.use_cache |=
NGX_HTTP_TFS_REMOTE_BLOCK_CACHE;
}
}
ngx_http_tfs_lookup_block_cache(t);
return;
}
ngx_http_tfs_finalize_state(t, NGX_OK);
break;
case NGX_HTTP_TFS_STATE_WRITE_STAT_DUP_FILE:
if (rc == NGX_OK) {
t->state = NGX_HTTP_TFS_STATE_WRITE_DONE;
ngx_http_tfs_finalize_state(t, NGX_DONE);
} else {
/* save tair(add ref count) failed,
* need save new tfs file, do not save tair
*/
t->state = NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS;
t->is_stat_dup_file = NGX_HTTP_TFS_NO;
t->use_dedup = NGX_HTTP_TFS_NO;
/* need reset output buf */
t->out_bufs = NULL;
/* need reset block id and file id */
t->file.segment_data[0].segment_info.block_id = 0;
t->file.segment_data[0].segment_info.file_id = 0;
ngx_http_tfs_finalize_state(t, NGX_OK);
}
break;
case NGX_HTTP_TFS_STATE_WRITE_DONE:
ngx_http_tfs_finalize_state(t, NGX_DONE);
break;
}
}
return;
}
