static ngx_int_t
ngx_conf_add_dump(ngx_conf_t *cf, ngx_str_t *filename)
{
off_t size;
u_char *p;
uint32_t hash;
ngx_buf_t *buf;
ngx_str_node_t *sn;
ngx_conf_dump_t *cd;
hash = ngx_crc32_long(filename->data, filename->len);
sn = ngx_str_rbtree_lookup(&cf->cycle->config_dump_rbtree, filename, hash);
if (sn) {
cf->conf_file->dump = NULL;
return NGX_OK;
}
p = ngx_pstrdup(cf->cycle->pool, filename);
if (p == NULL) {
return NGX_ERROR;
}
cd = ngx_array_push(&cf->cycle->config_dump);
if (cd == NULL) {
return NGX_ERROR;
}
size = ngx_file_size(&cf->conf_file->file.info);
buf = ngx_create_temp_buf(cf->cycle->pool, (size_t) size);
if (buf == NULL) {
return NGX_ERROR;
}
cd->name.data = p;
cd->name.len = filename->len;
cd->buffer = buf;
cf->conf_file->dump = buf;
sn = ngx_palloc(cf->temp_pool, sizeof(ngx_str_node_t));
if (sn == NULL) {
return NGX_ERROR;
}
sn->node.key = hash;
sn->str = cd->name;
ngx_rbtree_insert(&cf->cycle->config_dump_rbtree, &sn->node);
return NGX_OK;
}
/* The md5 hash vaule has better random value with the input string than crc32.
* And the virtual node can be dispersed uniformly in the ring.
* */
static int32_t
ngx_http_upstream_consistent_hash_node_point(u_char *str, size_t len)
{
u_char md5_buf[16];
ngx_md5_t md5;
ngx_md5_init(&md5);
ngx_md5_update(&md5, str, len);
ngx_md5_final(md5_buf, &md5);
return ngx_crc32_long(md5_buf, 16);
}
static int
ngx_http_lua_ngx_crc32_long(lua_State *L)
{
u_char *p;
size_t len;
if (lua_gettop(L) != 1) {
return luaL_error(L, "expecting one argument, but got %d",
lua_gettop(L));
}
p = (u_char *) luaL_checklstring(L, 1, &len);
lua_pushnumber(L, (lua_Number) ngx_crc32_long(p, len));
return 1;
}
ngx_inline ngx_array_t *
yy_sec_waf_re_cache_get_value(ngx_rbtree_t *rbtree, ngx_str_t *name)
{
uint32_t hash;
re_cache_node_t *cache_node;
hash = ngx_crc32_long(name->data, name->len);
cache_node = (re_cache_node_t *) ngx_str_rbtree_lookup(rbtree, name, hash);
if (cache_node != NULL) {
return cache_node->value;
}
return NULL;
}
static ngx_int_t ngx_http_upstream_init_q_chash_peer(ngx_http_request_t *r, ngx_http_upstream_srv_conf_t *us)
{
ngx_int_t rc;
ngx_http_upstream_q_chash_srv_conf_t *uchscf;
ngx_http_upstream_q_chash_peer_data_t *qchp;
ngx_http_upstream_q_chash_ring *q_chash_ring;
ngx_str_t evaluated_key_to_hash;
uchscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_q_chash_module);
if (uchscf == NULL) {
return NGX_ERROR;
}
q_chash_ring = uchscf->q_chash_ring;
qchp = ngx_pcalloc(r->pool, sizeof(*qchp));
if(qchp == NULL)
return NGX_ERROR;
r->upstream->peer.data = &qchp->rrp;
qchp->q_chash_ring = q_chash_ring;
qchp->get_rr_peer = ngx_http_upstream_get_round_robin_peer;
qchp->tries = 0;
qchp->ignore = 0;
qchp->rr_mode = 0;
rc = ngx_http_upstream_init_round_robin_peer(r, us);
if(rc != NGX_OK)
return NGX_ERROR;
r->upstream->peer.get = ngx_http_upstream_get_q_chash_peer;
// calculate the vnode_index
if(q_chash_ring->nr_valid_peers > 1) {
if (ngx_http_script_run(r, &evaluated_key_to_hash, uchscf->lengths->elts, 0, uchscf->values->elts) == NULL)
return NGX_ERROR;
qchp->point = (uint32_t)ngx_crc32_long(evaluated_key_to_hash.data, evaluated_key_to_hash.len);
qchp->vnode_index = q_chash_find(q_chash_ring, qchp->point);
ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "q_chash key %V, point %uD, vnode_index %ui", &evaluated_key_to_hash, qchp->point, qchp->vnode_index);
}
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_init_consistent_hash_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us)
{
ngx_str_t evaluated_key_to_hash;
ngx_http_upstream_consistent_hash_srv_conf_t *uchscf;
ngx_http_upstream_consistent_hash_peer_data_t *uchpd;
uchscf = ngx_http_conf_upstream_srv_conf(us,
ngx_http_upstream_consistent_hash_module);
if (uchscf == NULL) {
return NGX_ERROR;
}
uchpd = ngx_pcalloc(r->pool,
sizeof(ngx_http_upstream_consistent_hash_peer_data_t));
if (uchpd == NULL) {
return NGX_ERROR;
}
r->upstream->peer.data = &uchpd->rrp;
if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) {
return NGX_ERROR;
}
r->upstream->peer.get = ngx_http_upstream_get_consistent_hash_peer;
uchpd->buckets = uchscf->data;
uchpd->tries = 0;
if (ngx_http_script_run(r, &evaluated_key_to_hash,
uchscf->lengths->elts, 0, uchscf->values->elts) == NULL)
{
return NGX_ERROR;
}
uchpd->point =
ngx_crc32_long(evaluated_key_to_hash.data, evaluated_key_to_hash.len);
uchpd->get_rr_peer = ngx_http_upstream_get_round_robin_peer;
return NGX_OK;
}
ngx_int_t
yy_sec_waf_re_cache_set_value(ngx_pool_t *pool,
ngx_str_t *name, ngx_array_t *value, ngx_rbtree_t *rbtree)
{
uint32_t hash;
ngx_str_t *val;
re_cache_node_t *cache_node;
hash = ngx_crc32_long(name->data, name->len);
cache_node = (re_cache_node_t *) ngx_str_rbtree_lookup(rbtree, name, hash);
if (cache_node != NULL) {
return NGX_OK;
}
cache_node = ngx_palloc(pool, sizeof(re_cache_node_t));
if (cache_node == NULL) {
return NGX_ERROR;
}
val = ngx_palloc(pool, sizeof(ngx_str_t));
if (value == NULL) {
return NGX_ERROR;
}
val->len = value->len;
val->data = ngx_pstrdup(pool, value);
if (val->data == NULL) {
return NGX_ERROR;
}
cache_node->sn.node.key = hash;
cache_node->sn.str.len = name->len;
cache_node->sn.str.data = name->data;
cache_node->value = value;
ngx_rbtree_insert(rbtree, &cache_node->sn.node);
return NGX_OK;
}
static ngx_int_t
ngx_http_crc32_hash_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v,
uintptr_t data)
{
ngx_http_hash_ctx_t *ctx = (ngx_http_hash_ctx_t *)data;
ngx_str_t val;
if (ngx_http_complex_value(r, &ctx->value, &val) != NGX_OK) {
return NGX_ERROR;
}
v->len = sizeof("3C32515A") - 1;
if (v->len <= ctx->start) {
return NGX_ERROR;
}
if (v->len < ctx->end) {
return NGX_ERROR;
}
v->data = ngx_palloc(r->pool, v->len);
if (v->data == NULL) {
v->not_found = 1;
return NGX_OK;
}
ngx_sprintf(v->data, "%08XD", ngx_crc32_long(val.data, val.len));
v->valid = 1;
v->not_found = 0;
v->no_cacheable = 0;
if (ctx->end) {
v->data += ctx->start;
v->len = ctx->end;
}
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_init_consistent_hash_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us)
{
ngx_str_t evaluated_key_to_hash;
ngx_http_upstream_consistent_hash_srv_conf_t *uchscf;
ngx_http_upstream_consistent_hash_peer_data_t *uchpd;
uchscf = ngx_http_conf_upstream_srv_conf(us,
ngx_http_upstream_consistent_hash_module);
if (uchscf == NULL) {
return NGX_ERROR;
}
uchpd = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_consistent_hash_peer_data_t));
if (uchpd == NULL) {
return NGX_ERROR;
}
r->upstream->peer.data = uchpd->peers;
uchpd->peers = us->peer.data;
if (ngx_http_script_run(r, &evaluated_key_to_hash,
uchscf->lengths->elts, 0, uchscf->values->elts) == NULL)
{
return NGX_ERROR;
}
uchpd->point =
ngx_crc32_long(evaluated_key_to_hash.data, evaluated_key_to_hash.len);
r->upstream->peer.free = ngx_http_upstream_free_consistent_hash_peer;
r->upstream->peer.get = ngx_http_upstream_get_consistent_hash_peer;
r->upstream->peer.data = uchpd;
return NGX_OK;
}
ngx_int_t
ngx_open_cached_file(ngx_open_file_cache_t *cache, ngx_str_t *name,
ngx_open_file_info_t *of, ngx_pool_t *pool)
{
time_t now;
uint32_t hash;
ngx_int_t rc;
ngx_file_info_t fi;
ngx_pool_cleanup_t *cln;
ngx_cached_open_file_t *file;
ngx_pool_cleanup_file_t *clnf;
ngx_open_file_cache_cleanup_t *ofcln;
of->fd = NGX_INVALID_FILE;
of->err = 0;
if (cache == NULL) {
if (of->test_only) {
if (ngx_file_info_wrapper(name, of, &fi, pool->log)
== NGX_FILE_ERROR)
{
return NGX_ERROR;
}
of->uniq = ngx_file_uniq(&fi);
of->mtime = ngx_file_mtime(&fi);
of->size = ngx_file_size(&fi);
of->fs_size = ngx_file_fs_size(&fi);
of->is_dir = ngx_is_dir(&fi);
of->is_file = ngx_is_file(&fi);
of->is_link = ngx_is_link(&fi);
of->is_exec = ngx_is_exec(&fi);
return NGX_OK;
}
cln = ngx_pool_cleanup_add(pool, sizeof(ngx_pool_cleanup_file_t));
if (cln == NULL) {
return NGX_ERROR;
}
rc = ngx_open_and_stat_file(name, of, pool->log);
if (rc == NGX_OK && !of->is_dir) {
cln->handler = ngx_pool_cleanup_file;
clnf = cln->data;
clnf->fd = of->fd;
clnf->name = name->data;
clnf->log = pool->log;
}
return rc;
}
cln = ngx_pool_cleanup_add(pool, sizeof(ngx_open_file_cache_cleanup_t));
if (cln == NULL) {
return NGX_ERROR;
}
now = ngx_time();
// 计算hash
hash = ngx_crc32_long(name->data, name->len);
// 根据名字查找对应的file对象
file = ngx_open_file_lookup(cache, name, hash);
if (file) {
file->uses++;
ngx_queue_remove(&file->queue);
if (file->fd == NGX_INVALID_FILE && file->err == 0 && !file->is_dir) {
/* file was not used often enough to keep open */
rc = ngx_open_and_stat_file(name, of, pool->log);
if (rc != NGX_OK && (of->err == 0 || !of->errors)) {
goto failed;
}
goto add_event;
}
if (file->use_event
|| (file->event == NULL
&& (of->uniq == 0 || of->uniq == file->uniq)
&& now - file->created < of->valid
#if (NGX_HAVE_OPENAT)
&& of->disable_symlinks == file->disable_symlinks
&& of->disable_symlinks_from == file->disable_symlinks_from
#endif
))
{
if (file->err == 0) {
of->fd = file->fd;
of->uniq = file->uniq;
of->mtime = file->mtime;
of->size = file->size;
of->is_dir = file->is_dir;
of->is_file = file->is_file;
of->is_link = file->is_link;
of->is_exec = file->is_exec;
of->is_directio = file->is_directio;
if (!file->is_dir) {
file->count++;
ngx_open_file_add_event(cache, file, of, pool->log);
}
} else {
of->err = file->err;
#if (NGX_HAVE_OPENAT)
of->failed = file->disable_symlinks ? ngx_openat_file_n
: ngx_open_file_n;
#else
of->failed = ngx_open_file_n;
#endif
}
goto found;
}
ngx_log_debug4(NGX_LOG_DEBUG_CORE, pool->log, 0,
"retest open file: %s, fd:%d, c:%d, e:%d",
file->name, file->fd, file->count, file->err);
if (file->is_dir) {
/*
* chances that directory became file are very small
* so test_dir flag allows to use a single syscall
* in ngx_file_info() instead of three syscalls
*/
of->test_dir = 1;
}
of->fd = file->fd;
of->uniq = file->uniq;
// 打开文件，保存文件信息
rc = ngx_open_and_stat_file(name, of, pool->log);
if (rc != NGX_OK && (of->err == 0 || !of->errors)) {
goto failed;
}
if (of->is_dir) {
if (file->is_dir || file->err) {
goto update;
}
/* file became directory */
} else if (of->err == 0) { /* file */
if (file->is_dir || file->err) {
goto add_event;
}
if (of->uniq == file->uniq) {
if (file->event) {
file->use_event = 1;
}
of->is_directio = file->is_directio;
goto update;
}
/* file was changed */
} else { /* error to cache */
if (file->err || file->is_dir) {
goto update;
}
/* file was removed, etc. */
}
if (file->count == 0) {
ngx_open_file_del_event(file);
if (ngx_close_file(file->fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
ngx_close_file_n " \"%V\" failed", name);
}
goto add_event;
}
ngx_rbtree_delete(&cache->rbtree, &file->node);
cache->current--;
file->close = 1;
goto create;
}
/* not found */
rc = ngx_open_and_stat_file(name, of, pool->log);
if (rc != NGX_OK && (of->err == 0 || !of->errors)) {
goto failed;
}
create:
// max为open_file_cache命令中定义的那个max指令，
// 而current也就是当前cache的文件个数
if (cache->current >= cache->max) {
// 如果大于max，则需要强制expire几个元素
ngx_expire_old_cached_files(cache, 0, pool->log);
}
file = ngx_alloc(sizeof(ngx_cached_open_file_t), pool->log);
if (file == NULL) {
goto failed;
}
file->name = ngx_alloc(name->len + 1, pool->log);
if (file->name == NULL) {
ngx_free(file);
file = NULL;
goto failed;
}
ngx_cpystrn(file->name, name->data, name->len + 1);
file->node.key = hash;
ngx_rbtree_insert(&cache->rbtree, &file->node);
cache->current++;
file->uses = 1;
file->count = 0;
file->use_event = 0;
file->event = NULL;
add_event:
ngx_open_file_add_event(cache, file, of, pool->log);
update:
file->fd = of->fd;
file->err = of->err;
#if (NGX_HAVE_OPENAT)
file->disable_symlinks = of->disable_symlinks;
file->disable_symlinks_from = of->disable_symlinks_from;
#endif
if (of->err == 0) {
file->uniq = of->uniq;
file->mtime = of->mtime;
file->size = of->size;
file->close = 0;
file->is_dir = of->is_dir;
file->is_file = of->is_file;
file->is_link = of->is_link;
file->is_exec = of->is_exec;
file->is_directio = of->is_directio;
if (!of->is_dir) {
file->count++;
}
}
file->created = now;
found:
// 更新存取时间
file->accessed = now;
// 将文件插入到超时队列中
ngx_queue_insert_head(&cache->expire_queue, &file->queue);
ngx_log_debug5(NGX_LOG_DEBUG_CORE, pool->log, 0,
"cached open file: %s, fd:%d, c:%d, e:%d, u:%d",
file->name, file->fd, file->count, file->err, file->uses);
if (of->err == 0) {
if (!of->is_dir) {
// 这里很关键，将cln的handler
cln->handler = ngx_open_file_cleanup;
ofcln = cln->data;
ofcln->cache = cache;
ofcln->file = file;
ofcln->min_uses = of->min_uses;
ofcln->log = pool->log;
}
return NGX_OK;
}
return NGX_ERROR;
failed:
if (file) {
ngx_rbtree_delete(&cache->rbtree, &file->node);
cache->current--;
if (file->count == 0) {
if (file->fd != NGX_INVALID_FILE) {
if (ngx_close_file(file->fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
ngx_close_file_n " \"%s\" failed",
file->name);
}
}
ngx_free(file->name);
ngx_free(file);
} else {
file->close = 1;
}
}
if (of->fd != NGX_INVALID_FILE) {
if (ngx_close_file(of->fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
ngx_close_file_n " \"%V\" failed", name);
}
}
return NGX_ERROR;
}
ngx_int_t
ngx_http_upstream_init_consistent_hash(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us)
{
/* ip max 15, :port max 6, maxweight is highest number of uchar */
u_char hash_data[HASH_DATA_LENGTH];
uint32_t step;
ngx_uint_t i, j, k, n, points = 0;
ngx_http_upstream_server_t *server;
ngx_http_upstream_consistent_hash_buckets *buckets;
ngx_http_upstream_consistent_hash_continuum *continuum;
for (i=0;i<HASH_DATA_LENGTH;i++) hash_data[i] = 0;
step = (uint32_t) (0xffffffff / MMC_CONSISTENT_BUCKETS);
buckets = ngx_pcalloc(cf->pool,
sizeof(ngx_http_upstream_consistent_hash_buckets));
us->peer.init = ngx_http_upstream_init_consistent_hash_peer;
if (!us->servers) {
return NGX_ERROR;
}
server = us->servers->elts;
for (n = 0, i = 0; i < us->servers->nelts; i++) {
n += server[i].naddrs;
points += server[i].weight * server[i].naddrs * MMC_CONSISTENT_POINTS;
}
continuum = ngx_pcalloc(cf->pool,
sizeof(ngx_http_upstream_consistent_hash_continuum));
continuum->nodes = ngx_pcalloc(cf->pool,
sizeof(ngx_http_upstream_consistent_hash_node) * points);
for (i = 0; i < us->servers->nelts; i++) {
for (j = 0; j < server[i].naddrs; j++) {
for (k = 0; k < ((MMC_CONSISTENT_POINTS * server[i].weight) / server[i].naddrs); k++) {
ngx_snprintf(hash_data, 28, "%V-%ui", &server[i].addrs[j].name, k);
continuum->nodes[continuum->nnodes].sockaddr = server[i].addrs[j].sockaddr;
continuum->nodes[continuum->nnodes].socklen = server[i].addrs[j].socklen;
continuum->nodes[continuum->nnodes].name = server[i].addrs[j].name;
continuum->nodes[continuum->nnodes].name.data[server[i].addrs[j].name.len] = 0;
continuum->nodes[continuum->nnodes].point = ngx_crc32_long(hash_data, ngx_strlen(hash_data));
continuum->nnodes++;
}
}
}
qsort(continuum->nodes, continuum->nnodes,
sizeof(ngx_http_upstream_consistent_hash_node),
(const void*) ngx_http_upstream_consistent_hash_compare_continuum_nodes);
for (i = 0; i < MMC_CONSISTENT_BUCKETS; i++) {
buckets->buckets[i] =
ngx_http_upstream_consistent_hash_find(continuum, step * i);
}
#if (CONSISTENT_DEBUG)
ngx_http_upstream_consistent_hash_print_continuum(cf, continuum);
ngx_http_upstream_consistent_hash_print_buckets(cf, buckets);
#endif
buckets->continuum = continuum;
us->peer.data = buckets;
return NGX_OK;
}
ngx_int_t
ngx_open_cached_file(ngx_open_file_cache_t *cache, ngx_str_t *name,
ngx_open_file_info_t *of, ngx_pool_t *pool)
{
time_t now;
uint32_t hash;
ngx_int_t rc;
ngx_pool_cleanup_t *cln;
ngx_cached_open_file_t *file;
ngx_pool_cleanup_file_t *clnf;
ngx_open_file_cache_cleanup_t *ofcln;
of->err = 0;
if (cache == NULL) {
cln = ngx_pool_cleanup_add(pool, sizeof(ngx_pool_cleanup_file_t));
if (cln == NULL) {
return NGX_ERROR;
}
rc = ngx_open_and_stat_file(name->data, of, pool->log);
if (rc == NGX_OK && !of->is_dir) {
cln->handler = ngx_pool_cleanup_file;
clnf = cln->data;
clnf->fd = of->fd;
clnf->name = name->data;
clnf->log = pool->log;
}
return rc;
}
cln = ngx_pool_cleanup_add(pool, sizeof(ngx_open_file_cache_cleanup_t));
if (cln == NULL) {
return NGX_ERROR;
}
now = ngx_time();
hash = ngx_crc32_long(name->data, name->len);
file = ngx_open_file_lookup(cache, name, hash);
if (file) {
file->uses++;
ngx_queue_remove(&file->queue);
if (file->fd == NGX_INVALID_FILE && file->err == 0 && !file->is_dir) {
/* file was not used often enough to keep open */
rc = ngx_open_and_stat_file(name->data, of, pool->log);
if (rc != NGX_OK && (of->err == 0 || !of->errors)) {
goto failed;
}
goto add_event;
}
if ((file->event && file->use_event)
|| (file->event == NULL && now - file->created < of->valid))
{
if (file->err == 0) {
of->fd = file->fd;
of->uniq = file->uniq;
of->mtime = file->mtime;
of->size = file->size;
of->is_dir = file->is_dir;
of->is_file = file->is_file;
of->is_link = file->is_link;
of->is_exec = file->is_exec;
if (!file->is_dir) {
file->count++;
ngx_open_file_add_event(cache, file, of, pool->log);
}
} else {
of->err = file->err;
}
goto found;
}
ngx_log_debug4(NGX_LOG_DEBUG_CORE, pool->log, 0,
"retest open file: %s, fd:%d, c:%d, e:%d",
file->name, file->fd, file->count, file->err);
if (file->is_dir) {
/*
* chances that directory became file are very small
* so test_dir flag allows to use a single syscall
* in ngx_file_info() instead of three syscalls
*/
of->test_dir = 1;
}
rc = ngx_open_and_stat_file(name->data, of, pool->log);
if (rc != NGX_OK && (of->err == 0 || !of->errors)) {
goto failed;
}
if (of->is_dir) {
if (file->is_dir || file->err) {
goto update;
}
/* file became directory */
} else if (of->err == 0) { /* file */
if (file->is_dir || file->err) {
goto add_event;
}
if (of->uniq == file->uniq
&& of->mtime == file->mtime
&& of->size == file->size)
{
if (ngx_close_file(of->fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
ngx_close_file_n " \"%s\" failed",
name->data);
}
of->fd = file->fd;
file->count++;
if (file->event) {
file->use_event = 1;
goto renew;
}
ngx_open_file_add_event(cache, file, of, pool->log);
goto renew;
}
/* file was changed */
} else { /* error to cache */
if (file->err || file->is_dir) {
goto update;
}
/* file was removed, etc. */
}
if (file->count == 0) {
ngx_open_file_del_event(file);
if (ngx_close_file(file->fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
ngx_close_file_n " \"%s\" failed",
name->data);
}
goto add_event;
}
ngx_rbtree_delete(&cache->rbtree, &file->node);
cache->current--;
file->close = 1;
goto create;
}
/* not found */
rc = ngx_open_and_stat_file(name->data, of, pool->log);
if (rc != NGX_OK && (of->err == 0 || !of->errors)) {
goto failed;
}
create:
if (cache->current >= cache->max) {
ngx_expire_old_cached_files(cache, 0, pool->log);
}
file = ngx_alloc(sizeof(ngx_cached_open_file_t), pool->log);
if (file == NULL) {
goto failed;
}
file->name = ngx_alloc(name->len + 1, pool->log);
if (file->name == NULL) {
ngx_free(file);
file = NULL;
goto failed;
}
ngx_cpystrn(file->name, name->data, name->len + 1);
file->node.key = hash;
ngx_rbtree_insert(&cache->rbtree, &file->node);
cache->current++;
file->uses = 1;
file->count = 0;
file->use_event = 0;
file->event = NULL;
add_event:
ngx_open_file_add_event(cache, file, of, pool->log);
update:
file->fd = of->fd;
file->err = of->err;
if (of->err == 0) {
file->uniq = of->uniq;
file->mtime = of->mtime;
file->size = of->size;
file->close = 0;
file->is_dir = of->is_dir;
file->is_file = of->is_file;
file->is_link = of->is_link;
file->is_exec = of->is_exec;
if (!of->is_dir) {
file->count++;
}
}
renew:
file->created = now;
found:
file->accessed = now;
ngx_queue_insert_head(&cache->expire_queue, &file->queue);
ngx_log_debug5(NGX_LOG_DEBUG_CORE, pool->log, 0,
"cached open file: %s, fd:%d, c:%d, e:%d, u:%d",
file->name, file->fd, file->count, file->err, file->uses);
if (of->err == 0) {
if (!of->is_dir) {
cln->handler = ngx_open_file_cleanup;
ofcln = cln->data;
ofcln->cache = cache;
ofcln->file = file;
ofcln->min_uses = of->min_uses;
ofcln->log = pool->log;
}
return NGX_OK;
}
return NGX_ERROR;
failed:
if (file) {
ngx_rbtree_delete(&cache->rbtree, &file->node);
cache->current--;
if (file->count == 0) {
if (file->fd != NGX_INVALID_FILE) {
if (ngx_close_file(file->fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
ngx_close_file_n " \"%s\" failed",
file->name);
}
}
ngx_free(file->name);
ngx_free(file);
} else {
file->close = 1;
}
}
if (of->fd != NGX_INVALID_FILE) {
if (ngx_close_file(of->fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, pool->log, ngx_errno,
ngx_close_file_n " \"%s\" failed", name->data);
}
}
return NGX_ERROR;
}
// make our proposed ZIP-file chunk map
ngx_int_t
ngx_http_zip_generate_pieces(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx)
{
ngx_uint_t i, piece_i;
off_t offset = 0;
time_t unix_time = 0;
ngx_uint_t dos_time = 0;
ngx_http_zip_file_t *file;
ngx_http_zip_piece_t *header_piece, *file_piece, *trailer_piece, *cd_piece;
ngx_http_variable_value_t *vv;
if ((vv = ngx_palloc(r->pool, sizeof(ngx_http_variable_value_t))) == NULL)
return NGX_ERROR;
ctx->unicode_path = 0;
#ifdef NGX_ZIP_HAVE_ICONV
iconv_t *iconv_cd = NULL;
#endif
// Let's try to find special header that contains separator string.
// What for this strange separator string you ask?
// Sometimes there might be a problem converting UTF-8 to zips native
// charset(CP866), because it's not 1:1 conversion. So my solution is to
// developers provide their own version of converted filename and pass it
// to mod_zip along with UTF-8 filename which will go straight to Unicode
// path extra field (thanks to tony2001). So separator is a solution that doesn't
// break current format. And allows passing file name in both formats as one string.
//
// Normally we pass:
// CRC32 <size> <path> <filename>\n
// ...
// * <filename> passed to archive as filename w/o conversion
// * UFT-8 flag for filename is set
//
// tony2001's X-Archive-Charset: <charset> way:
// CRC32 <size> <path> <filename>\n
// ...
// * <filename> is accepted to be UTF-8 string
// * <filename>, converted to <charset> and passed to archive as filename
// * <filename> passed to Unicode path extra field
// * UFT-8 flag for filename is not set
//
// My X-Archive-Name-Sep: <sep> solution:
// CRC32 <size> <path> <native-filename><sep><utf8-filename>\n
// ...
// * <native-filename> passed to archive as filename w/o conversion
// * <utf8-filename> passed to Unicode path extra field
// * UFT-8 flag for filename is not set
//
// You just need to provide separator that won't interfere with file names. I suggest using '/'
// as it is ASCII character and forbidden on most (if not all) platforms as a part of filename.
//
// Empty separator string means no UTF-8 version provided. Usefull when we need to pass only
// names encoded in native charset. It's equal to 'X-Archive-Charset: native;'.
// Note: Currently it is impossible after '[PATCH] Support for UTF-8 file names.'(4f61592b)
// because UFT-8 flag (zip_utf8_flag) is set default for templates.
if(ngx_http_upstream_header_variable(r, vv, (uintptr_t)(&ngx_http_zip_header_name_separator)) == NGX_OK && !vv->not_found) {
ctx->native_charset = 1;
if(vv->len)
ctx->unicode_path = 1;
} else {
#ifdef NGX_ZIP_HAVE_ICONV
if (ngx_http_upstream_header_variable(r, vv, (uintptr_t)(&ngx_http_zip_header_charset_name)) == NGX_OK
&& !vv->not_found && ngx_strncmp(vv->data, "utf8", sizeof("utf8") - 1) != 0) {
if(ngx_strncmp(vv->data, "native", sizeof("native") - 1))
{
char encoding[ICONV_CSNMAXLEN];
snprintf(encoding, sizeof(encoding), "%s//TRANSLIT//IGNORE", vv->data);
iconv_cd = iconv_open((const char *)encoding, "utf-8");
if (iconv_cd == (iconv_t)(-1)) {
ngx_log_error(NGX_LOG_WARN, r->connection->log, errno,
"mod_zip: iconv_open('%s', 'utf-8') failed",
vv->data);
iconv_cd = NULL;
}
else
{
ctx->unicode_path = 1;
ctx->native_charset = 1;
}
}
else
ctx->native_charset = 1;
}
#endif
}
// pieces: for each file: header, data, footer (if needed) -> 2 or 3 per file
// plus file footer (CD + [zip64 end + zip64 locator +] end of cd) in one chunk
ctx->pieces_n = ctx->files.nelts * (2 + (!!ctx->missing_crc32)) + 1;
if ((ctx->pieces = ngx_palloc(r->pool, sizeof(ngx_http_zip_piece_t) * ctx->pieces_n)) == NULL)
return NGX_ERROR;
ctx->cd_size = 0;
unix_time = time(NULL);
dos_time = ngx_dos_time(unix_time);
for (piece_i = i = 0; i < ctx->files.nelts; i++) {
file = &((ngx_http_zip_file_t *)ctx->files.elts)[i];
file->offset = offset;
file->unix_time = unix_time;
file->dos_time = dos_time;
if(ctx->unicode_path) {
#ifdef NGX_ZIP_HAVE_ICONV
if (iconv_cd) {
size_t inlen = file->filename.len, outlen, outleft;
u_char *p, *in;
//inbuf
file->filename_utf8.data = ngx_pnalloc(r->pool, file->filename.len + 1);
ngx_memcpy(file->filename_utf8.data, file->filename.data, file->filename.len);
file->filename_utf8.len = file->filename.len;
file->filename_utf8.data[file->filename.len] = '\0';
//outbuf
outlen = outleft = inlen * sizeof(int) + 15;
file->filename.data = ngx_pnalloc(r->pool, outlen + 1);
in = file->filename_utf8.data;
p = file->filename.data;
//reset state
iconv(iconv_cd, NULL, NULL, NULL, NULL);
//convert the string
iconv(iconv_cd, (char **)&in, &inlen, (char **)&p, &outleft);
//XXX if (res == (size_t)-1) { ? }
file->filename.len = outlen - outleft;
file->filename_utf8_crc32 = ngx_crc32_long(file->filename_utf8.data, file->filename_utf8.len);
}
#endif
else if(vv->len) {
const char * sep = ngx_http_zip_strnrstr((const char*)file->filename.data, file->filename.len,
(const char*)vv->data, vv->len);
if(sep) {
size_t utf8_len = file->filename.len - vv->len - (size_t)(sep - (const char *)file->filename.data);
file->filename_utf8.data = ngx_pnalloc(r->pool, utf8_len);
file->filename_utf8.len = utf8_len;
ngx_memcpy(file->filename_utf8.data, sep + vv->len, utf8_len);
file->filename.len -= utf8_len + vv->len;
file->filename_utf8_crc32 = ngx_crc32_long(file->filename_utf8.data, file->filename_utf8.len);
} /* else { } */ // Separator not found. Okay, no extra field for this one then.
}
}
if(offset >= (off_t) NGX_MAX_UINT32_VALUE)
ctx->zip64_used = file->need_zip64_offset = 1;
if(file->size >= (off_t) NGX_MAX_UINT32_VALUE)
ctx->zip64_used = file->need_zip64 = 1;
ctx->cd_size += sizeof(ngx_zip_central_directory_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_central_t)
+ (file->need_zip64_offset ?
(file->need_zip64 ? sizeof(ngx_zip_extra_field_zip64_sizes_offset_t) : sizeof(ngx_zip_extra_field_zip64_offset_only_t)) :
(file->need_zip64 ? sizeof(ngx_zip_extra_field_zip64_sizes_only_t) : 0) +
(ctx->unicode_path && file->filename_utf8.len ? (sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len): 0)
);
header_piece = &ctx->pieces[piece_i++];
header_piece->type = zip_header_piece;
header_piece->file = file;
header_piece->range.start = offset;
header_piece->range.end = offset += sizeof(ngx_zip_local_file_header_t)
+ file->filename.len + sizeof(ngx_zip_extra_field_local_t) + (file->need_zip64? sizeof(ngx_zip_extra_field_zip64_sizes_only_t):0)
+ (ctx->unicode_path && file->filename_utf8.len ? (sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len): 0);
file_piece = &ctx->pieces[piece_i++];
file_piece->type = zip_file_piece;
file_piece->file = file;
file_piece->range.start = offset;
file_piece->range.end = offset += file->size; //!note: (sizeless chunks): we need file size here / or mark it and modify ranges after
if (file->missing_crc32) { // if incomplete header -> add footer with that info to file
trailer_piece = &ctx->pieces[piece_i++];
trailer_piece->type = zip_trailer_piece;
trailer_piece->file = file;
trailer_piece->range.start = offset;
trailer_piece->range.end = offset += file->need_zip64? sizeof(ngx_zip_data_descriptor_zip64_t) : sizeof(ngx_zip_data_descriptor_t);
//!!TODO: if we want Ranges support - here we know it is impossible for this set
//? check conf/some state and abort?
}
}
#ifdef NGX_ZIP_HAVE_ICONV
if (iconv_cd) {
iconv_close(iconv_cd);
}
#endif
ctx->zip64_used |= offset >= (off_t) NGX_MAX_UINT32_VALUE || ctx->files.nelts >= NGX_MAX_UINT16_VALUE;
ctx->cd_size += sizeof(ngx_zip_end_of_central_directory_record_t);
if (ctx->zip64_used)
ctx->cd_size += sizeof(ngx_zip_zip64_end_of_central_directory_record_t) + sizeof(ngx_zip_zip64_end_of_central_directory_locator_t);
cd_piece = &ctx->pieces[piece_i++];
cd_piece->type = zip_central_directory_piece;
cd_piece->range.start = offset;
cd_piece->range.end = offset += ctx->cd_size;
ctx->pieces_n = piece_i; //!! nasty hack (truncating allocated array without reallocation)
ctx->archive_size = offset;
return NGX_OK;
}
static ngx_int_t
ngx_http_kafka_lua_push(lua_State *L)
{
ngx_http_request_t *r;
ngx_http_kafka_main_conf_t *kmcf;
ngx_http_kafka_ctx_t *ctx;
ngx_kfk_toppar_t *toppar;
ngx_kfk_buf_t *buf;
ngx_str_t msg;
ngx_int_t rc;
size_t size, len;
int32_t *crc;
u_char *p, *q;
const char *err;
int i, n, type;
n = lua_gettop(L);
size = 0;
for (i = 3; i <= n; i++) {
type = lua_type(L, i);
switch (type) {
case LUA_TNUMBER:
case LUA_TSTRING:
lua_tolstring(L, i, &len);
size += len;
break;
case LUA_TNIL:
size += sizeof("nil") - 1;
break;
case LUA_TBOOLEAN:
if (lua_toboolean(L, i)) {
size += sizeof("true") - 1;
} else {
size += sizeof("false") - 1;
}
break;
case LUA_TLIGHTUSERDATA:
if (lua_touserdata(L, i) == NULL) {
size += sizeof("null") - 1;
break;
}
continue;
default:
err = lua_pushfstring(L, "string, number, boolean, or nil "
"expected, got %s", lua_typename(L, type));
return luaL_argerror(L, i, err);
}
}
r = ngx_http_lua_get_request(L);
kmcf = ngx_http_get_module_main_conf(r, ngx_http_kafka_module);
if (size > kmcf->msg_max_size) {
lua_pushnil(L);
lua_pushfstring(L, "message is too long, at most %d bytes",
(int)(kmcf->msg_max_size));
return 2;
}
ctx = ngx_http_get_module_ctx(r, ngx_http_kafka_module);
toppar = ctx->toppar;
buf = toppar->free;
if (buf == NULL) {
buf = ngx_http_kafka_get_buf(ngx_kfk, &ngx_kfk->free);
if (buf == NULL) {
lua_pushnil(L);
lua_pushliteral(L, "no bufs");
return 2;
}
if (buf->no_pool) {
if (ngx_http_kafka_init_buf_pool(ngx_kfk, buf) != NGX_OK) {
lua_pushnil(L);
lua_pushliteral(L, "no enough memory");
return 2;
}
}
toppar->free = buf;
}
p = ngx_pnalloc(buf->pool, NGX_KFK_MSG_HEADER_SIZE + size);
if (p == NULL) {
lua_pushnil(L);
lua_pushliteral(L, "no enough memory");
return 2;
}
msg.data = p;
msg.len = NGX_KFK_MSG_HEADER_SIZE + size;
/* offset */
*(int64_t *)p = 0;
p += sizeof(int64_t);
/* msg size */
*(int32_t *)p = (int32_t)htonl(14 + len);
p += sizeof(int32_t);
/* crc(later calculate) */
crc = (int32_t *)p;
p += sizeof(int32_t);
/* magic */
*(int8_t *)p = 0;
p += sizeof(int8_t);
/* attr */
*(int8_t *)p = 0;
p += sizeof(int8_t);
/* key len */
*(int32_t *)p = (int32_t)htonl(-1);
p += sizeof(int32_t);
/* value len */
*(int32_t *)p = (int32_t)htonl(len);
p += sizeof(int32_t);
for (i = 3; i <= n; i++) {
type = lua_type(L, i);
switch (type) {
case LUA_TNUMBER:
case LUA_TSTRING:
q = (u_char *) lua_tolstring(L, i, &len);
p = ngx_copy(p, q, len);
break;
case LUA_TNIL:
*p++ = 'n';
*p++ = 'i';
*p++ = 'l';
break;
case LUA_TBOOLEAN:
if (lua_toboolean(L, i)) {
*p++ = 't';
*p++ = 'r';
*p++ = 'u';
*p++ = 'e';
} else {
*p++ = 'f';
*p++ = 'a';
*p++ = 'l';
*p++ = 's';
*p++ = 'e';
}
break;
case LUA_TLIGHTUSERDATA:
*p++ = 'n';
*p++ = 'u';
*p++ = 'l';
*p++ = 'l';
break;
default:
ngx_http_kafka_recycle_msg(buf->pool, &msg);
return luaL_error(L, "impossible to reach here");
}
}
if (p - msg.data > (off_t)(NGX_KFK_MSG_HEADER_SIZE + size)) {
ngx_http_kafka_recycle_msg(buf->pool, &msg);
return luaL_error(L, "buffer error: %d > %d", (int) (p - msg.data),
(int) size);
}
*crc = (int32_t)htonl(ngx_crc32_long((u_char*)(crc + 1), 10 + size));
if (buf->cnt == -1) {
if (ngx_http_kafka_init_chain(buf, ctx) != NGX_OK) {
ngx_http_kafka_recycle_msg(buf->pool, &msg);
lua_pushnil(L);
lua_pushliteral(L, "no enough memory");
return 2;
}
}
ctx->msg = &msg;
rc = ngx_http_kafka_enq_buf(r, ctx);
if (rc != NGX_OK) {
lua_pushnil(L);
lua_pushliteral(L, "internal error");
return 2;
}
lua_pushinteger(L, 1);
return 1;
}
// make our proposed ZIP-file chunk map
ngx_int_t
ngx_http_zip_generate_pieces(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx)
{
ngx_uint_t i, piece_i;
off_t offset = 0;
ngx_http_zip_file_t *file;
ngx_http_zip_piece_t *header_piece, *file_piece, *trailer_piece, *cd_piece;
ngx_http_variable_value_t *vv;
if ((vv = ngx_palloc(r->pool, sizeof(ngx_http_variable_value_t))) == NULL)
return NGX_ERROR;
ctx->unicode_path = 0;
#ifdef NGX_ZIP_HAVE_ICONV
iconv_t *iconv_cd = NULL;
if (ngx_http_upstream_header_variable(r, vv, (uintptr_t)(&ngx_http_zip_header_charset_name)) == NGX_OK
&& !vv->not_found && ngx_strncmp(vv->data, "utf8", sizeof("utf8") - 1) != 0) {
char encoding[ICONV_CSNMAXLEN];
snprintf(encoding, sizeof(encoding), "%s//TRANSLIT//IGNORE", vv->data);
iconv_cd = iconv_open((const char *)encoding, "utf-8");
if (iconv_cd == (iconv_t)(-1)) {
ngx_log_error(NGX_LOG_WARN, r->connection->log, errno,
"mod_zip: iconv_open('%s', 'utf-8') failed",
vv->data);
iconv_cd = NULL;
}
}
if (iconv_cd) {
ctx->unicode_path = 1;
}
#endif
// pieces: for each file: header, data, footer (if needed) -> 2 or 3 per file
// plus file footer (CD + [zip64 end + zip64 locator +] end of cd) in one chunk
ctx->pieces_n = ctx->files.nelts * (2 + (!!ctx->missing_crc32)) + 1;
if ((ctx->pieces = ngx_palloc(r->pool, sizeof(ngx_http_zip_piece_t) * ctx->pieces_n)) == NULL)
return NGX_ERROR;
ctx->cd_size = 0;
for (piece_i = i = 0; i < ctx->files.nelts; i++) {
file = &((ngx_http_zip_file_t *)ctx->files.elts)[i];
file->offset = offset;
#ifdef NGX_ZIP_HAVE_ICONV
if (ctx->unicode_path) {
size_t inlen = file->filename.len, outlen, outleft;
u_char *p, *in;
//inbuf
file->filename_utf8.data = ngx_pnalloc(r->pool, file->filename.len + 1);
ngx_memcpy(file->filename_utf8.data, file->filename.data, file->filename.len);
file->filename_utf8.len = file->filename.len;
file->filename_utf8.data[file->filename.len] = '\0';
//outbuf
outlen = outleft = inlen * sizeof(int) + 15;
file->filename.data = ngx_pnalloc(r->pool, outlen + 1);
in = file->filename_utf8.data;
p = file->filename.data;
//reset state
iconv(iconv_cd, NULL, NULL, NULL, NULL);
//convert the string
iconv(iconv_cd, (char **)&in, &inlen, (char **)&p, &outleft);
//XXX if (res == (size_t)-1) { ? }
file->filename.len = outlen - outleft;
file->filename_utf8_crc32 = ngx_crc32_long(file->filename_utf8.data, file->filename_utf8.len);
}
#endif
if(offset >= (off_t) NGX_MAX_UINT32_VALUE)
ctx->zip64_used = file->need_zip64_offset = 1;
if(file->size >= (off_t) NGX_MAX_UINT32_VALUE)
ctx->zip64_used = file->need_zip64 = 1;
ctx->cd_size += sizeof(ngx_zip_central_directory_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_central_t)
+ (file->need_zip64_offset ?
(file->need_zip64 ? sizeof(ngx_zip_extra_field_zip64_sizes_offset_t) : sizeof(ngx_zip_extra_field_zip64_offset_only_t)) :
(file->need_zip64 ? sizeof(ngx_zip_extra_field_zip64_sizes_only_t) : 0) +
(ctx->unicode_path ? (sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len): 0)
);
header_piece = &ctx->pieces[piece_i++];
header_piece->type = zip_header_piece;
header_piece->file = file;
header_piece->range.start = offset;
header_piece->range.end = offset += sizeof(ngx_zip_local_file_header_t)
+ file->filename.len + sizeof(ngx_zip_extra_field_local_t) + (file->need_zip64? sizeof(ngx_zip_extra_field_zip64_sizes_only_t):0)
+ (ctx->unicode_path ? (sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len): 0);
file_piece = &ctx->pieces[piece_i++];
file_piece->type = zip_file_piece;
file_piece->file = file;
file_piece->range.start = offset;
file_piece->range.end = offset += file->size; //!note: (sizeless chunks): we need file size here / or mark it and modify ranges after
if (file->missing_crc32) { // if incomplete header -> add footer with that info to file
trailer_piece = &ctx->pieces[piece_i++];
trailer_piece->type = zip_trailer_piece;
trailer_piece->file = file;
trailer_piece->range.start = offset;
trailer_piece->range.end = offset += file->need_zip64? sizeof(ngx_zip_data_descriptor_zip64_t) : sizeof(ngx_zip_data_descriptor_t);
//!!TODO: if we want Ranges support - here we know it is impossible for this set
//? check conf/some state and abort?
}
}
#ifdef NGX_ZIP_HAVE_ICONV
if (ctx->unicode_path) {
iconv_close(iconv_cd);
}
#endif
ctx->zip64_used |= offset >= (off_t) NGX_MAX_UINT32_VALUE || ctx->files.nelts >= NGX_MAX_UINT16_VALUE;
ctx->cd_size += sizeof(ngx_zip_end_of_central_directory_record_t);
if (ctx->zip64_used)
ctx->cd_size += sizeof(ngx_zip_zip64_end_of_central_directory_record_t) + sizeof(ngx_zip_zip64_end_of_central_directory_locator_t);
cd_piece = &ctx->pieces[piece_i++];
cd_piece->type = zip_central_directory_piece;
cd_piece->range.start = offset;
cd_piece->range.end = offset += ctx->cd_size;
ctx->pieces_n = piece_i; //!! nasty hack (truncating allocated array without reallocation)
ctx->archive_size = offset;
return NGX_OK;
}
static ngx_int_t
ngx_http_kafka_script_run(ngx_http_request_t *r, ngx_pool_t *pool,
ngx_str_t *msg, void *code_lengths, size_t len, void *code_values)
{
ngx_http_kafka_main_conf_t *kmcf;
ngx_http_script_code_pt code;
ngx_http_script_len_code_pt lcode;
ngx_http_script_engine_t e;
ngx_http_core_main_conf_t *cmcf;
ngx_uint_t i;
u_char *p;
int32_t *crc;
cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module);
for (i = 0; i < cmcf->variables.nelts; i++) {
if (r->variables[i].no_cacheable) {
r->variables[i].valid = 0;
r->variables[i].not_found = 0;
}
}
ngx_memzero(&e, sizeof(ngx_http_script_engine_t));
e.ip = code_lengths;
e.request = r;
e.flushed = 1;
while (*(uintptr_t *) e.ip) {
lcode = *(ngx_http_script_len_code_pt *) e.ip;
len += lcode(&e);
}
if (len == 0) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"[kafka] message is null, we ignore it");
return NGX_DECLINED;
}
kmcf = ngx_http_get_module_main_conf(r, ngx_http_kafka_module);
if (len > kmcf->msg_max_size) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"[kafka] message is too large, we ignore it");
return NGX_DECLINED;
}
p = ngx_pnalloc(pool, NGX_KFK_MSG_HEADER_SIZE + len);
if (p == NULL) {
return NGX_BUSY;
}
msg->data = p;
msg->len = NGX_KFK_MSG_HEADER_SIZE + len;
/* offset */
*(int64_t *)p = 0;
p += sizeof(int64_t);
/* msg size */
*(int32_t *)p = (int32_t)htonl(14 + len);
p += sizeof(int32_t);
/* crc(later calculate) */
crc = (int32_t *)p;
p += sizeof(int32_t);
/* magic */
*(int8_t *)p = 0;
p += sizeof(int8_t);
/* attr */
*(int8_t *)p = 0;
p += sizeof(int8_t);
/* key len */
*(int32_t *)p = (int32_t)htonl(-1);
p += sizeof(int32_t);
/* value len */
*(int32_t *)p = (int32_t)htonl(len);
p += sizeof(int32_t);
e.ip = code_values;
e.pos = p;
while (*(uintptr_t *) e.ip) {
code = *(ngx_http_script_code_pt *) e.ip;
code((ngx_http_script_engine_t *) &e);
}
if (e.pos == NULL) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"[kafka] some unexpected things happened when get request body");
ngx_http_kafka_recycle_msg(pool, msg);
return NGX_ERROR;
}
*crc = (int32_t)htonl(ngx_crc32_long((u_char*)(crc + 1), 10 + len));
return NGX_OK;
}