// make our proposed ZIP-file chunk map
ngx_int_t
ngx_http_zip_generate_pieces(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx)
{
ngx_uint_t i, piece_i;
off_t offset = 0;
time_t unix_time = 0;
ngx_uint_t dos_time = 0;
ngx_http_zip_file_t *file;
ngx_http_zip_piece_t *header_piece, *file_piece, *trailer_piece, *cd_piece;
ngx_http_variable_value_t *vv;
if ((vv = ngx_palloc(r->pool, sizeof(ngx_http_variable_value_t))) == NULL)
return NGX_ERROR;
ctx->unicode_path = 0;
#ifdef NGX_ZIP_HAVE_ICONV
iconv_t *iconv_cd = NULL;
#endif
// Let's try to find special header that contains separator string.
// What for this strange separator string you ask?
// Sometimes there might be a problem converting UTF-8 to zips native
// charset(CP866), because it's not 1:1 conversion. So my solution is to
// developers provide their own version of converted filename and pass it
// to mod_zip along with UTF-8 filename which will go straight to Unicode
// path extra field (thanks to tony2001). So separator is a solution that doesn't
// break current format. And allows passing file name in both formats as one string.
//
// Normally we pass:
// CRC32 <size> <path> <filename>\n
// ...
// * <filename> passed to archive as filename w/o conversion
// * UFT-8 flag for filename is set
//
// tony2001's X-Archive-Charset: <charset> way:
// CRC32 <size> <path> <filename>\n
// ...
// * <filename> is accepted to be UTF-8 string
// * <filename>, converted to <charset> and passed to archive as filename
// * <filename> passed to Unicode path extra field
// * UFT-8 flag for filename is not set
//
// My X-Archive-Name-Sep: <sep> solution:
// CRC32 <size> <path> <native-filename><sep><utf8-filename>\n
// ...
// * <native-filename> passed to archive as filename w/o conversion
// * <utf8-filename> passed to Unicode path extra field
// * UFT-8 flag for filename is not set
//
// You just need to provide separator that won't interfere with file names. I suggest using '/'
// as it is ASCII character and forbidden on most (if not all) platforms as a part of filename.
//
// Empty separator string means no UTF-8 version provided. Usefull when we need to pass only
// names encoded in native charset. It's equal to 'X-Archive-Charset: native;'.
// Note: Currently it is impossible after '[PATCH] Support for UTF-8 file names.'(4f61592b)
// because UFT-8 flag (zip_utf8_flag) is set default for templates.
if(ngx_http_upstream_header_variable(r, vv, (uintptr_t)(&ngx_http_zip_header_name_separator)) == NGX_OK && !vv->not_found) {
ctx->native_charset = 1;
if(vv->len)
ctx->unicode_path = 1;
} else {
#ifdef NGX_ZIP_HAVE_ICONV
if (ngx_http_upstream_header_variable(r, vv, (uintptr_t)(&ngx_http_zip_header_charset_name)) == NGX_OK
&& !vv->not_found && ngx_strncmp(vv->data, "utf8", sizeof("utf8") - 1) != 0) {
if(ngx_strncmp(vv->data, "native", sizeof("native") - 1))
{
char encoding[ICONV_CSNMAXLEN];
snprintf(encoding, sizeof(encoding), "%s//TRANSLIT//IGNORE", vv->data);
iconv_cd = iconv_open((const char *)encoding, "utf-8");
if (iconv_cd == (iconv_t)(-1)) {
ngx_log_error(NGX_LOG_WARN, r->connection->log, errno,
"mod_zip: iconv_open('%s', 'utf-8') failed",
vv->data);
iconv_cd = NULL;
}
else
{
ctx->unicode_path = 1;
ctx->native_charset = 1;
}
}
else
ctx->native_charset = 1;
}
#endif
}
// pieces: for each file: header, data, footer (if needed) -> 2 or 3 per file
// plus file footer (CD + [zip64 end + zip64 locator +] end of cd) in one chunk
ctx->pieces_n = ctx->files.nelts * (2 + (!!ctx->missing_crc32)) + 1;
if ((ctx->pieces = ngx_palloc(r->pool, sizeof(ngx_http_zip_piece_t) * ctx->pieces_n)) == NULL)
return NGX_ERROR;
ctx->cd_size = 0;
unix_time = time(NULL);
dos_time = ngx_dos_time(unix_time);
for (piece_i = i = 0; i < ctx->files.nelts; i++) {
file = &((ngx_http_zip_file_t *)ctx->files.elts)[i];
file->offset = offset;
file->unix_time = unix_time;
file->dos_time = dos_time;
if(ctx->unicode_path) {
#ifdef NGX_ZIP_HAVE_ICONV
if (iconv_cd) {
size_t inlen = file->filename.len, outlen, outleft;
u_char *p, *in;
//inbuf
file->filename_utf8.data = ngx_pnalloc(r->pool, file->filename.len + 1);
ngx_memcpy(file->filename_utf8.data, file->filename.data, file->filename.len);
file->filename_utf8.len = file->filename.len;
file->filename_utf8.data[file->filename.len] = '\0';
//outbuf
outlen = outleft = inlen * sizeof(int) + 15;
file->filename.data = ngx_pnalloc(r->pool, outlen + 1);
in = file->filename_utf8.data;
p = file->filename.data;
//reset state
iconv(iconv_cd, NULL, NULL, NULL, NULL);
//convert the string
iconv(iconv_cd, (char **)&in, &inlen, (char **)&p, &outleft);
//XXX if (res == (size_t)-1) { ? }
file->filename.len = outlen - outleft;
file->filename_utf8_crc32 = ngx_crc32_long(file->filename_utf8.data, file->filename_utf8.len);
}
#endif
else if(vv->len) {
const char * sep = ngx_http_zip_strnrstr((const char*)file->filename.data, file->filename.len,
(const char*)vv->data, vv->len);
if(sep) {
size_t utf8_len = file->filename.len - vv->len - (size_t)(sep - (const char *)file->filename.data);
file->filename_utf8.data = ngx_pnalloc(r->pool, utf8_len);
file->filename_utf8.len = utf8_len;
ngx_memcpy(file->filename_utf8.data, sep + vv->len, utf8_len);
file->filename.len -= utf8_len + vv->len;
file->filename_utf8_crc32 = ngx_crc32_long(file->filename_utf8.data, file->filename_utf8.len);
} /* else { } */ // Separator not found. Okay, no extra field for this one then.
}
}
if(offset >= (off_t) NGX_MAX_UINT32_VALUE)
ctx->zip64_used = file->need_zip64_offset = 1;
if(file->size >= (off_t) NGX_MAX_UINT32_VALUE)
ctx->zip64_used = file->need_zip64 = 1;
ctx->cd_size += sizeof(ngx_zip_central_directory_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_central_t)
+ (file->need_zip64_offset ?
(file->need_zip64 ? sizeof(ngx_zip_extra_field_zip64_sizes_offset_t) : sizeof(ngx_zip_extra_field_zip64_offset_only_t)) :
(file->need_zip64 ? sizeof(ngx_zip_extra_field_zip64_sizes_only_t) : 0) +
(ctx->unicode_path && file->filename_utf8.len ? (sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len): 0)
);
header_piece = &ctx->pieces[piece_i++];
header_piece->type = zip_header_piece;
header_piece->file = file;
header_piece->range.start = offset;
header_piece->range.end = offset += sizeof(ngx_zip_local_file_header_t)
+ file->filename.len + sizeof(ngx_zip_extra_field_local_t) + (file->need_zip64? sizeof(ngx_zip_extra_field_zip64_sizes_only_t):0)
+ (ctx->unicode_path && file->filename_utf8.len ? (sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len): 0);
file_piece = &ctx->pieces[piece_i++];
file_piece->type = zip_file_piece;
file_piece->file = file;
file_piece->range.start = offset;
file_piece->range.end = offset += file->size; //!note: (sizeless chunks): we need file size here / or mark it and modify ranges after
if (file->missing_crc32) { // if incomplete header -> add footer with that info to file
trailer_piece = &ctx->pieces[piece_i++];
trailer_piece->type = zip_trailer_piece;
trailer_piece->file = file;
trailer_piece->range.start = offset;
trailer_piece->range.end = offset += file->need_zip64? sizeof(ngx_zip_data_descriptor_zip64_t) : sizeof(ngx_zip_data_descriptor_t);
//!!TODO: if we want Ranges support - here we know it is impossible for this set
//? check conf/some state and abort?
}
}
#ifdef NGX_ZIP_HAVE_ICONV
if (iconv_cd) {
iconv_close(iconv_cd);
}
#endif
ctx->zip64_used |= offset >= (off_t) NGX_MAX_UINT32_VALUE || ctx->files.nelts >= NGX_MAX_UINT16_VALUE;
ctx->cd_size += sizeof(ngx_zip_end_of_central_directory_record_t);
if (ctx->zip64_used)
ctx->cd_size += sizeof(ngx_zip_zip64_end_of_central_directory_record_t) + sizeof(ngx_zip_zip64_end_of_central_directory_locator_t);
cd_piece = &ctx->pieces[piece_i++];
cd_piece->type = zip_central_directory_piece;
cd_piece->range.start = offset;
cd_piece->range.end = offset += ctx->cd_size;
ctx->pieces_n = piece_i; //!! nasty hack (truncating allocated array without reallocation)
ctx->archive_size = offset;
return NGX_OK;
}
// make Local File Header chunk with extra fields
ngx_chain_t*
ngx_http_zip_file_header_chain_link(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range)
{
ngx_chain_t *link;
ngx_buf_t *b;
ngx_http_zip_file_t *file = piece->file;
ngx_zip_extra_field_local_t extra_field_local;
ngx_zip_extra_field_zip64_sizes_only_t extra_field_zip64;
ngx_zip_local_file_header_t local_file_header;
ngx_zip_extra_field_unicode_path_t extra_field_unicode_path;
size_t len = sizeof(ngx_zip_local_file_header_t) + file->filename.len
+ sizeof(ngx_zip_extra_field_local_t) + (file->need_zip64? sizeof(ngx_zip_extra_field_zip64_sizes_only_t):0
+ (ctx->unicode_path ? (sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len): 0));
if ((link = ngx_alloc_chain_link(r->pool)) == NULL || (b = ngx_calloc_buf(r->pool)) == NULL
|| (b->pos = ngx_pcalloc(r->pool, len)) == NULL)
return NULL;
b->memory = 1;
b->last = b->pos + len;
file->unix_time = time(NULL);
file->dos_time = ngx_dos_time(file->unix_time);
/* A note about the ZIP format: in order to appease all ZIP software I
* could find, the local file header contains the file sizes but not the
* CRC-32, even though setting the third bit of the general purpose bit
* flag would indicate that all three fields should be zeroed out.
*/
local_file_header = ngx_zip_local_file_header_template;
local_file_header.mtime = file->dos_time;
local_file_header.filename_len = file->filename.len;
if (ctx->unicode_path) {
local_file_header.flags &= ~zip_utf8_flag;
}
extra_field_zip64 = ngx_zip_extra_field_zip64_sizes_only_template;
if (file->need_zip64) {
local_file_header.version = zip_version_zip64;
local_file_header.extra_field_len = sizeof(ngx_zip_extra_field_zip64_sizes_only_t) + sizeof(ngx_zip_extra_field_local_t);
extra_field_zip64.uncompressed_size = extra_field_zip64.compressed_size = file->size;
} else {
local_file_header.compressed_size = file->size;
local_file_header.uncompressed_size = file->size;
}
extra_field_unicode_path = ngx_zip_extra_field_unicode_path_template;
if (ctx->unicode_path) {
extra_field_unicode_path.crc32 = file->filename_utf8_crc32;
extra_field_unicode_path.size = sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len;
local_file_header.extra_field_len += sizeof(ngx_zip_extra_field_unicode_path_t) + file->filename_utf8.len;
}
if (!file->missing_crc32) {
local_file_header.flags &= ~zip_missing_crc32_flag;
local_file_header.crc32 = file->crc32;
}
extra_field_local = ngx_zip_extra_field_local_template;
extra_field_local.mtime = file->unix_time;
extra_field_local.atime = file->unix_time;
ngx_memcpy(b->pos, &local_file_header, sizeof(ngx_zip_local_file_header_t));
ngx_memcpy(b->pos + sizeof(ngx_zip_local_file_header_t), file->filename.data, file->filename.len);
ngx_memcpy(b->pos + sizeof(ngx_zip_local_file_header_t) + file->filename.len,
&extra_field_local, sizeof(ngx_zip_extra_field_local_t));
if (file->need_zip64) {
ngx_memcpy(b->pos + sizeof(ngx_zip_local_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_local_t),
&extra_field_zip64, sizeof(ngx_zip_extra_field_zip64_sizes_only_t));
if (ctx->unicode_path) {
ngx_memcpy(b->pos + sizeof(ngx_zip_local_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_local_t) + sizeof(ngx_zip_extra_field_zip64_sizes_only_t), &extra_field_unicode_path, sizeof(ngx_zip_extra_field_unicode_path_t));
ngx_memcpy(b->pos + sizeof(ngx_zip_local_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_local_t) + sizeof(ngx_zip_extra_field_zip64_sizes_only_t) + sizeof(ngx_zip_extra_field_unicode_path_t), file->filename_utf8.data, file->filename_utf8.len);
}
} else if (ctx->unicode_path) {
ngx_memcpy(b->pos + sizeof(ngx_zip_local_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_local_t), &extra_field_unicode_path, sizeof(ngx_zip_extra_field_unicode_path_t));
ngx_memcpy(b->pos + sizeof(ngx_zip_local_file_header_t) + file->filename.len + sizeof(ngx_zip_extra_field_local_t) + sizeof(ngx_zip_extra_field_unicode_path_t), file->filename_utf8.data, file->filename_utf8.len);
}
ngx_http_zip_truncate_buffer(b, &piece->range, range);
link->buf = b;
link->next = NULL;
return link;
}
