cv::Mat ZipByteReader::Read(size_t seqId, const std::string& path, bool grayscale)
{
// Find index of the file in .zip file.
auto r = m_seqIdToIndex.find(seqId);
if (r == m_seqIdToIndex.end())
RuntimeError("Could not find file %s in the zip file, sequence id = %lu", path.c_str(), (long)seqId);
zip_uint64_t index = std::get<0>((*r).second);
zip_uint64_t size = std::get<1>((*r).second);
auto contents = m_workspace.pop_or_create([size]() { return vector<unsigned char>(size); });
if (contents.size() < size)
contents.resize(size);
auto zipFile = m_zips.pop_or_create([this]() { return OpenZip(); });
{
std::unique_ptr<zip_file_t, void(*)(zip_file_t*)> file(
zip_fopen_index(zipFile.get(), index, 0),
[](zip_file_t* f)
{
assert(f != nullptr);
int err = zip_fclose(f);
assert(ZIP_ER_OK == err);
#ifdef NDEBUG
UNUSED(err);
#endif
});
assert(nullptr != file);
if (nullptr == file)
{
RuntimeError("Could not open file %s in the zip file, sequence id = %lu, zip library error: %s",
path.c_str(), (long)seqId, GetZipError(zip_error_code_zip(zip_get_error(zipFile.get()))).c_str());
}
assert(contents.size() >= size);
zip_uint64_t bytesRead = zip_fread(file.get(), contents.data(), size);
assert(bytesRead == size);
if (bytesRead != size)
{
RuntimeError("Bytes read %lu != expected %lu while reading file %s",
(long)bytesRead, (long)size, path.c_str());
}
}
m_zips.push(std::move(zipFile));
cv::Mat img;
if (grayscale)
img = cv::imdecode(cv::Mat(1, (int)size, CV_8UC1, contents.data()), cv::IMREAD_GRAYSCALE);
else
img = cv::imdecode(cv::Mat(1, (int)size, CV_8UC1, contents.data()), cv::IMREAD_COLOR);
assert(nullptr != img.data);
m_workspace.push(std::move(contents));
return img;
}
void
_zip_set_open_error(int *zep, const zip_error_t *err, int ze)
{
if (err) {
ze = zip_error_code_zip(err);
if (zip_error_system_type(err) == ZIP_ET_SYS) {
errno = zip_error_code_system(err);
}
}
if (zep)
*zep = ze;
}
zip_int64_t
zip_error_to_data(const zip_error_t *error, void *data, zip_uint64_t length)
{
int *e = (int *)data;
if (length < sizeof(int)*2) {
return -1;
}
e[0] = zip_error_code_zip(error);
e[1] = zip_error_code_system(error);
return sizeof(int)*2;
}
/*
* tests for file existence
*/
static exists_t
_zip_file_exists(zip_source_t *src, zip_error_t *error)
{
struct zip_stat st;
zip_stat_init(&st);
if (zip_source_stat(src, &st) != 0) {
zip_error_t *src_error = zip_source_error(src);
if (zip_error_code_zip(src_error) == ZIP_ER_READ && zip_error_code_system(src_error) == ENOENT) {
return EXISTS_NOT;
}
_zip_error_copy(error, src_error);
return EXISTS_ERROR;
}
return (st.valid & ZIP_STAT_SIZE) && st.size == 0 ? EXISTS_EMPTY : EXISTS_NONEMPTY;
}
static zip_int64_t
compress_read(zip_source_t *src, struct deflate *ctx, void *data, zip_uint64_t len)
{
int end, ret;
zip_int64_t n;
zip_uint64_t out_offset;
uInt out_len;
if (zip_error_code_zip(&ctx->error) != ZIP_ER_OK)
return -1;
if (len == 0 || ctx->is_stored) {
return 0;
}
out_offset = 0;
out_len = (uInt)ZIP_MIN(UINT_MAX, len);
ctx->zstr.next_out = (Bytef *)data;
ctx->zstr.avail_out = out_len;
end = 0;
while (!end) {
ret = deflate(&ctx->zstr, ctx->eof ? Z_FINISH : 0);
switch (ret) {
case Z_STREAM_END:
if (ctx->can_store && ctx->zstr.total_in <= ctx->zstr.total_out) {
ctx->is_stored = true;
ctx->size = ctx->zstr.total_in;
memcpy(data, ctx->buffer, ctx->size);
return (zip_int64_t)ctx->size;
}
/* fallthrough */
case Z_OK:
/* all ok */
if (ctx->zstr.avail_out == 0) {
out_offset += out_len;
if (out_offset < len) {
out_len = (uInt)ZIP_MIN(UINT_MAX, len-out_offset);
ctx->zstr.next_out = (Bytef *)data+out_offset;
ctx->zstr.avail_out = out_len;
}
else {
ctx->can_store = false;
end = 1;
}
}
else if (ctx->eof && ctx->zstr.avail_in == 0)
end = 1;
break;
case Z_BUF_ERROR:
if (ctx->zstr.avail_in == 0) {
if (ctx->eof) {
end = 1;
break;
}
if ((n=zip_source_read(src, ctx->buffer, sizeof(ctx->buffer))) < 0) {
_zip_error_set_from_source(&ctx->error, src);
end = 1;
break;
}
else if (n == 0) {
ctx->eof = true;
/* TODO: check against stat of src? */
ctx->size = ctx->zstr.total_in;
}
else {
if (ctx->zstr.total_in > 0) {
/* we overwrote a previously filled ctx->buffer */
ctx->can_store = false;
}
ctx->zstr.next_in = (Bytef *)ctx->buffer;
ctx->zstr.avail_in = (uInt)n;
}
continue;
}
/* fallthrough */
case Z_NEED_DICT:
case Z_DATA_ERROR:
case Z_STREAM_ERROR:
case Z_MEM_ERROR:
zip_error_set(&ctx->error, ZIP_ER_ZLIB, ret);
end = 1;
break;
}
}
if (ctx->zstr.avail_out < len) {
ctx->can_store = false;
return (zip_int64_t)(len - ctx->zstr.avail_out);
}
return (zip_error_code_zip(&ctx->error) == ZIP_ER_OK) ? 0 : -1;
}
static zip_int64_t
decompress_read(zip_source_t *src, struct deflate *ctx, void *data, zip_uint64_t len)
{
int end, ret;
zip_int64_t n;
zip_uint64_t out_offset;
uInt out_len;
if (zip_error_code_zip(&ctx->error) != ZIP_ER_OK)
return -1;
if (len == 0)
return 0;
out_offset = 0;
out_len = (uInt)ZIP_MIN(UINT_MAX, len);
ctx->zstr.next_out = (Bytef *)data;
ctx->zstr.avail_out = out_len;
end = 0;
while (!end) {
ret = inflate(&ctx->zstr, Z_SYNC_FLUSH);
switch (ret) {
case Z_OK:
if (ctx->zstr.avail_out == 0) {
out_offset += out_len;
if (out_offset < len) {
out_len = (uInt)ZIP_MIN(UINT_MAX, len-out_offset);
ctx->zstr.next_out = (Bytef *)data+out_offset;
ctx->zstr.avail_out = out_len;
}
else {
end = 1;
}
}
break;
case Z_STREAM_END:
ctx->eof = 1;
end = 1;
break;
case Z_BUF_ERROR:
if (ctx->zstr.avail_in == 0) {
if (ctx->eof) {
end = 1;
break;
}
if ((n=zip_source_read(src, ctx->buffer, sizeof(ctx->buffer))) < 0) {
_zip_error_set_from_source(&ctx->error, src);
end = 1;
break;
}
else if (n == 0) {
ctx->eof = 1;
}
else {
ctx->zstr.next_in = (Bytef *)ctx->buffer;
ctx->zstr.avail_in = (uInt)n;
}
continue;
}
/* fallthrough */
case Z_NEED_DICT:
case Z_DATA_ERROR:
case Z_STREAM_ERROR:
case Z_MEM_ERROR:
zip_error_set(&ctx->error, ZIP_ER_ZLIB, ret);
end = 1;
break;
}
}
if (ctx->zstr.avail_out < len)
return (zip_int64_t)(len - ctx->zstr.avail_out);
return (zip_error_code_zip(&ctx->error) == ZIP_ER_OK) ? 0 : -1;
}
static zip_cdir_t *
_zip_find_central_dir(zip_t *za, zip_uint64_t len)
{
zip_cdir_t *cdir, *cdirnew;
zip_uint8_t *match;
zip_int64_t buf_offset;
zip_uint64_t buflen;
zip_int64_t a;
zip_int64_t best;
zip_error_t error;
zip_buffer_t *buffer;
if (len < EOCDLEN) {
zip_error_set(&za->error, ZIP_ER_NOZIP, 0);
return NULL;
}
buflen = (len < CDBUFSIZE ? len : CDBUFSIZE);
if (zip_source_seek(za->src, -(zip_int64_t)buflen, SEEK_END) < 0) {
zip_error_t *src_error = zip_source_error(za->src);
if (zip_error_code_zip(src_error) != ZIP_ER_SEEK || zip_error_code_system(src_error) != EFBIG) {
/* seek before start of file on my machine */
_zip_error_copy(&za->error, src_error);
return NULL;
}
}
if ((buf_offset = zip_source_tell(za->src)) < 0) {
_zip_error_set_from_source(&za->error, za->src);
return NULL;
}
if ((buffer = _zip_buffer_new_from_source(za->src, buflen, NULL, &za->error)) == NULL) {
return NULL;
}
best = -1;
cdir = NULL;
if (buflen >= CDBUFSIZE) {
/* EOCD64 locator is before EOCD, so leave place for it */
_zip_buffer_set_offset(buffer, EOCD64LOCLEN);
}
zip_error_set(&error, ZIP_ER_NOZIP, 0);
match = _zip_buffer_get(buffer, 0);
while ((match=_zip_memmem(match, _zip_buffer_left(buffer)-(EOCDLEN-4), (const unsigned char *)EOCD_MAGIC, 4)) != NULL) {
_zip_buffer_set_offset(buffer, (zip_uint64_t)(match - _zip_buffer_data(buffer)));
if ((cdirnew = _zip_read_cdir(za, buffer, (zip_uint64_t)buf_offset, &error)) != NULL) {
if (cdir) {
if (best <= 0) {
best = _zip_checkcons(za, cdir, &error);
}
a = _zip_checkcons(za, cdirnew, &error);
if (best < a) {
_zip_cdir_free(cdir);
cdir = cdirnew;
best = a;
}
else {
_zip_cdir_free(cdirnew);
}
}
else {
cdir = cdirnew;
if (za->open_flags & ZIP_CHECKCONS)
best = _zip_checkcons(za, cdir, &error);
else {
best = 0;
}
}
cdirnew = NULL;
}
match++;
_zip_buffer_set_offset(buffer, (zip_uint64_t)(match - _zip_buffer_data(buffer)));
}
_zip_buffer_free(buffer);
if (best < 0) {
_zip_error_copy(&za->error, &error);
_zip_cdir_free(cdir);
return NULL;
}
return cdir;
}
static zip_cdir_t *
_zip_read_cdir(zip_t *za, zip_buffer_t *buffer, zip_uint64_t buf_offset, zip_error_t *error)
{
zip_cdir_t *cd;
zip_uint16_t comment_len;
zip_uint64_t i, left;
zip_uint64_t eocd_offset = _zip_buffer_offset(buffer);
zip_buffer_t *cd_buffer;
if (_zip_buffer_left(buffer) < EOCDLEN) {
/* not enough bytes left for comment */
zip_error_set(error, ZIP_ER_NOZIP, 0);
return NULL;
}
/* check for end-of-central-dir magic */
if (memcmp(_zip_buffer_get(buffer, 4), EOCD_MAGIC, 4) != 0) {
zip_error_set(error, ZIP_ER_NOZIP, 0);
return NULL;
}
if (eocd_offset >= EOCD64LOCLEN && memcmp(_zip_buffer_data(buffer) + eocd_offset - EOCD64LOCLEN, EOCD64LOC_MAGIC, 4) == 0) {
_zip_buffer_set_offset(buffer, eocd_offset - EOCD64LOCLEN);
cd = _zip_read_eocd64(za->src, buffer, buf_offset, za->flags, error);
}
else {
_zip_buffer_set_offset(buffer, eocd_offset);
cd = _zip_read_eocd(buffer, buf_offset, za->flags, error);
}
if (cd == NULL)
return NULL;
_zip_buffer_set_offset(buffer, eocd_offset + 20);
comment_len = _zip_buffer_get_16(buffer);
if (cd->offset + cd->size > buf_offset + eocd_offset) {
/* cdir spans past EOCD record */
zip_error_set(error, ZIP_ER_INCONS, 0);
_zip_cdir_free(cd);
return NULL;
}
if (comment_len || (za->open_flags & ZIP_CHECKCONS)) {
zip_uint64_t tail_len;
_zip_buffer_set_offset(buffer, eocd_offset + EOCDLEN);
tail_len = _zip_buffer_left(buffer);
if (tail_len < comment_len || ((za->open_flags & ZIP_CHECKCONS) && tail_len != comment_len)) {
zip_error_set(error, ZIP_ER_INCONS, 0);
_zip_cdir_free(cd);
return NULL;
}
if (comment_len) {
if ((cd->comment=_zip_string_new(_zip_buffer_get(buffer, comment_len), comment_len, ZIP_FL_ENC_GUESS, error)) == NULL) {
_zip_cdir_free(cd);
return NULL;
}
}
}
if (cd->offset >= buf_offset) {
zip_uint8_t *data;
/* if buffer already read in, use it */
_zip_buffer_set_offset(buffer, cd->offset - buf_offset);
if ((data = _zip_buffer_get(buffer, cd->size)) == NULL) {
zip_error_set(error, ZIP_ER_INCONS, 0);
_zip_cdir_free(cd);
return NULL;
}
if ((cd_buffer = _zip_buffer_new(data, cd->size)) == NULL) {
zip_error_set(error, ZIP_ER_MEMORY, 0);
_zip_cdir_free(cd);
return NULL;
}
}
else {
cd_buffer = NULL;
if (zip_source_seek(za->src, (zip_int64_t)cd->offset, SEEK_SET) < 0) {
_zip_error_set_from_source(error, za->src);
_zip_cdir_free(cd);
return NULL;
}
/* possible consistency check: cd->offset = len-(cd->size+cd->comment_len+EOCDLEN) ? */
if (zip_source_tell(za->src) != (zip_int64_t)cd->offset) {
zip_error_set(error, ZIP_ER_NOZIP, 0);
_zip_cdir_free(cd);
return NULL;
}
}
left = (zip_uint64_t)cd->size;
i=0;
while (left > 0) {
bool grown = false;
zip_int64_t entry_size;
if (i == cd->nentry) {
/* InfoZIP has a hack to avoid using Zip64: it stores nentries % 0x10000 */
/* This hack isn't applicable if we're using Zip64, or if there is no central directory entry following. */
if (cd->is_zip64 || left < CDENTRYSIZE) {
break;
}
if (!_zip_cdir_grow(cd, 0x10000, error)) {
_zip_cdir_free(cd);
_zip_buffer_free(cd_buffer);
return NULL;
}
grown = true;
}
if ((cd->entry[i].orig=_zip_dirent_new()) == NULL || (entry_size = _zip_dirent_read(cd->entry[i].orig, za->src, cd_buffer, false, error)) < 0) {
if (grown && zip_error_code_zip(error) == ZIP_ER_NOZIP) {
zip_error_set(error, ZIP_ER_INCONS, 0);
}
_zip_cdir_free(cd);
_zip_buffer_free(cd_buffer);
return NULL;
}
i++;
left -= (zip_uint64_t)entry_size;
}
if (i != cd->nentry || left > 0) {
zip_error_set(error, ZIP_ER_INCONS, 0);
_zip_buffer_free(cd_buffer);
_zip_cdir_free(cd);
return NULL;
}
if (za->open_flags & ZIP_CHECKCONS) {
bool ok;
if (cd_buffer) {
ok = _zip_buffer_eof(cd_buffer);
}
else {
zip_int64_t offset = zip_source_tell(za->src);
if (offset < 0) {
_zip_error_set_from_source(error, za->src);
_zip_cdir_free(cd);
return NULL;
}
ok = ((zip_uint64_t)offset == cd->offset + cd->size);
}
if (!ok) {
zip_error_set(error, ZIP_ER_INCONS, 0);
_zip_buffer_free(cd_buffer);
_zip_cdir_free(cd);
return NULL;
}
}
_zip_buffer_free(cd_buffer);
return cd;
}