int
dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
void *tag, dmu_buf_t **dbp, int flags)
{
dnode_t *dn;
uint64_t blkid;
dmu_buf_impl_t *db;
int err;
int db_flags = DB_RF_CANFAIL;
if (flags & DMU_READ_NO_PREFETCH)
db_flags |= DB_RF_NOPREFETCH;
err = dnode_hold(os, object, FTAG, &dn);
if (err)
return (err);
blkid = dbuf_whichblock(dn, offset);
rw_enter(&dn->dn_struct_rwlock, RW_READER);
db = dbuf_hold(dn, blkid, tag);
rw_exit(&dn->dn_struct_rwlock);
if (db == NULL) {
err = EIO;
} else {
err = dbuf_read(db, NULL, db_flags);
if (err) {
dbuf_rele(db, tag);
db = NULL;
}
}
dnode_rele(dn, FTAG);
*dbp = &db->db; /* NULL db plus first field offset is NULL */
return (err);
}
/*
* returns ENOENT, EIO, or 0.
*/
int
dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp)
{
dnode_t *dn;
dmu_buf_impl_t *db;
int error;
error = dnode_hold(os->os, object, FTAG, &dn);
if (error)
return (error);
rw_enter(&dn->dn_struct_rwlock, RW_READER);
if (dn->dn_bonus == NULL) {
rw_exit(&dn->dn_struct_rwlock);
rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
if (dn->dn_bonus == NULL)
dbuf_create_bonus(dn);
}
db = dn->dn_bonus;
rw_exit(&dn->dn_struct_rwlock);
/* as long as the bonus buf is held, the dnode will be held */
if (refcount_add(&db->db_holds, tag) == 1)
VERIFY(dnode_add_ref(dn, db));
dnode_rele(dn, FTAG);
VERIFY(0 == dbuf_read(db, NULL, DB_RF_MUST_SUCCEED));
*dbp = &db->db;
return (0);
}
i64 dbuf_getu32le_p(dbuf *f, i64 *ppos)
{
u8 m[4];
dbuf_read(f, m, *ppos, 4);
(*ppos) += 4;
return de_getu32le_direct(m);
}
int
dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
void *tag, dmu_buf_t **dbp)
{
dnode_t *dn;
uint64_t blkid;
dmu_buf_impl_t *db;
int err;
err = dnode_hold(os->os, object, FTAG, &dn);
if (err)
return (err);
blkid = dbuf_whichblock(dn, offset);
rw_enter(&dn->dn_struct_rwlock, RW_READER);
db = dbuf_hold(dn, blkid, tag);
rw_exit(&dn->dn_struct_rwlock);
if (db == NULL) {
err = EIO;
} else {
err = dbuf_read(db, NULL, DB_RF_CANFAIL);
if (err) {
dbuf_rele(db, tag);
db = NULL;
}
}
dnode_rele(dn, FTAG);
*dbp = &db->db;
return (err);
}
i64 dbuf_getu16le_p(dbuf *f, i64 *ppos)
{
u8 m[2];
dbuf_read(f, m, *ppos, 2);
(*ppos) += 2;
return de_getu16le_direct(m);
}
u32 dbuf_getRGB(dbuf *f, i64 pos, unsigned int flags)
{
u8 buf[3];
dbuf_read(f, buf, pos, 3);
if(flags&DE_GETRGBFLAG_BGR)
return DE_MAKE_RGB(buf[2], buf[1], buf[0]);
return DE_MAKE_RGB(buf[0], buf[1], buf[2]);
}
static de_uint32 x_dbuf_crc32(dbuf *f, de_int64 pos, de_int64 len)
{
de_uint32 crc;
de_byte *buf;
buf = de_malloc(f->c, len);
dbuf_read(f, buf, pos, len);
crc = de_crc32(buf, len);
de_free(f->c, buf);
return crc;
}
static i64 dbuf_getint_ext_x(dbuf *f, i64 pos, unsigned int nbytes, int is_le)
{
u8 m[24];
if(nbytes>(unsigned int)sizeof(m)) return 0;
dbuf_read(f, m, pos, (i64)nbytes);
if(is_le) {
return 0; // TODO
}
return dbuf_getint_ext_be_direct(m, nbytes);
}
// A function that works a little more like a standard read/fread function than
// does dbuf_read. It returns the number of bytes read, won't read past end of
// file, and helps track the file position.
i64 dbuf_standard_read(dbuf *f, u8 *buf, i64 n, i64 *fpos)
{
i64 amt_to_read;
if(*fpos < 0 || *fpos >= f->len) return 0;
amt_to_read = n;
if(*fpos + amt_to_read > f->len) amt_to_read = f->len - *fpos;
dbuf_read(f, buf, *fpos, amt_to_read);
*fpos += amt_to_read;
return amt_to_read;
}
int dbuf_read_ascii_number(dbuf *f, i64 pos, i64 fieldsize,
int base, i64 *value)
{
char buf[32];
*value = 0;
if(fieldsize>(i64)(sizeof(buf)-1)) return 0;
dbuf_read(f, (u8*)buf, pos, fieldsize);
buf[fieldsize] = '\0';
*value = de_strtoll(buf, NULL, base);
return 1;
}
static int
dmu_tx_check_ioerr(zio_t *zio, dnode_t *dn, int level, uint64_t blkid)
{
int err;
dmu_buf_impl_t *db;
rw_enter(&dn->dn_struct_rwlock, RW_READER);
db = dbuf_hold_level(dn, level, blkid, FTAG);
rw_exit(&dn->dn_struct_rwlock);
if (db == NULL)
return (SET_ERROR(EIO));
err = dbuf_read(db, zio, DB_RF_CANFAIL | DB_RF_NOPREFETCH);
dbuf_rele(db, FTAG);
return (err);
}
// Read (up to) len bytes from f, translate them to characters, and append
// them to s.
void dbuf_read_to_ucstring(dbuf *f, i64 pos, i64 len,
de_ucstring *s, unsigned int conv_flags, int encoding)
{
u8 *buf = NULL;
deark *c = f->c;
if(conv_flags & DE_CONVFLAG_STOP_AT_NUL) {
i64 foundpos = 0;
if(dbuf_search_byte(f, 0x00, pos, len, &foundpos)) {
len = foundpos - pos;
}
}
buf = de_malloc(c, len);
dbuf_read(f, buf, pos, len);
ucstring_append_bytes(s, buf, len, 0, encoding);
de_free(c, buf);
}
// TODO: Make the linereader more efficient and flexible, and make
// it a standard library function.
static int de_linereader_readnextline(deark *c, struct de_linereader *lr,
char *buf, size_t buf_len, unsigned int flags)
{
int ret;
i64 content_len = 0;
i64 total_len = 0;
buf[0] = '\0';
ret = dbuf_find_line(lr->f, lr->f_pos, &content_len, &total_len);
if(!ret) return 0;
if(content_len > (i64)buf_len-1) {
lr->f_pos += total_len;
return 0;
}
dbuf_read(lr->f, (u8*)buf, lr->f_pos, content_len);
buf[content_len] = '\0';
lr->f_pos += total_len;
return 1;
}
void dbuf_copy(dbuf *inf, i64 input_offset, i64 input_len, dbuf *outf)
{
u8 tmpbuf[256];
if(inf->btype==DBUF_TYPE_MEMBUF &&
(input_offset>=0) && (input_offset+input_len<=inf->len))
{
// Fast path if the data to copy is all in memory
dbuf_write(outf, &inf->membuf_buf[input_offset], input_len);
return;
}
if(input_len<=(i64)sizeof(tmpbuf)) {
// Fast path for small sizes
dbuf_read(inf, tmpbuf, input_offset, input_len);
dbuf_write(outf, tmpbuf, input_len);
return;
}
dbuf_buffered_read(inf, input_offset, input_len, copy_cbfn, (void*)outf);
}
int dbuf_memcmp(dbuf *f, i64 pos, const void *s, size_t n)
{
u8 buf1[128];
if(f->cache &&
pos >= f->cache_start_pos &&
pos + (i64)n <= f->cache_start_pos + f->cache_bytes_used)
{
// Fastest path: Compare directly to cache.
return de_memcmp(s, &f->cache[pos - f->cache_start_pos], n);
}
if(n<=sizeof(buf1)) {
// Use a stack buffer if small enough.
dbuf_read(f, buf1, pos, n);
return de_memcmp(buf1, s, n);
}
// Fallback method.
return !dbuf_buffered_read(f, pos, n, dbufmemcmp_cbfn, (void*)s);
}
u8 dbuf_getbyte(dbuf *f, i64 pos)
{
switch(f->btype) {
case DBUF_TYPE_MEMBUF:
// Optimization for memory buffers
if(pos>=0 && pos<f->len) {
return f->membuf_buf[pos];
}
break;
default:
if(f->cache2_bytes_used>0 && pos==f->cache2_start_pos) {
return f->cache2[0];
}
dbuf_read(f, &f->cache2[0], pos, 1);
f->cache2_bytes_used = 1;
f->cache2_start_pos = pos;
return f->cache2[0];
}
return 0x00;
}
/*
* returns ENOENT, EIO, or 0.
*/
int
dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp)
{
dnode_t *dn;
dmu_buf_impl_t *db;
int error;
error = dnode_hold(os, object, FTAG, &dn);
if (error)
return (error);
rw_enter(&dn->dn_struct_rwlock, RW_READER);
if (dn->dn_bonus == NULL) {
rw_exit(&dn->dn_struct_rwlock);
rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
if (dn->dn_bonus == NULL)
dbuf_create_bonus(dn);
}
db = dn->dn_bonus;
/* as long as the bonus buf is held, the dnode will be held */
if (refcount_add(&db->db_holds, tag) == 1) {
VERIFY(dnode_add_ref(dn, db));
atomic_inc_32(&dn->dn_dbufs_count);
}
/*
* Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
* hold and incrementing the dbuf count to ensure that dnode_move() sees
* a dnode hold for every dbuf.
*/
rw_exit(&dn->dn_struct_rwlock);
dnode_rele(dn, FTAG);
VERIFY(0 == dbuf_read(db, NULL, DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH));
*dbp = &db->db;
return (0);
}
/*
* returns ENOENT, EIO, or 0.
*
* This interface will allocate a blank spill dbuf when a spill blk
* doesn't already exist on the dnode.
*
* if you only want to find an already existing spill db, then
* dmu_spill_hold_existing() should be used.
*/
int
dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp)
{
dmu_buf_impl_t *db = NULL;
int err;
if ((flags & DB_RF_HAVESTRUCT) == 0)
rw_enter(&dn->dn_struct_rwlock, RW_READER);
db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);
if ((flags & DB_RF_HAVESTRUCT) == 0)
rw_exit(&dn->dn_struct_rwlock);
ASSERT(db != NULL);
err = dbuf_read(db, NULL, flags);
if (err == 0)
*dbp = &db->db;
else
dbuf_rele(db, tag);
return (err);
}
int
dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
void *tag, dmu_buf_t **dbp, int flags)
{
int err;
int db_flags = DB_RF_CANFAIL;
if (flags & DMU_READ_NO_PREFETCH)
db_flags |= DB_RF_NOPREFETCH;
err = dmu_buf_hold_noread(os, object, offset, tag, dbp);
if (err == 0) {
dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
err = dbuf_read(db, NULL, db_flags);
if (err != 0) {
dbuf_rele(db, tag);
*dbp = NULL;
}
}
return (err);
}
static void
dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)
{
dmu_buf_impl_t *db;
int txgoff = tx->tx_txg & TXG_MASK;
int nblkptr = dn->dn_phys->dn_nblkptr;
int old_toplvl = dn->dn_phys->dn_nlevels - 1;
int new_level = dn->dn_next_nlevels[txgoff];
int i;
rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
/* this dnode can't be paged out because it's dirty */
ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);
ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0);
db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);
ASSERT(db != NULL);
dn->dn_phys->dn_nlevels = new_level;
dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset,
dn->dn_object, dn->dn_phys->dn_nlevels);
/* check for existing blkptrs in the dnode */
for (i = 0; i < nblkptr; i++)
if (!BP_IS_HOLE(&dn->dn_phys->dn_blkptr[i]))
break;
if (i != nblkptr) {
/* transfer dnode's block pointers to new indirect block */
(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED|DB_RF_HAVESTRUCT);
ASSERT(db->db.db_data);
ASSERT(arc_released(db->db_buf));
ASSERT3U(sizeof (blkptr_t) * nblkptr, <=, db->db.db_size);
bcopy(dn->dn_phys->dn_blkptr, db->db.db_data,
sizeof (blkptr_t) * nblkptr);
arc_buf_freeze(db->db_buf);
}
static void
dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)
{
dmu_buf_impl_t *db;
int txgoff = tx->tx_txg & TXG_MASK;
int nblkptr = dn->dn_phys->dn_nblkptr;
int old_toplvl = dn->dn_phys->dn_nlevels - 1;
int new_level = dn->dn_next_nlevels[txgoff];
int i;
rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
/* this dnode can't be paged out because it's dirty */
ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);
ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0);
db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);
ASSERT(db != NULL);
dn->dn_phys->dn_nlevels = new_level;
dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset,
dn->dn_object, dn->dn_phys->dn_nlevels);
/* transfer dnode's block pointers to new indirect block */
(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED|DB_RF_HAVESTRUCT);
ASSERT(db->db.db_data);
ASSERT(arc_released(db->db_buf));
ASSERT3U(sizeof (blkptr_t) * nblkptr, <=, db->db.db_size);
bcopy(dn->dn_phys->dn_blkptr, db->db.db_data,
sizeof (blkptr_t) * nblkptr);
arc_buf_freeze(db->db_buf);
/* set dbuf's parent pointers to new indirect buf */
for (i = 0; i < nblkptr; i++) {
dmu_buf_impl_t *child =
dbuf_find(dn->dn_objset, dn->dn_object, old_toplvl, i);
if (child == NULL)
continue;
#ifdef DEBUG
DB_DNODE_ENTER(child);
ASSERT3P(DB_DNODE(child), ==, dn);
DB_DNODE_EXIT(child);
#endif /* DEBUG */
if (child->db_parent && child->db_parent != dn->dn_dbuf) {
ASSERT(child->db_parent->db_level == db->db_level);
ASSERT(child->db_blkptr !=
&dn->dn_phys->dn_blkptr[child->db_blkid]);
mutex_exit(&child->db_mtx);
continue;
}
ASSERT(child->db_parent == NULL ||
child->db_parent == dn->dn_dbuf);
child->db_parent = db;
dbuf_add_ref(db, child);
if (db->db.db_data)
child->db_blkptr = (blkptr_t *)db->db.db_data + i;
else
child->db_blkptr = NULL;
dprintf_dbuf_bp(child, child->db_blkptr,
"changed db_blkptr to new indirect %s", "");
mutex_exit(&child->db_mtx);
}
bzero(dn->dn_phys->dn_blkptr, sizeof (blkptr_t) * nblkptr);
dbuf_rele(db, FTAG);
rw_exit(&dn->dn_struct_rwlock);
}
i64 dbuf_getu16le(dbuf *f, i64 pos)
{
u8 m[2];
dbuf_read(f, m, pos, 2);
return de_getu16le_direct(m);
}
i64 dbuf_getu32le(dbuf *f, i64 pos)
{
u8 m[4];
dbuf_read(f, m, pos, 4);
return de_getu32le_direct(m);
}
u64 dbuf_getu64le(dbuf *f, i64 pos)
{
u8 m[8];
dbuf_read(f, m, pos, 8);
return de_getu64le_direct(m);
}
// TGA transparency is kind of a mess. Multiple ways of labeling it, some of
// which are ambiguous... Files that have inconsistent labels... Files that
// claim to have transparency but don't, or that claim not to but do...
static void do_prescan_image(deark *c, lctx *d, dbuf *unc_pixels)
{
i64 num_pixels;
i64 i;
u8 b[4];
int has_alpha_0 = 0;
int has_alpha_partial = 0;
int has_alpha_255 = 0;
if(d->pixel_depth!=32 || d->color_type!=TGA_CLRTYPE_TRUECOLOR) {
return;
}
if(d->num_attribute_bits!=0 && d->num_attribute_bits!=8) {
de_warn(c, "%d-bit attribute channel not supported. Transparency disabled.",
(int)d->num_attribute_bits);
return;
}
if(d->has_extension_area) {
if(d->attributes_type==0) {
// attributes_type==0 technically also means there is no transparency,
// but this cannot be trusted.
;
}
if(d->attributes_type==1 || d->attributes_type==2) {
// Indicates that attribute data can be ignored.
return;
}
else if(d->attributes_type==3 && d->num_attribute_bits==8) {
// Alpha channel seems to be labeled correctly.
// Trust it, and don't scan the image.
d->has_alpha_channel = 1;
return;
}
else if(d->attributes_type==4) {
// Sigh. The spec shows that Field 24 (Attributes Type) == 4 is for
// pre-multiplied alpha. Then the discussion section says that
// Field *23* ("Attributes Type") == *3* is for premultiplied alpha.
// I have to guess that the "23" and "3" are clerical errors, but that
// doesn't do me much good unless all TGA developers made the same guess.
de_warn(c, "Pre-multiplied alpha is not supported. Disabling transparency.");
return;
}
}
de_dbg(c, "pre-scanning image");
num_pixels = d->main_image.width * d->main_image.height;
for(i=0; i<num_pixels; i++) {
// TODO: This may run a lot slower than it ought to.
dbuf_read(unc_pixels, b, i*4, 4);
// BGRA order
if(b[3]==0x00) {
has_alpha_0 = 1;
}
else if(b[3]==0xff) {
has_alpha_255 = 1;
}
else {
has_alpha_partial = 1;
break;
}
if(has_alpha_0 && has_alpha_255) {
break;
}
}
// Note that the has_alpha_* variables may not all be accurate at this point,
// because we stop scanning when we have the info we need.
if(has_alpha_partial || (has_alpha_0 && has_alpha_255)) {
if(d->num_attribute_bits==0) {
de_warn(c, "Detected likely alpha channel. Enabling transparency, even though "
"the image is labeled as non-transparent.");
}
d->has_alpha_channel = 1;
return;
}
else if(has_alpha_0) { // All 0x00
if(d->num_attribute_bits!=0) {
de_warn(c, "Non-visible image detected. Disabling transparency.");
}
else {
de_dbg(c, "potential alpha channel ignored: all 0 bits");
}
return;
}
else { // All 0xff
de_dbg(c, "potential alpha channel is moot: all 1 bits");
return;
}
}
static int do_box(deark *c, struct de_boxesctx *bctx, i64 pos, i64 len,
int level, i64 *pbytes_consumed)
{
i64 size32, size64;
i64 header_len; // Not including UUIDs
i64 payload_len; // Including UUIDs
i64 total_len;
struct de_fourcc box4cc;
char uuid_string[50];
int ret;
int retval = 0;
struct de_boxdata *parentbox;
struct de_boxdata *curbox;
parentbox = bctx->curbox;
bctx->curbox = de_malloc(c, sizeof(struct de_boxdata));
curbox = bctx->curbox;
curbox->parent = parentbox;
if(len<8) {
de_dbg(c, "(ignoring %d extra bytes at %"I64_FMT")", (int)len, pos);
goto done;
}
size32 = dbuf_getu32be(bctx->f, pos);
dbuf_read_fourcc(bctx->f, pos+4, &box4cc, 4, 0x0);
curbox->boxtype = box4cc.id;
if(size32>=8) {
header_len = 8;
payload_len = size32-8;
}
else if(size32==0) {
header_len = 8;
payload_len = len-8;
}
else if(size32==1) {
if(len<16) {
de_dbg(c, "(ignoring %d extra bytes at %"I64_FMT")", (int)len, pos);
goto done;
}
header_len = 16;
size64 = dbuf_geti64be(bctx->f, pos+8);
if(size64<16) goto done;
payload_len = size64-16;
}
else {
de_err(c, "Invalid or unsupported box format");
goto done;
}
total_len = header_len + payload_len;
if(curbox->boxtype==DE_BOX_uuid && payload_len>=16) {
curbox->is_uuid = 1;
dbuf_read(bctx->f, curbox->uuid, pos+header_len, 16);
}
curbox->level = level;
curbox->box_pos = pos;
curbox->box_len = total_len;
curbox->payload_pos = pos+header_len;
curbox->payload_len = payload_len;
if(curbox->is_uuid) {
curbox->payload_pos += 16;
curbox->payload_len -= 16;
}
if(bctx->identify_box_fn) {
bctx->identify_box_fn(c, bctx);
}
if(c->debug_level>0) {
char name_str[80];
if(curbox->box_name) {
de_snprintf(name_str, sizeof(name_str), " (%s)", curbox->box_name);
}
else {
name_str[0] = '\0';
}
if(curbox->is_uuid) {
de_fmtutil_render_uuid(c, curbox->uuid, uuid_string, sizeof(uuid_string));
de_dbg(c, "box '%s'{%s}%s at %"I64_FMT", len=%"I64_FMT,
box4cc.id_dbgstr, uuid_string, name_str,
pos, total_len);
}
else {
de_dbg(c, "box '%s'%s at %"I64_FMT", len=%"I64_FMT", dlen=%"I64_FMT,
box4cc.id_dbgstr, name_str, pos,
total_len, payload_len);
}
}
if(total_len > len) {
de_err(c, "Invalid oversized box, or unexpected end of file "
"(box at %"I64_FMT" ends at %"I64_FMT", "
"parent ends at %"I64_FMT")",
pos, pos+total_len, pos+len);
goto done;
}
de_dbg_indent(c, 1);
ret = bctx->handle_box_fn(c, bctx);
de_dbg_indent(c, -1);
if(!ret) goto done;
if(curbox->is_superbox) {
i64 children_pos, children_len;
i64 max_nchildren;
de_dbg_indent(c, 1);
children_pos = pos+header_len + curbox->extra_bytes_before_children;
children_len = payload_len - curbox->extra_bytes_before_children;
max_nchildren = (curbox->num_children_is_known) ? curbox->num_children : -1;
do_box_sequence(c, bctx, children_pos, children_len, max_nchildren, level+1);
de_dbg_indent(c, -1);
}
*pbytes_consumed = total_len;
retval = 1;
done:
de_free(c, bctx->curbox);
bctx->curbox = parentbox; // Restore the curbox pointer
return retval;
}
double dbuf_getfloat32x(dbuf *f, i64 pos, int is_le)
{
u8 buf[4];
dbuf_read(f, buf, pos, 4);
return de_getfloat32x_direct(f->c, buf, is_le);
}
double dbuf_getfloat64x(dbuf *f, i64 pos, int is_le)
{
u8 buf[8];
dbuf_read(f, buf, pos, 8);
return de_getfloat64x_direct(f->c, buf, is_le);
}
// An advanced function for reading a string from a file.
// The issue is that some strings are both human-readable and machine-readable.
// In such a case, we'd like to read some data from a file into a nice printable
// ucstring, while also making some or all of the raw bytes available, say for
// byte-for-byte string comparisons.
// Plus (for NUL-terminated/padded strings), we may need to know the actual length
// of the string in the file, so that it can be skipped over, even if we don't
// care about the whole string.
// Caller is responsible for calling destroy_stringreader() on the returned value.
// max_bytes_to_scan: The maximum number of bytes to read from the file.
// max_bytes_to_keep: The maximum (or in some cases the exact) number of bytes,
// not counting any NUL terminator, to return in ->sz.
// The ->str field is a Unicode version of ->sz, so this also affects ->str.
// If DE_CONVFLAG_STOP_AT_NUL is not set, it is assumed we are reading a string
// of known length, that may have internal NUL bytes. The caller must set
// max_bytes_to_scan and max_bytes_to_keep to the same value. The ->sz field will
// always be allocated with this many bytes, plus one more for an artificial NUL
// terminator.
// If DE_CONVFLAG_WANT_UTF8 is set, then the ->sz_utf8 field will be set to a
// UTF-8 version of ->str. This is mainly useful if the original string was
// UTF-16. sz_utf8 is not "printable" -- use ucstring_get_printable_sz_n(str) for
// that.
// Recognized flags:
// - DE_CONVFLAG_STOP_AT_NUL
// - DE_CONVFLAG_WANT_UTF8
struct de_stringreaderdata *dbuf_read_string(dbuf *f, i64 pos,
i64 max_bytes_to_scan,
i64 max_bytes_to_keep,
unsigned int flags, int encoding)
{
deark *c = f->c;
struct de_stringreaderdata *srd;
i64 foundpos = 0;
int ret;
i64 bytes_avail_to_read;
i64 bytes_to_malloc;
i64 x_strlen;
srd = de_malloc(c, sizeof(struct de_stringreaderdata));
srd->str = ucstring_create(c);
bytes_avail_to_read = max_bytes_to_scan;
if(bytes_avail_to_read > f->len-pos) {
bytes_avail_to_read = f->len-pos;
}
srd->bytes_consumed = bytes_avail_to_read; // default
// From here on, we can safely bail out ("goto done"). The
// de_stringreaderdata struct is sufficiently valid.
if(!(flags&DE_CONVFLAG_STOP_AT_NUL) &&
(max_bytes_to_scan != max_bytes_to_keep))
{
// To reduce possible confusion, we require that
// max_bytes_to_scan==max_bytes_to_keep in this case.
srd->sz = de_malloc(c, max_bytes_to_keep+1);
goto done;
}
if(flags&DE_CONVFLAG_STOP_AT_NUL) {
ret = dbuf_search_byte(f, 0x00, pos, bytes_avail_to_read, &foundpos);
if(ret) {
srd->found_nul = 1;
}
else {
// No NUL byte found. Could be an error in some formats, but in
// others NUL is used as separator or as padding, not a terminator.
foundpos = pos+bytes_avail_to_read;
}
x_strlen = foundpos-pos;
srd->bytes_consumed = x_strlen+1;
}
else {
x_strlen = max_bytes_to_keep;
srd->bytes_consumed = x_strlen;
}
bytes_to_malloc = x_strlen+1;
if(bytes_to_malloc>(max_bytes_to_keep+1)) {
bytes_to_malloc = max_bytes_to_keep+1;
srd->was_truncated = 1;
}
srd->sz = de_malloc(c, bytes_to_malloc);
dbuf_read(f, (u8*)srd->sz, pos, bytes_to_malloc-1); // The last byte remains NUL
ucstring_append_bytes(srd->str, (const u8*)srd->sz, bytes_to_malloc-1, 0, encoding);
if(flags&DE_CONVFLAG_WANT_UTF8) {
srd->sz_utf8_strlen = (size_t)ucstring_count_utf8_bytes(srd->str);
srd->sz_utf8 = de_malloc(c, srd->sz_utf8_strlen + 1);
ucstring_to_sz(srd->str, srd->sz_utf8, srd->sz_utf8_strlen + 1, 0, DE_ENCODING_UTF8);
}
done:
if(!srd->sz) {
// Always return a valid sz, even on failure.
srd->sz = de_malloc(c, 1);
}
if((flags&DE_CONVFLAG_WANT_UTF8) && !srd->sz_utf8) {
// Always return a valid sz_utf8 if it was requested, even on failure.
srd->sz_utf8 = de_malloc(c, 1);
srd->sz_utf8_strlen = 0;
}
return srd;
}
// Read len bytes, starting at file position pos, into buf.
// Unread bytes will be set to 0.
void dbuf_read(dbuf *f, u8 *buf, i64 pos, i64 len)
{
i64 bytes_read = 0;
i64 bytes_to_read;
deark *c;
c = f->c;
bytes_to_read = len;
if(pos >= f->len) {
bytes_to_read = 0;
}
else if(pos + bytes_to_read > f->len) {
bytes_to_read = f->len - pos;
}
if(bytes_to_read<1) {
goto done_read;
}
if(!f->cache && f->cache_policy==DE_CACHE_POLICY_ENABLED) {
populate_cache(f);
}
// If the data we need is all cached, get it from cache.
if(f->cache &&
pos >= f->cache_start_pos &&
pos + bytes_to_read <= f->cache_start_pos + f->cache_bytes_used)
{
de_memcpy(buf, &f->cache[pos - f->cache_start_pos], (size_t)bytes_to_read);
bytes_read = bytes_to_read;
goto done_read;
}
switch(f->btype) {
case DBUF_TYPE_IFILE:
if(!f->fp) {
de_err(c, "Internal: File not open");
de_fatalerror(c);
return;
}
// For performance reasons, don't call fseek if we're already at the
// right position.
if(!f->file_pos_known || f->file_pos!=pos) {
de_fseek(f->fp, pos, SEEK_SET);
}
bytes_read = fread(buf, 1, (size_t)bytes_to_read, f->fp);
f->file_pos = pos + bytes_read;
f->file_pos_known = 1;
break;
case DBUF_TYPE_IDBUF:
// Recursive call to the parent dbuf.
dbuf_read(f->parent_dbuf, buf, f->offset_into_parent_dbuf+pos, bytes_to_read);
// The parent dbuf always writes 'bytes_to_read' bytes.
bytes_read = bytes_to_read;
break;
case DBUF_TYPE_MEMBUF:
de_memcpy(buf, &f->membuf_buf[pos], (size_t)bytes_to_read);
bytes_read = bytes_to_read;
break;
default:
de_err(c, "Internal: getbytes from this I/O type not implemented");
de_fatalerror(c);
return;
}
done_read:
// Zero out any requested bytes that were not read.
if(bytes_read < len) {
de_zeromem(buf+bytes_read, (size_t)(len - bytes_read));
}
}
