BufferPtr IteratorBase::fetchPointBuffer(Statement statement, sdo_pc* pc)
{
boost::int32_t id = statement->GetInteger(&pc->pc_id);
BufferMap::const_iterator i = m_buffers.find(id);
if (i != m_buffers.end())
{
getReader().log()->get(logDEBUG2) << "IteratorBase::fetchPointBuffer: found existing PointBuffer with id " << id << std::endl;
return i->second;
}
else
{
boost::uint32_t block_capacity(0);
Schema schema = m_reader.fetchSchema(statement, pc, block_capacity, getReader().getName());
m_orientation = schema.getOrientation();
getReader().log()->get(logDEBUG2) << "Incoming schema orientation is " << m_orientation << std::endl;
BufferPtr output = BufferPtr(new PointBuffer(schema, block_capacity));
std::pair<int, BufferPtr> p(id, output);
m_buffers.insert(p);
getReader().log()->get(logDEBUG2) << "IteratorBase::fetchPointBuffer: creating new PointBuffer with id " << id << std::endl;
return p.second;
}
}
/*
* filesystem operations
*/
static s32_t cpiofs_mount(struct mount_t * m, char * dev, s32_t flag)
{
struct block_t * blk;
struct cpio_newc_header header;
if(dev == NULL)
return EINVAL;
blk = (struct block_t *)m->m_dev;
if(!blk)
return EACCES;
if(block_capacity(blk) <= sizeof(struct cpio_newc_header))
return EINTR;
if(block_read(blk, (u8_t *)(&header), 0, sizeof(struct cpio_newc_header)) != sizeof(struct cpio_newc_header))
return EIO;
if(strncmp((const char *)(header.c_magic), (const char *)"070701", 6) != 0)
return EINVAL;
m->m_flags = (flag & MOUNT_MASK) | MOUNT_RDONLY;
m->m_root->v_data = 0;
m->m_data = NULL;
return 0;
}
static void read_thread(void) {
off_t offset = ftello(gInFile);
wanted_t *w = gWantedFiles;
lzma_index_iter iter;
lzma_index_iter_init(&iter, gIndex);
while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_BLOCK)) {
// Don't decode the file-index
off_t boffset = iter.block.compressed_file_offset;
size_t bsize = iter.block.total_size;
if (gFileIndexOffset && boffset == gFileIndexOffset)
continue;
// Do we need this block?
if (gWantedFiles && gExplicitFiles) {
off_t uend = iter.block.uncompressed_file_offset +
iter.block.uncompressed_size;
if (!w || w->start >= uend) {
debug("read: skip %llu", iter.block.number_in_file);
continue;
}
for ( ; w && w->end < uend; w = w->next) ;
}
debug("read: want %llu", iter.block.number_in_file);
// Seek if needed, and get the data
if (offset != boffset) {
fseeko(gInFile, boffset, SEEK_SET);
offset = boffset;
}
if (iter.block.uncompressed_size > MAXSPLITSIZE) { // must stream
if (gRbuf)
rbuf_consume(gRbuf->insize); // clear
read_block(true, iter.stream.flags->check,
iter.block.uncompressed_file_offset);
} else {
// Get a block to work with
pipeline_item_t *pi;
queue_pop(gPipelineStartQ, (void**)&pi);
io_block_t *ib = (io_block_t*)(pi->data);
block_capacity(ib, iter.block.unpadded_size,
iter.block.uncompressed_size);
ib->insize = fread(ib->input, 1, bsize, gInFile);
if (ib->insize < bsize)
die("Error reading block contents");
offset += bsize;
ib->uoffset = iter.block.uncompressed_file_offset;
ib->check = iter.stream.flags->check;
ib->btype = BLOCK_SIZED; // Indexed blocks always sized
pipeline_split(pi);
}
}
pipeline_stop();
}
static ssize_t block_read_capacity(struct kobj_t * kobj, void * buf, size_t size)
{
struct block_t * blk = (struct block_t *)kobj->priv;
return sprintf(buf, "%lld", block_capacity(blk));
}
u64_t block_write(struct block_t * blk, u8_t * buf, u64_t offset, u64_t count)
{
u64_t blkno, blksz, blkcnt, capacity;
u64_t len, tmp;
u64_t ret = 0;
u8_t * p;
if(!blk || !buf || !count)
return 0;
blksz = block_size(blk);
blkcnt = block_count(blk);
if(!blksz || !blkcnt)
return 0;
capacity = block_capacity(blk);
if(offset >= capacity)
return 0;
tmp = capacity - offset;
if(count > tmp)
count = tmp;
p = malloc(blksz);
if(!p)
return 0;
blkno = offset / blksz;
tmp = offset % blksz;
if(tmp > 0)
{
len = blksz - tmp;
if(count < len)
len = count;
if(blk->read(blk, p, blkno, 1) != 1)
{
free(p);
return ret;
}
memcpy((void *)(&p[tmp]), (const void *)buf, len);
if(blk->write(blk, p, blkno, 1) != 1)
{
free(p);
return ret;
}
buf += len;
count -= len;
ret += len;
blkno += 1;
}
tmp = count / blksz;
if(tmp > 0)
{
len = tmp * blksz;
if(blk->write(blk, buf, blkno, tmp) != tmp)
{
free(p);
return ret;
}
buf += len;
count -= len;
ret += len;
blkno += tmp;
}
if(count > 0)
{
len = count;
if(blk->read(blk, p, blkno, 1) != 1)
{
free(p);
return ret;
}
memcpy((void *)(&p[0]), (const void *)buf, len);
if(blk->write(blk, p, blkno, 1) != 1)
{
free(p);
return ret;
}
ret += len;
}
free(p);
return ret;
}
static bool read_block(bool force_stream, lzma_check check, off_t uoffset) {
lzma_filter filters[LZMA_FILTERS_MAX + 1];
lzma_block block = { .filters = filters, .check = check, .version = 0 };
if (rbuf_read(1) != RBUF_FULL)
die("Error reading block header size");
if (gRbuf->input[0] == 0)
return false;
block.header_size = lzma_block_header_size_decode(gRbuf->input[0]);
if (block.header_size > LZMA_BLOCK_HEADER_SIZE_MAX)
die("Block header size too large");
if (rbuf_read(block.header_size) != RBUF_FULL)
die("Error reading block header");
if (lzma_block_header_decode(&block, NULL, gRbuf->input) != LZMA_OK)
die("Error decoding block header");
size_t comp = block.compressed_size, outsize = block.uncompressed_size;
bool sized = (comp != LZMA_VLI_UNKNOWN && outsize != LZMA_VLI_UNKNOWN);
if (force_stream || !sized || outsize > MAXSPLITSIZE) {
read_streaming(&block, sized ? BLOCK_SIZED : BLOCK_UNSIZED, uoffset);
} else {
block_capacity(gRbuf, 0, outsize);
gRbuf->outsize = outsize;
gRbuf->check = check;
gRbuf->btype = BLOCK_SIZED;
if (rbuf_read(lzma_block_total_size(&block)) != RBUF_FULL)
die("Error reading block contents");
rbuf_dispatch();
}
return true;
}
static void read_streaming(lzma_block *block, block_type sized, off_t uoffset) {
lzma_stream stream = LZMA_STREAM_INIT;
if (lzma_block_decoder(&stream, block) != LZMA_OK)
die("Error initializing streaming block decode");
rbuf_cycle(&stream, true, block->header_size);
stream.avail_out = 0;
bool first = true;
pipeline_item_t *pi = NULL;
io_block_t *ib = NULL;
lzma_ret err = LZMA_OK;
while (err != LZMA_STREAM_END) {
if (err != LZMA_OK)
die("Error decoding streaming block");
if (stream.avail_out == 0) {
if (ib) {
ib->outsize = ib->outcap;
ib->uoffset = uoffset;
uoffset += ib->outsize;
pipeline_dispatch(pi, gPipelineMergeQ);
first = false;
}
queue_pop(gPipelineStartQ, (void**)&pi);
ib = (io_block_t*)pi->data;
ib->btype = (first ? sized : BLOCK_CONTINUATION);
block_capacity(ib, 0, STREAMSIZE);
stream.next_out = ib->output;
stream.avail_out = ib->outcap;
}
if (stream.avail_in == 0 && !rbuf_cycle(&stream, false, 0))
die("Error reading streaming block");
err = lzma_code(&stream, LZMA_RUN);
}
if (ib && stream.avail_out != ib->outcap) {
ib->outsize = ib->outcap - stream.avail_out;
pipeline_dispatch(pi, gPipelineMergeQ);
}
rbuf_consume(gRbuf->insize - stream.avail_in);
lzma_end(&stream);
}
static void wanted_files(size_t count, char **specs) {
if (!gFileIndexOffset) {
if (count)
die("Can't filter non-tarball");
gWantedFiles = NULL;
return;
}
// Remove trailing slashes from specs
for (char **spec = specs; spec < specs + count; ++spec) {
char *c = *spec;
while (*c++) ; // forward to end
while (--c >= *spec && *c == '/')
*c = '\0';
}
bool matched[count]; // for each spec, does it match?
memset(matched, 0, sizeof(matched));
wanted_t *last = NULL;
// Check each file in order, to see if we want it
for (file_index_t *f = gFileIndex; f->name; f = f->next) {
bool match = !count;
for (char **spec = specs; spec < specs + count; ++spec) {
if (spec_match(*spec, f->name)) {
match = true;
matched[spec - specs] = true;
break;
}
}
if (match) {
wanted_t *w = malloc(sizeof(wanted_t));
*w = (wanted_t){ .name = f->name, .start = f->offset,
.end = f->next->offset, .next = NULL };
w->size = w->end - w->start;
if (last) {
last->next = w;
} else {
gWantedFiles = w;
}
last = w;
}
}
// Make sure each spec matched
for (size_t i = 0; i < count; ++i) {
if (!matched[i])
die("\"%s\" not found in archive", *(specs + i));
}
}
#pragma mark READ
static void block_capacity(io_block_t *ib, size_t incap, size_t outcap) {
if (incap > ib->incap) {
ib->incap = incap;
ib->input = realloc(ib->input, incap);
}
if (outcap > ib->outcap) {
ib->outcap = outcap;
ib->output = malloc(outcap);
}
}
// Ensure at least this many bytes available
// Return 1 on success, zero on EOF, -1 on error
static rbuf_read_status rbuf_read(size_t bytes) {
if (!gRbufPI) {
queue_pop(gPipelineStartQ, (void**)&gRbufPI);
gRbuf = (io_block_t*)(gRbufPI->data);
gRbuf->insize = gRbuf->outsize = 0;
}
if (gRbuf->insize >= bytes)
return RBUF_FULL;
block_capacity(gRbuf, bytes, 0);
size_t r = fread(gRbuf->input + gRbuf->insize, 1, bytes - gRbuf->insize,
gInFile);
gRbuf->insize += r;
if (r)
return (gRbuf->insize == bytes) ? RBUF_FULL : RBUF_PART;
return feof(gInFile) ? RBUF_EOF : RBUF_ERR;
}
static bool rbuf_cycle(lzma_stream *stream, bool start, size_t skip) {
if (!start) {
rbuf_consume(gRbuf->insize);
if (rbuf_read(CHUNKSIZE) < RBUF_PART)
return false;
}
stream->next_in = gRbuf->input + skip;
stream->avail_in = gRbuf->insize - skip;
return true;
}
static void rbuf_consume(size_t bytes) {
if (bytes < gRbuf->insize)
memmove(gRbuf->input, gRbuf->input + bytes, gRbuf->insize - bytes);
gRbuf->insize -= bytes;
}
static void rbuf_dispatch(void) {
pipeline_split(gRbufPI);
gRbufPI = NULL;
gRbuf = NULL;
}