InStream*
InStream_Reopen_IMP(InStream *self, String *filename, int64_t offset,
int64_t len) {
InStreamIVARS *const ivars = InStream_IVARS(self);
if (!ivars->file_handle) {
THROW(ERR, "Can't Reopen() closed InStream %o", ivars->filename);
}
if (offset + len > FH_Length(ivars->file_handle)) {
THROW(ERR, "Offset + length too large (%i64 + %i64 > %i64)",
offset, len, FH_Length(ivars->file_handle));
}
Class *klass = InStream_get_class(self);
InStream *other = (InStream*)Class_Make_Obj(klass);
InStreamIVARS *const ovars = InStream_IVARS(other);
InStream_do_open(other, (Obj*)ivars->file_handle);
if (filename != NULL) {
String *temp = ovars->filename;
ovars->filename = Str_Clone(filename);
DECREF(temp);
}
ovars->offset = offset;
ovars->len = len;
InStream_Seek(other, 0);
return other;
}
static void
test_refill(TestBatchRunner *runner) {
RAMFile *file = RAMFile_new(NULL, false);
OutStream *outstream = OutStream_open((Obj*)file);
InStream *instream;
char scratch[5];
InStreamIVARS *ivars;
for (int32_t i = 0; i < 1023; i++) {
OutStream_Write_U8(outstream, 'x');
}
OutStream_Write_U8(outstream, 'y');
OutStream_Write_U8(outstream, 'z');
OutStream_Close(outstream);
instream = InStream_open((Obj*)file);
ivars = InStream_IVARS(instream);
InStream_Refill(instream);
TEST_INT_EQ(runner, ivars->limit - ivars->buf, IO_STREAM_BUF_SIZE,
"Refill");
TEST_INT_EQ(runner, (long)InStream_Tell(instream), 0,
"Correct file pos after standing-start Refill()");
DECREF(instream);
instream = InStream_open((Obj*)file);
ivars = InStream_IVARS(instream);
InStream_Fill(instream, 30);
TEST_INT_EQ(runner, ivars->limit - ivars->buf, 30, "Fill()");
TEST_INT_EQ(runner, (long)InStream_Tell(instream), 0,
"Correct file pos after standing-start Fill()");
DECREF(instream);
instream = InStream_open((Obj*)file);
ivars = InStream_IVARS(instream);
InStream_Read_Bytes(instream, scratch, 5);
TEST_INT_EQ(runner, ivars->limit - ivars->buf,
IO_STREAM_BUF_SIZE - 5, "small read triggers refill");
DECREF(instream);
instream = InStream_open((Obj*)file);
ivars = InStream_IVARS(instream);
TEST_INT_EQ(runner, InStream_Read_U8(instream), 'x', "Read_U8");
InStream_Seek(instream, 1023);
TEST_INT_EQ(runner, (long)FileWindow_IVARS(ivars->window)->offset, 0,
"no unnecessary refill on Seek");
TEST_INT_EQ(runner, (long)InStream_Tell(instream), 1023, "Seek/Tell");
TEST_INT_EQ(runner, InStream_Read_U8(instream), 'y',
"correct data after in-buffer Seek()");
TEST_INT_EQ(runner, InStream_Read_U8(instream), 'z', "automatic Refill");
TEST_TRUE(runner, (FileWindow_IVARS(ivars->window)->offset != 0),
"refilled");
DECREF(instream);
DECREF(outstream);
DECREF(file);
}
static void
S_fill(InStream *self, int64_t amount) {
InStreamIVARS *const ivars = InStream_IVARS(self);
FileWindow *const window = ivars->window;
const int64_t virtual_file_pos = SI_tell(self);
const int64_t real_file_pos = virtual_file_pos + ivars->offset;
const int64_t remaining = ivars->len - virtual_file_pos;
// Throw an error if the requested amount would take us beyond EOF.
if (amount > remaining) {
THROW(ERR, "Read past EOF of %o (pos: %u64 len: %u64 request: %u64)",
ivars->filename, virtual_file_pos, ivars->len, amount);
}
// Make the request.
if (FH_Window(ivars->file_handle, window, real_file_pos, amount)) {
char *fw_buf = FileWindow_Get_Buf(window);
int64_t fw_offset = FileWindow_Get_Offset(window);
int64_t fw_len = FileWindow_Get_Len(window);
char *const window_limit = fw_buf + fw_len;
ivars->buf = fw_buf
- fw_offset // theoretical start of real file
+ ivars->offset // top of virtual file
+ virtual_file_pos; // position within virtual file
ivars->limit = window_limit - ivars->buf > remaining
? ivars->buf + remaining
: window_limit;
}
else {
Err *error = Err_get_error();
CB_catf(Err_Get_Mess(error), " (%o)", ivars->filename);
RETHROW(INCREF(error));
}
}
void
InStream_seek(InStream *self, int64_t target) {
InStreamIVARS *const ivars = InStream_IVARS(self);
FileWindow *const window = ivars->window;
char *fw_buf = FileWindow_Get_Buf(window);
int64_t fw_offset = FileWindow_Get_Offset(window);
int64_t fw_len = FileWindow_Get_Len(window);
int64_t virtual_window_top = fw_offset - ivars->offset;
int64_t virtual_window_end = virtual_window_top + fw_len;
if (target < 0) {
THROW(ERR, "Can't Seek '%o' to negative target %i64", ivars->filename,
target);
}
// Seek within window if possible.
else if (target >= virtual_window_top
&& target <= virtual_window_end
) {
ivars->buf = fw_buf - fw_offset + ivars->offset + target;
}
else if (target > ivars->len) {
THROW(ERR, "Can't Seek '%o' past EOF (%i64 > %i64)", ivars->filename,
target, ivars->len);
}
else {
// Target is outside window. Set all buffer and limit variables to
// NULL to trigger refill on the next read. Store the file position
// in the FileWindow's offset.
FH_Release_Window(ivars->file_handle, window);
ivars->buf = NULL;
ivars->limit = NULL;
FileWindow_Set_Offset(window, ivars->offset + target);
}
}
static INLINE int64_t
SI_tell(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
char *fw_buf = FileWindow_Get_Buf(ivars->window);
int64_t pos_in_buf = PTR_TO_I64(ivars->buf) - PTR_TO_I64(fw_buf);
return pos_in_buf + FileWindow_Get_Offset(ivars->window) - ivars->offset;
}
char*
InStream_buf(InStream *self, size_t request) {
InStreamIVARS *const ivars = InStream_IVARS(self);
const int64_t bytes_in_buf = PTR_TO_I64(ivars->limit) - PTR_TO_I64(ivars->buf);
/* It's common for client code to overestimate how much is needed, because
* the request has to figure in worst-case for compressed data. However,
* if we can still serve them everything they request (e.g. they ask for 5
* bytes, they really need 1 byte, and there's 1k in the buffer), we can
* skip the following refill block. */
if ((int64_t)request > bytes_in_buf) {
const int64_t remaining_in_file = ivars->len - SI_tell(self);
int64_t amount = request;
// Try to bump up small requests.
if (amount < IO_STREAM_BUF_SIZE) { amount = IO_STREAM_BUF_SIZE; }
// Don't read past EOF.
if (remaining_in_file < amount) { amount = remaining_in_file; }
// Only fill if the recalculated, possibly smaller request exceeds the
// amount available in the buffer.
if (amount > bytes_in_buf) {
S_fill(self, amount);
}
}
return ivars->buf;
}
static void
test_Seek_and_Tell(TestBatchRunner *runner) {
int64_t gb1 = INT64_C(0x40000000);
int64_t gb3 = gb1 * 3;
int64_t gb6 = gb1 * 6;
int64_t gb12 = gb1 * 12;
FileHandle *fh = (FileHandle*)MockFileHandle_new(NULL, gb12);
InStream *instream = InStream_open((Obj*)fh);
InStreamIVARS *const ivars = InStream_IVARS(instream);
InStream_Buf(instream, 10000);
TEST_TRUE(runner, ivars->limit == ((char*)NULL) + 10000,
"InStream_Buf sets limit");
InStream_Seek(instream, gb6);
TEST_TRUE(runner, InStream_Tell(instream) == gb6,
"Tell after seek forwards outside buffer");
TEST_TRUE(runner, ivars->buf == NULL,
"Seek forwards outside buffer sets buf to NULL");
TEST_TRUE(runner, ivars->limit == NULL,
"Seek forwards outside buffer sets limit to NULL");
TEST_TRUE(runner, FileWindow_IVARS(ivars->window)->offset == gb6,
"Seek forwards outside buffer tracks pos in window offset");
InStream_Buf(instream, (size_t)gb1);
TEST_TRUE(runner, ivars->limit == ((char*)NULL) + gb1,
"InStream_Buf sets limit");
InStream_Seek(instream, gb6 + 10);
TEST_TRUE(runner, InStream_Tell(instream) == gb6 + 10,
"Tell after seek forwards within buffer");
TEST_TRUE(runner, ivars->buf == ((char*)NULL) + 10,
"Seek within buffer sets buf");
TEST_TRUE(runner, ivars->limit == ((char*)NULL) + gb1,
"Seek within buffer leaves limit alone");
InStream_Seek(instream, gb6 + 1);
TEST_TRUE(runner, InStream_Tell(instream) == gb6 + 1,
"Tell after seek backwards within buffer");
TEST_TRUE(runner, ivars->buf == ((char*)NULL) + 1,
"Seek backwards within buffer sets buf");
TEST_TRUE(runner, ivars->limit == ((char*)NULL) + gb1,
"Seek backwards within buffer leaves limit alone");
InStream_Seek(instream, gb3);
TEST_TRUE(runner, InStream_Tell(instream) == gb3,
"Tell after seek backwards outside buffer");
TEST_TRUE(runner, ivars->buf == NULL,
"Seek backwards outside buffer sets buf to NULL");
TEST_TRUE(runner, ivars->limit == NULL,
"Seek backwards outside buffer sets limit to NULL");
TEST_TRUE(runner, FileWindow_IVARS(ivars->window)->offset == gb3,
"Seek backwards outside buffer tracks pos in window offset");
DECREF(instream);
DECREF(fh);
}
int
InStream_read_raw_c64(InStream *self, char *buf) {
InStreamIVARS *const ivars = InStream_IVARS(self);
uint8_t *dest = (uint8_t*)buf;
do {
*dest = SI_read_u8(self, ivars);
} while ((*dest++ & 0x80) != 0);
return dest - (uint8_t*)buf;
}
InStream*
InStream_Clone_IMP(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
Class *klass = InStream_get_class(self);
InStream *twin = (InStream*)Class_Make_Obj(klass);
InStream_do_open(twin, (Obj*)ivars->file_handle);
InStream_Seek(twin, SI_tell(self));
return twin;
}
InStream*
InStream_clone(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
VTable *vtable = InStream_Get_VTable(self);
InStream *twin = (InStream*)VTable_Make_Obj(vtable);
InStream_do_open(twin, (Obj*)ivars->file_handle);
InStream_Seek(twin, SI_tell(self));
return twin;
}
void
InStream_destroy(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
if (ivars->file_handle) {
InStream_Close(self);
}
DECREF(ivars->filename);
DECREF(ivars->window);
SUPER_DESTROY(self, INSTREAM);
}
static void
test_Close(TestBatchRunner *runner) {
RAMFile *file = RAMFile_new(NULL, false);
InStream *instream = InStream_open((Obj*)file);
InStream_Close(instream);
TEST_TRUE(runner, InStream_IVARS(instream)->file_handle == NULL,
"Close decrements FileHandle's refcount");
DECREF(instream);
DECREF(file);
}
void
InStream_close(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
if (ivars->file_handle) {
FH_Release_Window(ivars->file_handle, ivars->window);
// Note that we don't close the FileHandle, because it's probably
// shared.
DECREF(ivars->file_handle);
ivars->file_handle = NULL;
}
}
uint64_t
InStream_read_c64(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
uint64_t retval = 0;
while (1) {
const uint8_t ubyte = SI_read_u8(self, ivars);
retval = (retval << 7) | (ubyte & 0x7f);
if ((ubyte & 0x80) == 0) {
break;
}
}
return retval;
}
InStream*
InStream_reopen(InStream *self, const CharBuf *filename, int64_t offset,
int64_t len) {
InStreamIVARS *const ivars = InStream_IVARS(self);
if (!ivars->file_handle) {
THROW(ERR, "Can't Reopen() closed InStream %o", ivars->filename);
}
if (offset + len > FH_Length(ivars->file_handle)) {
THROW(ERR, "Offset + length too large (%i64 + %i64 > %i64)",
offset, len, FH_Length(ivars->file_handle));
}
VTable *vtable = InStream_Get_VTable(self);
InStream *other = (InStream*)VTable_Make_Obj(vtable);
InStreamIVARS *const ovars = InStream_IVARS(other);
InStream_do_open(other, (Obj*)ivars->file_handle);
if (filename != NULL) { CB_Mimic(ovars->filename, (Obj*)filename); }
ovars->offset = offset;
ovars->len = len;
InStream_Seek(other, 0);
return other;
}
void
InStream_advance_buf(InStream *self, char *buf) {
InStreamIVARS *const ivars = InStream_IVARS(self);
if (buf > ivars->limit) {
int64_t overrun = PTR_TO_I64(buf) - PTR_TO_I64(ivars->limit);
THROW(ERR, "Supplied value is %i64 bytes beyond end of buffer",
overrun);
}
else if (buf < ivars->buf) {
int64_t underrun = PTR_TO_I64(ivars->buf) - PTR_TO_I64(buf);
THROW(ERR, "Can't Advance_Buf backwards: (underrun: %i64))", underrun);
}
else {
ivars->buf = buf;
}
}
static CFISH_INLINE void
SI_read_bytes(InStream *self, char* buf, size_t len) {
InStreamIVARS *const ivars = InStream_IVARS(self);
const int64_t available
= CHY_PTR_TO_I64(ivars->limit) - CHY_PTR_TO_I64(ivars->buf);
if (available >= (int64_t)len) {
// Request is entirely within buffer, so copy.
memcpy(buf, ivars->buf, len);
ivars->buf += len;
}
else {
// Pass along whatever we've got in the buffer.
if (available > 0) {
memcpy(buf, ivars->buf, (size_t)available);
buf += available;
len -= (size_t)available;
ivars->buf += available;
}
if (len < IO_STREAM_BUF_SIZE) {
// Ensure that we have enough mapped, then copy the rest.
int64_t got = S_refill(self);
if (got < (int64_t)len) {
int64_t orig_pos = SI_tell(self) - available;
int64_t orig_len = len + available;
THROW(ERR, "Read past EOF of %o (pos: %i64 len: %i64 "
"request: %i64)", ivars->filename, orig_pos,
ivars->len, orig_len);
}
memcpy(buf, ivars->buf, len);
ivars->buf += len;
}
else {
// Too big to handle via the buffer, so resort to a brute-force
// read.
const int64_t sub_file_pos = SI_tell(self);
const int64_t real_file_pos = sub_file_pos + ivars->offset;
bool success
= FH_Read(ivars->file_handle, buf, real_file_pos, len);
if (!success) {
RETHROW(INCREF(Err_get_error()));
}
InStream_Seek_IMP(self, sub_file_pos + len);
}
}
}
InStream*
InStream_do_open(InStream *self, Obj *file) {
InStreamIVARS *const ivars = InStream_IVARS(self);
// Init.
ivars->buf = NULL;
ivars->limit = NULL;
ivars->offset = 0;
ivars->window = FileWindow_new();
// Obtain a FileHandle.
if (Obj_Is_A(file, FILEHANDLE)) {
ivars->file_handle = (FileHandle*)INCREF(file);
}
else if (Obj_Is_A(file, RAMFILE)) {
ivars->file_handle
= (FileHandle*)RAMFH_open(NULL, FH_READ_ONLY, (RAMFile*)file);
}
else if (Obj_Is_A(file, CHARBUF)) {
ivars->file_handle
= (FileHandle*)FSFH_open((CharBuf*)file, FH_READ_ONLY);
}
else {
Err_set_error(Err_new(CB_newf("Invalid type for param 'file': '%o'",
Obj_Get_Class_Name(file))));
DECREF(self);
return NULL;
}
if (!ivars->file_handle) {
ERR_ADD_FRAME(Err_get_error());
DECREF(self);
return NULL;
}
// Get length and filename from the FileHandle.
ivars->filename = CB_Clone(FH_Get_Path(ivars->file_handle));
ivars->len = FH_Length(ivars->file_handle);
if (ivars->len == -1) {
ERR_ADD_FRAME(Err_get_error());
DECREF(self);
return NULL;
}
return self;
}
static int64_t
S_refill(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
// Determine the amount to request.
const int64_t sub_file_pos = SI_tell(self);
const int64_t remaining = ivars->len - sub_file_pos;
const int64_t amount = remaining < IO_STREAM_BUF_SIZE
? remaining
: IO_STREAM_BUF_SIZE;
if (!remaining) {
THROW(ERR, "Read past EOF of '%o' (offset: %i64 len: %i64)",
ivars->filename, ivars->offset, ivars->len);
}
// Make the request.
S_fill(self, amount);
return amount;
}
static void
test_Buf(TestBatchRunner *runner) {
RAMFile *file = RAMFile_new(NULL, false);
OutStream *outstream = OutStream_open((Obj*)file);
size_t size = IO_STREAM_BUF_SIZE * 2 + 5;
InStream *instream;
char *buf;
for (uint32_t i = 0; i < size; i++) {
OutStream_Write_U8(outstream, 'a');
}
OutStream_Close(outstream);
instream = InStream_open((Obj*)file);
InStreamIVARS *const ivars = InStream_IVARS(instream);
buf = InStream_Buf(instream, 5);
TEST_INT_EQ(runner, ivars->limit - buf, IO_STREAM_BUF_SIZE,
"Small request bumped up");
buf += IO_STREAM_BUF_SIZE - 10; // 10 bytes left in buffer.
InStream_Advance_Buf(instream, buf);
buf = InStream_Buf(instream, 10);
TEST_INT_EQ(runner, ivars->limit - buf, 10,
"Exact request doesn't trigger refill");
buf = InStream_Buf(instream, 11);
TEST_INT_EQ(runner, ivars->limit - buf, IO_STREAM_BUF_SIZE,
"Requesting over limit triggers refill");
int64_t expected = InStream_Length(instream) - InStream_Tell(instream);
char *buff = InStream_Buf(instream, 100000);
int64_t got = PTR_TO_I64(ivars->limit) - PTR_TO_I64(buff);
TEST_TRUE(runner, got == expected,
"Requests greater than file size get pared down");
DECREF(instream);
DECREF(outstream);
DECREF(file);
}
String*
InStream_Get_Filename_IMP(InStream *self) {
return InStream_IVARS(self)->filename;
}
uint8_t
InStream_read_u8(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
return SI_read_u8(self, ivars);
}
int64_t
InStream_Length_IMP(InStream *self) {
return InStream_IVARS(self)->len;
}
int64_t
InStream_length(InStream *self) {
return InStream_IVARS(self)->len;
}
int8_t
InStream_Read_I8_IMP(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
return (int8_t)SI_read_u8(self, ivars);
}
CharBuf*
InStream_get_filename(InStream *self) {
return InStream_IVARS(self)->filename;
}
