string_t
stream_read_line(stream_t* stream, char delimiter) {
char buffer[128];
char* outbuffer = 0;
size_t outsize = 0;
size_t cursize = 0;
size_t read, i;
size_t want_read = 128;
if (!(stream->mode & STREAM_IN))
return (string_t) { 0, 0 };
//Need to read one byte at a time since we can't scan back if overreading
if (stream_is_sequential(stream))
want_read = 1;
while (!stream_eos(stream)) {
read = stream->vtable->read(stream, buffer, want_read);
if (!read)
break;
for (i = 0; i < read; ++i) {
if (buffer[i] == delimiter)
break;
}
if (cursize + i > outsize) {
size_t nextsize;
if (!outbuffer) {
nextsize = (i >= 32 ? i + 1 : (i > 1 ? i + 1 : 32));
outbuffer = memory_allocate(0, nextsize, 0, MEMORY_PERSISTENT);
}
else {
nextsize = (outsize < 511 ? 512 : outsize + 513); //Always aligns to 512 multiples
FOUNDATION_ASSERT(!(nextsize % 512));
outbuffer = memory_reallocate(outbuffer, nextsize, 0, outsize + 1);
}
outsize = nextsize - 1;
}
if (i) {
memcpy(outbuffer + cursize, buffer, i); //lint !e613
cursize += i;
}
if (i < read) {
if ((i + 1) < read) {
//Sequential should never end up here reading one byte at a time
FOUNDATION_ASSERT(!stream_is_sequential(stream));
stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT);
}
break;
}
}
if (outbuffer)
outbuffer[cursize] = 0;
return (string_t) { outbuffer, cursize };
}
char* stream_read_line( stream_t* stream, char delimiter )
{
char buffer[128];
char* outbuffer;
int outsize = 32;
int cursize = 0;
int read, i;
int want_read = 128;
FOUNDATION_ASSERT( stream );
if( !( stream->mode & STREAM_IN ) )
return 0;
FOUNDATION_ASSERT( stream->vtable->read );
if( stream_is_sequential( stream ) ) //Need to read one byte at a time since we can't scan back if overreading
want_read = 1;
outbuffer = memory_allocate( outsize + 1, 0, MEMORY_PERSISTENT );
while( !stream_eos( stream ) )
{
read = (int)stream->vtable->read( stream, buffer, want_read );
if( !read )
break;
for( i = 0; i < read; ++i )
{
if( buffer[i] == delimiter )
break;
}
if( cursize + i > outsize )
{
outsize += 512;
outbuffer = memory_reallocate( outbuffer, outsize + 1, 0, cursize );
}
memcpy( outbuffer + cursize, buffer, i );
cursize += i;
if( i < read )
{
if( ( i + 1 ) < read )
{
FOUNDATION_ASSERT( !stream_is_sequential( stream ) ); //Sequential should never end up here reading one byte at a time
stream_seek( stream, 1 + i - read, STREAM_SEEK_CURRENT );
}
break;
}
}
outbuffer[cursize] = 0;
return outbuffer;
}
uint64_t stream_read_line_buffer( stream_t* stream, char* dest, unsigned int count, char delimiter )
{
int i, read, total, limit;
FOUNDATION_ASSERT( stream );
FOUNDATION_ASSERT( dest );
if( !( stream->mode & STREAM_IN ) || ( count < 2 ) )
return 0;
FOUNDATION_ASSERT( stream->vtable->read );
total = 0;
--count;
while( !stream_eos( stream ) )
{
limit = count - total;
if( limit > 128 )
limit = 128;
if( !limit )
break;
if( stream_is_sequential( stream ) ) //Need to read one byte at a time since we can't scan back if overreading
limit = 1;
read = (int)stream->vtable->read( stream, dest + total, limit );
if( !read )
break;
for( i = 0; i < read; ++i )
{
if( dest[total+i] == delimiter )
break;
}
total += i;
if( i < read )
{
if( ( i + 1 ) < read )
{
FOUNDATION_ASSERT( !stream_is_sequential( stream ) ); //Sequential should never end up here reading one byte at a time
stream_seek( stream, 1 + i - read, STREAM_SEEK_CURRENT );
}
break;
}
}
dest[total] = 0;
return total;
}
void stream_determine_binary_mode( stream_t* stream, unsigned int num )
{
char* buf;
int64_t cur;
uint64_t actual_read, i;
FOUNDATION_ASSERT( stream );
if( !( stream->mode & STREAM_IN ) || stream_is_sequential( stream ) )
return;
if( !num )
num = 8;
buf = memory_allocate( num, 0, MEMORY_TEMPORARY );
memset( buf, 32, num );
cur = stream_tell( stream );
actual_read = stream_read( stream, buf, num );
stream_seek( stream, cur, STREAM_SEEK_BEGIN );
stream->mode &= ~STREAM_BINARY;
for( i = 0; i < actual_read; ++i )
{
//TODO: What about UTF-8?
if( ( ( buf[i] < 0x20 ) && ( buf[i] != 0x09 ) && ( buf[i] != 0x0a ) && ( buf[i] != 0x0d ) ) || ( buf[i] > 0x7e ) )
{
stream->mode |= STREAM_BINARY;
break;
}
}
memory_deallocate( buf );
}
void
stream_determine_binary_mode(stream_t* stream, size_t num) {
char fixed_buffer[32];
char* buf;
size_t cur;
size_t actual_read, i;
if (!(stream->mode & STREAM_IN) || stream_is_sequential(stream))
return;
if (!num)
num = 8;
buf = (num <= sizeof(fixed_buffer)) ?
fixed_buffer : memory_allocate(0, num, 0, MEMORY_TEMPORARY);
memset(buf, 32, num);
cur = stream_tell(stream);
actual_read = stream_read(stream, buf, num);
stream_seek(stream, (ssize_t)cur, STREAM_SEEK_BEGIN);
stream->mode &= ~STREAM_BINARY;
for (i = 0; i < actual_read; ++i) {
//TODO: What about UTF-8?
if (((buf[i] < 0x20) && (buf[i] != 0x09) && (buf[i] != 0x0a) && (buf[i] != 0x0d)) ||
(buf[i] > 0x7e)) {
stream->mode |= STREAM_BINARY;
break;
}
}
if (buf != fixed_buffer)
memory_deallocate(buf);
}
string_t
stream_read_line_buffer(stream_t* stream, char* dest, size_t count, char delimiter) {
size_t i, read, total, limit, hardlimit;
if (!(stream->mode & STREAM_IN) || !dest || (count < 2)) {
if (dest && count)
dest[0] = 0;
return (string_t) { dest, 0 };
}
total = 0;
hardlimit = stream_is_sequential(stream) ? 1 : 128;
//Need to read one byte at a time since we can't scan back if overreading in sequential streams
--count;
while (!stream_eos(stream)) {
limit = count - total;
if (limit > hardlimit)
limit = hardlimit;
if (!limit)
break;
//This will initialize range [total,total+read) in dest array, making
//access of potentially uninitialized dest array safe (see coverity markup below)
read = stream->vtable->read(stream, dest + total, limit);
if (!read)
break;
for (i = 0; i < read; ++i) {
/* coverity[read_parm] */
if (dest[total + i] == delimiter)
break;
}
total += i;
if (i < read) {
if ((i + 1) < read) {
//Sequential should never end up here reading one byte at a time
FOUNDATION_ASSERT(!stream_is_sequential(stream));
stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT);
}
break;
}
}
dest[total] = 0;
return (string_t) { dest, total };
}
uint128_t stream_md5( stream_t* stream )
{
int64_t cur, ic, offset, lastc, num;
md5_t* md5;
uint128_t ret = {0};
unsigned char buf[1025];
bool ignore_lf = false;
FOUNDATION_ASSERT( stream );
if( stream->vtable->md5 )
return stream->vtable->md5( stream );
if( !stream || stream_is_sequential( stream ) || !( stream->mode & STREAM_IN ) )
return ret;
FOUNDATION_ASSERT( stream->vtable->read );
cur = stream_tell( stream );
stream_seek( stream, 0, STREAM_SEEK_BEGIN );
md5 = md5_allocate();
while( !stream_eos( stream ) )
{
num = (int64_t)stream->vtable->read( stream, buf, 1024 );
if( !num )
continue;
if( stream->mode & STREAM_BINARY )
md5_digest_raw( md5, buf, (size_t)num );
else
{
//If last buffer ended with CR, ignore a leading LF
offset = 0;
lastc = 0;
if( ignore_lf && ( buf[0] == '\n' ) )
offset = 1;
ignore_lf = false;
//Treat all line endings (LF, CR, CR+LF) as Unix style LF. If file has mixed line endings (for example, one line ending in a single CR and next ending in a single LF), it will not work!
for( ic = 0; ic < num; ++ic )
{
bool was_cr = ( buf[ic] == '\r' );
bool was_lf = ( buf[ic] == '\n' );
if( was_cr || was_lf )
{
if( was_cr && ( ic >= 1023 ) )
ignore_lf = true;
buf[ic] = '\n';
md5_digest_raw( md5, buf + lastc, (size_t)( ic - lastc + 1 ) ); //Include the LF
if( was_cr && ( buf[ic+1] == '\n' ) ) //Check for CR+LF
++ic;
lastc = ic + 1;
}
}
if( lastc < num )
md5_digest_raw( md5, buf + lastc, (size_t)( num - lastc ) );
}
}
stream_seek( stream, cur, STREAM_SEEK_BEGIN );
md5_finalize( md5 );
ret = md5_get_digest_raw( md5 );
md5_deallocate( md5 );
return ret;
}
char* stream_read_string( stream_t* stream )
{
char buffer[128];
char* outbuffer;
int outsize = 128;
int cursize = 0;
int read, i;
bool binary = stream_is_binary( stream );
FOUNDATION_ASSERT( stream );
if( !( stream->mode & STREAM_IN ) )
return 0;
FOUNDATION_ASSERT( stream->vtable->read );
outbuffer = memory_allocate( outsize, 0, MEMORY_PERSISTENT );
if( stream_is_sequential( stream ) )
{
//Single byte reading since we can't seek backwards (and don't want to block on network sockets)
char c;
if( !binary )
{
//Consume whitespace
while( !stream_eos( stream ) )
{
read = (int)stream->vtable->read( stream, &c, 1 );
if( !read )
break;
if( ( c != ' ' ) && ( c != '\n' ) && ( c != '\r' ) && ( c != '\t' ) )
{
outbuffer[cursize++] = c;
break;
}
}
}
while( !stream_eos( stream ) )
{
read = (int)stream->vtable->read( stream, &c, 1 );
if( !read )
break;
if( !c )
break;
if( !binary && ( ( c == ' ' ) || ( c == '\n' ) || ( c == '\r' ) || ( c == '\t' ) ) )
break;
if( cursize + 1 > outsize )
{
outsize += 512;
outbuffer = memory_reallocate( outbuffer, outsize, 0, cursize );
}
outbuffer[cursize++] = c;
}
outbuffer[cursize] = 0;
}
else
{
if( !binary )
{
//Consume whitespace
while( !stream_eos( stream ) )
{
read = (int)stream->vtable->read( stream, buffer, 16 );
if( !read )
break;
for( i = 0; i < read; ++i )
{
char c = buffer[i];
if( ( c != ' ' ) && ( c != '\n' ) && ( c != '\r' ) && ( c != '\t' ) )
break;
}
if( i < read )
{
stream_seek( stream, i - read, STREAM_SEEK_CURRENT );
break;
}
}
}
while( !stream_eos( stream ) )
{
read = (int)stream->vtable->read( stream, buffer, 128 );
if( !read )
break;
for( i = 0; i < read; ++i )
{
char c = buffer[i];
if( !c )
break;
if( !binary && ( ( c == ' ' ) || ( c == '\n' ) || ( c == '\r' ) || ( c == '\t' ) ) )
break;
}
if( i )
{
if( cursize + i > outsize )
{
outsize += 512;
outbuffer = memory_reallocate( outbuffer, outsize, 0, cursize );
}
memcpy( outbuffer + cursize, buffer, i );
cursize += i;
}
if( i < 128 )
{
if( ( i + 1 ) < read )
stream_seek( stream, 1 + i - read, STREAM_SEEK_CURRENT );
break;
}
}
outbuffer[cursize] = 0;
}
return outbuffer;
}
uint128_t
stream_md5(stream_t* stream) {
size_t cur, ic, lastc, num, limit;
md5_t md5;
uint128_t ret = uint128_null();
unsigned char buf[1025];
bool ignore_lf = false;
if (stream->vtable->md5)
return stream->vtable->md5(stream);
if (stream_is_sequential(stream) || !(stream->mode & STREAM_IN))
return ret;
cur = stream_tell(stream);
stream_seek(stream, 0, STREAM_SEEK_BEGIN);
md5_initialize(&md5);
limit = sizeof(buf)-1;
buf[limit] = 0;
while (!stream_eos(stream)) {
num = stream->vtable->read(stream, buf, limit);
if (!num)
continue;
if (stream->mode & STREAM_BINARY)
md5_digest(&md5, buf, (size_t)num);
else {
//If last buffer ended with CR, ignore a leading LF
lastc = 0;
if (ignore_lf && (buf[0] == '\n'))
lastc = 1;
ignore_lf = false;
//Digest one line at a time
//Treat all line endings (LF, CR, CR+LF) as Unix style LF. If file has mixed line endings
//(for example, one line ending in a single CR and next is empty and ending in a single LF),
//it will not work!
/*lint -e{850} */
for (ic = lastc; ic < num && ic < limit; ++ic) {
bool was_cr = (buf[ic] == '\r');
bool was_lf = (buf[ic] == '\n');
if (was_cr || was_lf) {
if (was_cr && (ic == limit-1))
ignore_lf = true; //Make next buffer ignore leading LF as it is part of CR+LF
buf[ic] = '\n';
md5_digest(&md5, buf + lastc, (size_t)((ic - lastc) + 1)); //Include the LF
if (was_cr && (buf[ic + 1] == '\n')) //Check for CR+LF
++ic;
lastc = ic + 1;
}
}
if (lastc < num)
md5_digest(&md5, buf + lastc, (size_t)(num - lastc));
}
}
stream_seek(stream, (ssize_t)cur, STREAM_SEEK_BEGIN);
md5_digest_finalize(&md5);
ret = md5_get_digest_raw(&md5);
md5_finalize(&md5);
return ret;
}
string_t
stream_read_string_buffer(stream_t* stream, char* outbuffer, size_t size) {
char buffer[128];
size_t cursize = 0;
size_t read, i;
bool binary = stream_is_binary(stream);
if (!(stream->mode & STREAM_IN) || !outbuffer || !size) {
if (outbuffer && size)
outbuffer[0] = 0;
return (string_t) { outbuffer, 0 };
}
--size;
if (stream_is_sequential(stream)) {
//Single byte reading since we can't seek backwards (and don't want to block on network sockets)
char c;
if (!binary) {
//Consume whitespace
while (!stream_eos(stream)) {
read = stream->vtable->read(stream, &c, 1);
if (!read)
break;
if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t')) {
outbuffer[cursize++] = c;
break;
}
}
}
if (cursize > 0) {
while (!stream_eos(stream) && (cursize < size)) {
read = stream->vtable->read(stream, &c, 1);
if (!read)
break;
if (!c)
break;
if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t')))
break;
outbuffer[cursize++] = c;
}
}
}
else {
if (!binary) {
//Consume whitespace
while (!stream_eos(stream)) {
read = stream->vtable->read(stream, buffer, 16);
if (!read)
break;
for (i = 0; i < read; ++i) {
char c = buffer[i];
if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t'))
break;
}
if (i < read) {
stream_seek(stream, (ssize_t)i - (ssize_t)read, STREAM_SEEK_CURRENT);
break;
}
}
}
while (!stream_eos(stream) && (cursize < size)) {
read = stream->vtable->read(stream, buffer, 128);
if (!read)
break;
for (i = 0; i < read; ++i) {
char c = buffer[i];
if (!c)
break;
if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t')))
break;
}
if (!i)
break;
if (cursize + i > size)
i = size - cursize;
memcpy(outbuffer + cursize, buffer, i);
cursize += i;
if (i < 128) {
if ((i + 1) < read)
stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT);
break;
}
}
}
if (cursize < size)
outbuffer[cursize] = 0;
return (string_t) {outbuffer, cursize};
}
string_t
stream_read_string(stream_t* stream) {
char buffer[128];
char* outbuffer = buffer;
size_t outsize = sizeof(buffer);
size_t cursize = 0;
size_t read, i;
bool binary = stream_is_binary(stream);
if (!(stream->mode & STREAM_IN))
return (string_t) { 0, 0 };
if (stream_is_sequential(stream)) {
//Single byte reading since we can't seek backwards (and don't want to block on network sockets)
char c;
if (!binary) {
//Consume whitespace
while (!stream_eos(stream)) {
read = stream->vtable->read(stream, &c, 1);
if (!read)
break;
if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t')) {
buffer[cursize++] = c;
break;
}
}
}
if (cursize > 0) {
while (!stream_eos(stream)) {
read = stream->vtable->read(stream, &c, 1);
if (!read)
break;
if (!c)
break;
if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t')))
break;
if (cursize + 1 >= outsize) {
outsize += 512;
if (outbuffer != buffer) {
outbuffer = memory_reallocate(outbuffer, outsize, 0, cursize);
}
else {
outbuffer = memory_allocate(0, outsize, 0, MEMORY_PERSISTENT);
memcpy(outbuffer, buffer, sizeof(buffer));
}
}
outbuffer[cursize++] = c;
}
outbuffer[cursize] = 0; //lint !e661
}
}
else {
if (!binary) {
//Consume whitespace
while (!stream_eos(stream)) {
read = stream->vtable->read(stream, buffer, 16);
if (!read)
break;
for (i = 0; i < read; ++i) {
char c = buffer[i];
if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t'))
break;
}
if (i < read) {
stream_seek(stream, (ssize_t)i - (ssize_t)read, STREAM_SEEK_CURRENT);
break;
}
}
}
while (!stream_eos(stream)) {
if (outbuffer != buffer)
read = stream->vtable->read(stream, buffer, sizeof(buffer));
else
read = stream->vtable->read(stream, buffer + cursize, sizeof(buffer) - cursize);
if (!read)
break;
for (i = 0; i < read; ++i) {
char c = buffer[i];
if (!c)
break;
if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t')))
break;
}
if (i) {
if (cursize + i >= outsize) {
outsize += 512;
if (outbuffer != buffer) {
outbuffer = memory_reallocate(outbuffer, outsize, 0, cursize);
}
else {
FOUNDATION_ASSERT(cursize == 0); //Or internal assumptions about code flow is incorrect
outbuffer = memory_allocate(0, outsize, 0, MEMORY_PERSISTENT);
memcpy(outbuffer, buffer, i);
}
}
else if (outbuffer != buffer)
memcpy(outbuffer + cursize, buffer, i);
cursize += i;
}
if (i < sizeof(buffer)) {
if ((i + 1) < read)
stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT);
break;
}
}
outbuffer[cursize] = 0;
}
if (outbuffer == buffer) {
if (cursize == 0)
return (string_t) { 0, 0 };
outbuffer = memory_allocate(0, cursize + 1, 0, MEMORY_PERSISTENT);
memcpy(outbuffer, buffer, cursize);
outbuffer[cursize] = 0;
}
return (string_t) { outbuffer, cursize };
}
