string_t stream_read_line(stream_t* stream, char delimiter) { char buffer[128]; char* outbuffer = 0; size_t outsize = 0; size_t cursize = 0; size_t read, i; size_t want_read = 128; if (!(stream->mode & STREAM_IN)) return (string_t) { 0, 0 }; //Need to read one byte at a time since we can't scan back if overreading if (stream_is_sequential(stream)) want_read = 1; while (!stream_eos(stream)) { read = stream->vtable->read(stream, buffer, want_read); if (!read) break; for (i = 0; i < read; ++i) { if (buffer[i] == delimiter) break; } if (cursize + i > outsize) { size_t nextsize; if (!outbuffer) { nextsize = (i >= 32 ? i + 1 : (i > 1 ? i + 1 : 32)); outbuffer = memory_allocate(0, nextsize, 0, MEMORY_PERSISTENT); } else { nextsize = (outsize < 511 ? 512 : outsize + 513); //Always aligns to 512 multiples FOUNDATION_ASSERT(!(nextsize % 512)); outbuffer = memory_reallocate(outbuffer, nextsize, 0, outsize + 1); } outsize = nextsize - 1; } if (i) { memcpy(outbuffer + cursize, buffer, i); //lint !e613 cursize += i; } if (i < read) { if ((i + 1) < read) { //Sequential should never end up here reading one byte at a time FOUNDATION_ASSERT(!stream_is_sequential(stream)); stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT); } break; } } if (outbuffer) outbuffer[cursize] = 0; return (string_t) { outbuffer, cursize }; }
char* stream_read_line( stream_t* stream, char delimiter ) { char buffer[128]; char* outbuffer; int outsize = 32; int cursize = 0; int read, i; int want_read = 128; FOUNDATION_ASSERT( stream ); if( !( stream->mode & STREAM_IN ) ) return 0; FOUNDATION_ASSERT( stream->vtable->read ); if( stream_is_sequential( stream ) ) //Need to read one byte at a time since we can't scan back if overreading want_read = 1; outbuffer = memory_allocate( outsize + 1, 0, MEMORY_PERSISTENT ); while( !stream_eos( stream ) ) { read = (int)stream->vtable->read( stream, buffer, want_read ); if( !read ) break; for( i = 0; i < read; ++i ) { if( buffer[i] == delimiter ) break; } if( cursize + i > outsize ) { outsize += 512; outbuffer = memory_reallocate( outbuffer, outsize + 1, 0, cursize ); } memcpy( outbuffer + cursize, buffer, i ); cursize += i; if( i < read ) { if( ( i + 1 ) < read ) { FOUNDATION_ASSERT( !stream_is_sequential( stream ) ); //Sequential should never end up here reading one byte at a time stream_seek( stream, 1 + i - read, STREAM_SEEK_CURRENT ); } break; } } outbuffer[cursize] = 0; return outbuffer; }
uint64_t stream_read_line_buffer( stream_t* stream, char* dest, unsigned int count, char delimiter ) { int i, read, total, limit; FOUNDATION_ASSERT( stream ); FOUNDATION_ASSERT( dest ); if( !( stream->mode & STREAM_IN ) || ( count < 2 ) ) return 0; FOUNDATION_ASSERT( stream->vtable->read ); total = 0; --count; while( !stream_eos( stream ) ) { limit = count - total; if( limit > 128 ) limit = 128; if( !limit ) break; if( stream_is_sequential( stream ) ) //Need to read one byte at a time since we can't scan back if overreading limit = 1; read = (int)stream->vtable->read( stream, dest + total, limit ); if( !read ) break; for( i = 0; i < read; ++i ) { if( dest[total+i] == delimiter ) break; } total += i; if( i < read ) { if( ( i + 1 ) < read ) { FOUNDATION_ASSERT( !stream_is_sequential( stream ) ); //Sequential should never end up here reading one byte at a time stream_seek( stream, 1 + i - read, STREAM_SEEK_CURRENT ); } break; } } dest[total] = 0; return total; }
void stream_determine_binary_mode( stream_t* stream, unsigned int num ) { char* buf; int64_t cur; uint64_t actual_read, i; FOUNDATION_ASSERT( stream ); if( !( stream->mode & STREAM_IN ) || stream_is_sequential( stream ) ) return; if( !num ) num = 8; buf = memory_allocate( num, 0, MEMORY_TEMPORARY ); memset( buf, 32, num ); cur = stream_tell( stream ); actual_read = stream_read( stream, buf, num ); stream_seek( stream, cur, STREAM_SEEK_BEGIN ); stream->mode &= ~STREAM_BINARY; for( i = 0; i < actual_read; ++i ) { //TODO: What about UTF-8? if( ( ( buf[i] < 0x20 ) && ( buf[i] != 0x09 ) && ( buf[i] != 0x0a ) && ( buf[i] != 0x0d ) ) || ( buf[i] > 0x7e ) ) { stream->mode |= STREAM_BINARY; break; } } memory_deallocate( buf ); }
void stream_determine_binary_mode(stream_t* stream, size_t num) { char fixed_buffer[32]; char* buf; size_t cur; size_t actual_read, i; if (!(stream->mode & STREAM_IN) || stream_is_sequential(stream)) return; if (!num) num = 8; buf = (num <= sizeof(fixed_buffer)) ? fixed_buffer : memory_allocate(0, num, 0, MEMORY_TEMPORARY); memset(buf, 32, num); cur = stream_tell(stream); actual_read = stream_read(stream, buf, num); stream_seek(stream, (ssize_t)cur, STREAM_SEEK_BEGIN); stream->mode &= ~STREAM_BINARY; for (i = 0; i < actual_read; ++i) { //TODO: What about UTF-8? if (((buf[i] < 0x20) && (buf[i] != 0x09) && (buf[i] != 0x0a) && (buf[i] != 0x0d)) || (buf[i] > 0x7e)) { stream->mode |= STREAM_BINARY; break; } } if (buf != fixed_buffer) memory_deallocate(buf); }
string_t stream_read_line_buffer(stream_t* stream, char* dest, size_t count, char delimiter) { size_t i, read, total, limit, hardlimit; if (!(stream->mode & STREAM_IN) || !dest || (count < 2)) { if (dest && count) dest[0] = 0; return (string_t) { dest, 0 }; } total = 0; hardlimit = stream_is_sequential(stream) ? 1 : 128; //Need to read one byte at a time since we can't scan back if overreading in sequential streams --count; while (!stream_eos(stream)) { limit = count - total; if (limit > hardlimit) limit = hardlimit; if (!limit) break; //This will initialize range [total,total+read) in dest array, making //access of potentially uninitialized dest array safe (see coverity markup below) read = stream->vtable->read(stream, dest + total, limit); if (!read) break; for (i = 0; i < read; ++i) { /* coverity[read_parm] */ if (dest[total + i] == delimiter) break; } total += i; if (i < read) { if ((i + 1) < read) { //Sequential should never end up here reading one byte at a time FOUNDATION_ASSERT(!stream_is_sequential(stream)); stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT); } break; } } dest[total] = 0; return (string_t) { dest, total }; }
uint128_t stream_md5( stream_t* stream ) { int64_t cur, ic, offset, lastc, num; md5_t* md5; uint128_t ret = {0}; unsigned char buf[1025]; bool ignore_lf = false; FOUNDATION_ASSERT( stream ); if( stream->vtable->md5 ) return stream->vtable->md5( stream ); if( !stream || stream_is_sequential( stream ) || !( stream->mode & STREAM_IN ) ) return ret; FOUNDATION_ASSERT( stream->vtable->read ); cur = stream_tell( stream ); stream_seek( stream, 0, STREAM_SEEK_BEGIN ); md5 = md5_allocate(); while( !stream_eos( stream ) ) { num = (int64_t)stream->vtable->read( stream, buf, 1024 ); if( !num ) continue; if( stream->mode & STREAM_BINARY ) md5_digest_raw( md5, buf, (size_t)num ); else { //If last buffer ended with CR, ignore a leading LF offset = 0; lastc = 0; if( ignore_lf && ( buf[0] == '\n' ) ) offset = 1; ignore_lf = false; //Treat all line endings (LF, CR, CR+LF) as Unix style LF. If file has mixed line endings (for example, one line ending in a single CR and next ending in a single LF), it will not work! for( ic = 0; ic < num; ++ic ) { bool was_cr = ( buf[ic] == '\r' ); bool was_lf = ( buf[ic] == '\n' ); if( was_cr || was_lf ) { if( was_cr && ( ic >= 1023 ) ) ignore_lf = true; buf[ic] = '\n'; md5_digest_raw( md5, buf + lastc, (size_t)( ic - lastc + 1 ) ); //Include the LF if( was_cr && ( buf[ic+1] == '\n' ) ) //Check for CR+LF ++ic; lastc = ic + 1; } } if( lastc < num ) md5_digest_raw( md5, buf + lastc, (size_t)( num - lastc ) ); } } stream_seek( stream, cur, STREAM_SEEK_BEGIN ); md5_finalize( md5 ); ret = md5_get_digest_raw( md5 ); md5_deallocate( md5 ); return ret; }
char* stream_read_string( stream_t* stream ) { char buffer[128]; char* outbuffer; int outsize = 128; int cursize = 0; int read, i; bool binary = stream_is_binary( stream ); FOUNDATION_ASSERT( stream ); if( !( stream->mode & STREAM_IN ) ) return 0; FOUNDATION_ASSERT( stream->vtable->read ); outbuffer = memory_allocate( outsize, 0, MEMORY_PERSISTENT ); if( stream_is_sequential( stream ) ) { //Single byte reading since we can't seek backwards (and don't want to block on network sockets) char c; if( !binary ) { //Consume whitespace while( !stream_eos( stream ) ) { read = (int)stream->vtable->read( stream, &c, 1 ); if( !read ) break; if( ( c != ' ' ) && ( c != '\n' ) && ( c != '\r' ) && ( c != '\t' ) ) { outbuffer[cursize++] = c; break; } } } while( !stream_eos( stream ) ) { read = (int)stream->vtable->read( stream, &c, 1 ); if( !read ) break; if( !c ) break; if( !binary && ( ( c == ' ' ) || ( c == '\n' ) || ( c == '\r' ) || ( c == '\t' ) ) ) break; if( cursize + 1 > outsize ) { outsize += 512; outbuffer = memory_reallocate( outbuffer, outsize, 0, cursize ); } outbuffer[cursize++] = c; } outbuffer[cursize] = 0; } else { if( !binary ) { //Consume whitespace while( !stream_eos( stream ) ) { read = (int)stream->vtable->read( stream, buffer, 16 ); if( !read ) break; for( i = 0; i < read; ++i ) { char c = buffer[i]; if( ( c != ' ' ) && ( c != '\n' ) && ( c != '\r' ) && ( c != '\t' ) ) break; } if( i < read ) { stream_seek( stream, i - read, STREAM_SEEK_CURRENT ); break; } } } while( !stream_eos( stream ) ) { read = (int)stream->vtable->read( stream, buffer, 128 ); if( !read ) break; for( i = 0; i < read; ++i ) { char c = buffer[i]; if( !c ) break; if( !binary && ( ( c == ' ' ) || ( c == '\n' ) || ( c == '\r' ) || ( c == '\t' ) ) ) break; } if( i ) { if( cursize + i > outsize ) { outsize += 512; outbuffer = memory_reallocate( outbuffer, outsize, 0, cursize ); } memcpy( outbuffer + cursize, buffer, i ); cursize += i; } if( i < 128 ) { if( ( i + 1 ) < read ) stream_seek( stream, 1 + i - read, STREAM_SEEK_CURRENT ); break; } } outbuffer[cursize] = 0; } return outbuffer; }
uint128_t stream_md5(stream_t* stream) { size_t cur, ic, lastc, num, limit; md5_t md5; uint128_t ret = uint128_null(); unsigned char buf[1025]; bool ignore_lf = false; if (stream->vtable->md5) return stream->vtable->md5(stream); if (stream_is_sequential(stream) || !(stream->mode & STREAM_IN)) return ret; cur = stream_tell(stream); stream_seek(stream, 0, STREAM_SEEK_BEGIN); md5_initialize(&md5); limit = sizeof(buf)-1; buf[limit] = 0; while (!stream_eos(stream)) { num = stream->vtable->read(stream, buf, limit); if (!num) continue; if (stream->mode & STREAM_BINARY) md5_digest(&md5, buf, (size_t)num); else { //If last buffer ended with CR, ignore a leading LF lastc = 0; if (ignore_lf && (buf[0] == '\n')) lastc = 1; ignore_lf = false; //Digest one line at a time //Treat all line endings (LF, CR, CR+LF) as Unix style LF. If file has mixed line endings //(for example, one line ending in a single CR and next is empty and ending in a single LF), //it will not work! /*lint -e{850} */ for (ic = lastc; ic < num && ic < limit; ++ic) { bool was_cr = (buf[ic] == '\r'); bool was_lf = (buf[ic] == '\n'); if (was_cr || was_lf) { if (was_cr && (ic == limit-1)) ignore_lf = true; //Make next buffer ignore leading LF as it is part of CR+LF buf[ic] = '\n'; md5_digest(&md5, buf + lastc, (size_t)((ic - lastc) + 1)); //Include the LF if (was_cr && (buf[ic + 1] == '\n')) //Check for CR+LF ++ic; lastc = ic + 1; } } if (lastc < num) md5_digest(&md5, buf + lastc, (size_t)(num - lastc)); } } stream_seek(stream, (ssize_t)cur, STREAM_SEEK_BEGIN); md5_digest_finalize(&md5); ret = md5_get_digest_raw(&md5); md5_finalize(&md5); return ret; }
string_t stream_read_string_buffer(stream_t* stream, char* outbuffer, size_t size) { char buffer[128]; size_t cursize = 0; size_t read, i; bool binary = stream_is_binary(stream); if (!(stream->mode & STREAM_IN) || !outbuffer || !size) { if (outbuffer && size) outbuffer[0] = 0; return (string_t) { outbuffer, 0 }; } --size; if (stream_is_sequential(stream)) { //Single byte reading since we can't seek backwards (and don't want to block on network sockets) char c; if (!binary) { //Consume whitespace while (!stream_eos(stream)) { read = stream->vtable->read(stream, &c, 1); if (!read) break; if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t')) { outbuffer[cursize++] = c; break; } } } if (cursize > 0) { while (!stream_eos(stream) && (cursize < size)) { read = stream->vtable->read(stream, &c, 1); if (!read) break; if (!c) break; if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t'))) break; outbuffer[cursize++] = c; } } } else { if (!binary) { //Consume whitespace while (!stream_eos(stream)) { read = stream->vtable->read(stream, buffer, 16); if (!read) break; for (i = 0; i < read; ++i) { char c = buffer[i]; if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t')) break; } if (i < read) { stream_seek(stream, (ssize_t)i - (ssize_t)read, STREAM_SEEK_CURRENT); break; } } } while (!stream_eos(stream) && (cursize < size)) { read = stream->vtable->read(stream, buffer, 128); if (!read) break; for (i = 0; i < read; ++i) { char c = buffer[i]; if (!c) break; if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t'))) break; } if (!i) break; if (cursize + i > size) i = size - cursize; memcpy(outbuffer + cursize, buffer, i); cursize += i; if (i < 128) { if ((i + 1) < read) stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT); break; } } } if (cursize < size) outbuffer[cursize] = 0; return (string_t) {outbuffer, cursize}; }
string_t stream_read_string(stream_t* stream) { char buffer[128]; char* outbuffer = buffer; size_t outsize = sizeof(buffer); size_t cursize = 0; size_t read, i; bool binary = stream_is_binary(stream); if (!(stream->mode & STREAM_IN)) return (string_t) { 0, 0 }; if (stream_is_sequential(stream)) { //Single byte reading since we can't seek backwards (and don't want to block on network sockets) char c; if (!binary) { //Consume whitespace while (!stream_eos(stream)) { read = stream->vtable->read(stream, &c, 1); if (!read) break; if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t')) { buffer[cursize++] = c; break; } } } if (cursize > 0) { while (!stream_eos(stream)) { read = stream->vtable->read(stream, &c, 1); if (!read) break; if (!c) break; if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t'))) break; if (cursize + 1 >= outsize) { outsize += 512; if (outbuffer != buffer) { outbuffer = memory_reallocate(outbuffer, outsize, 0, cursize); } else { outbuffer = memory_allocate(0, outsize, 0, MEMORY_PERSISTENT); memcpy(outbuffer, buffer, sizeof(buffer)); } } outbuffer[cursize++] = c; } outbuffer[cursize] = 0; //lint !e661 } } else { if (!binary) { //Consume whitespace while (!stream_eos(stream)) { read = stream->vtable->read(stream, buffer, 16); if (!read) break; for (i = 0; i < read; ++i) { char c = buffer[i]; if ((c != ' ') && (c != '\n') && (c != '\r') && (c != '\t')) break; } if (i < read) { stream_seek(stream, (ssize_t)i - (ssize_t)read, STREAM_SEEK_CURRENT); break; } } } while (!stream_eos(stream)) { if (outbuffer != buffer) read = stream->vtable->read(stream, buffer, sizeof(buffer)); else read = stream->vtable->read(stream, buffer + cursize, sizeof(buffer) - cursize); if (!read) break; for (i = 0; i < read; ++i) { char c = buffer[i]; if (!c) break; if (!binary && ((c == ' ') || (c == '\n') || (c == '\r') || (c == '\t'))) break; } if (i) { if (cursize + i >= outsize) { outsize += 512; if (outbuffer != buffer) { outbuffer = memory_reallocate(outbuffer, outsize, 0, cursize); } else { FOUNDATION_ASSERT(cursize == 0); //Or internal assumptions about code flow is incorrect outbuffer = memory_allocate(0, outsize, 0, MEMORY_PERSISTENT); memcpy(outbuffer, buffer, i); } } else if (outbuffer != buffer) memcpy(outbuffer + cursize, buffer, i); cursize += i; } if (i < sizeof(buffer)) { if ((i + 1) < read) stream_seek(stream, (ssize_t)(1 + i) - (ssize_t)read, STREAM_SEEK_CURRENT); break; } } outbuffer[cursize] = 0; } if (outbuffer == buffer) { if (cursize == 0) return (string_t) { 0, 0 }; outbuffer = memory_allocate(0, cursize + 1, 0, MEMORY_PERSISTENT); memcpy(outbuffer, buffer, cursize); outbuffer[cursize] = 0; } return (string_t) { outbuffer, cursize }; }