size_t readfull(int f, T ** buf) { off_t sz = lseek(f, 0, SEEK_END); lseek(f, 0, SEEK_SET); *buf = (char*)malloc(sz); preada(f, *buf, sz, 0); return sz; }
inline U read_val() { if (blockptr + sizeof(U) > block + blocklen) { // Read blocklen = std::min(blocksize, total_to_process - fpos); preada(fd, block, blocklen, fpos); blockptr = block; } U res = *((U*)blockptr); blockptr += sizeof(U); fpos += sizeof(U); return res; }
void read_compressed(int f, T * tbuf, size_t nbytes) { #ifndef GRAPHCHI_DISABLE_COMPRESSION unsigned char * buf = (unsigned char*)tbuf; int ret; unsigned have; z_stream strm; int CHUNK = (int) std::max((size_t)4096 * 1024, nbytes); size_t fsize = lseek(f, 0, SEEK_END); unsigned char * in = (unsigned char *) malloc(fsize); lseek(f, 0, SEEK_SET); /* allocate inflate state */ strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; ret = inflateInit(&strm); if (ret != Z_OK) assert(false); /* decompress until deflate stream ends or end of file */ do { ssize_t a = 0; do { a = read(f, in + strm.avail_in, fsize - strm.avail_in); //fread(in, 1, CHUNK, source); strm.avail_in += (int) a; IASSERT(a != (ssize_t)(-1)); } while (a > 0); if (strm.avail_in == 0) break; strm.next_in = in; /* run inflate() on input until output buffer not full */ do { strm.avail_out = CHUNK; strm.next_out = buf; ret = inflate(&strm, Z_NO_FLUSH); IASSERT(ret != Z_STREAM_ERROR); /* state not clobbered */ switch (ret) { case Z_NEED_DICT: ret = Z_DATA_ERROR; /* and fall through */ case Z_DATA_ERROR: case Z_MEM_ERROR: assert(false); } have = CHUNK - strm.avail_out; buf += have; } while (strm.avail_out == 0); /* done when inflate() says it's done */ } while (ret != Z_STREAM_END); // std::cout << "Read: " << (buf - (unsigned char*)tbuf) << std::endl; /* clean up and return */ (void)inflateEnd(&strm); free(in); #else preada(f, tbuf, nbytes, 0); #endif }