void sort_run( indri::file::File& out, indri::file::File& in, size_t memory ) {
// read the data in
UINT64 length = in.size();
char* data = new char[length];
in.read( data, 0, length );
in.close();
qsort( data, length / 12, 12, sort_comparator );
out.write( data, 0, length );
delete[] data;
}
void copy_region( indri::file::File& out, indri::file::File& in, UINT64 position, UINT64 length ) {
char* buffer = new char[1024*1024];
UINT64 bufLength = 1024*1024;
UINT64 total = 0;
while( length > total ) {
UINT64 chunk = lemur_compat::min<UINT64>( bufLength, length - total );
in.read( buffer, position + total, chunk );
out.write( buffer, total, chunk );
total += chunk;
}
delete[] buffer;
}
static void zlib_read_document( z_stream_s& stream, indri::file::File& infile, UINT64 offset, indri::utility::Buffer& outputBuffer ) {
// read in data from the file until the stream ends
// split up the data as necessary
// decompress positional info
// read some data
char inputBuffer[INPUT_BUFFER_SIZE];
outputBuffer.grow( INPUT_BUFFER_SIZE );
outputBuffer.write( sizeof(indri::api::ParsedDocument) );
stream.avail_in = 0;
stream.avail_out = 0;
while(true) {
if( !stream.avail_in ) {
UINT64 readSize = infile.read( inputBuffer, offset, sizeof inputBuffer );
offset += readSize;
stream.avail_in = readSize;
stream.next_in = (Bytef*) inputBuffer;
}
stream.avail_out = outputBuffer.size() - outputBuffer.position();
stream.next_out = (Bytef*) outputBuffer.write( outputBuffer.size() - outputBuffer.position() );
int result = inflate( &stream, Z_NO_FLUSH );
outputBuffer.unwrite( stream.avail_out );
if( result == Z_STREAM_END ) {
result = inflate( &stream, Z_FINISH );
if( result < 0 )
LEMUR_THROW( result, "Something bad happened while trying to finish decompressing a document." );
inflateEnd( &stream );
break;
}
if( result < 0 ) {
LEMUR_THROW( result, "Something bad happened while trying to decompress a document." );
}
if( stream.avail_out == 0 ) {
outputBuffer.grow();
}
}
}