Пример #1
0
void sort_run( indri::file::File& out, indri::file::File& in, size_t memory ) {
  // read the data in
  UINT64 length = in.size();
  char* data = new char[length];
  in.read( data, 0, length );
  in.close();

  qsort( data, length / 12, 12, sort_comparator );

  out.write( data, 0, length );
  delete[] data;
}
Пример #2
0
void copy_region( indri::file::File& out, indri::file::File& in, UINT64 position, UINT64 length ) {
  char* buffer = new char[1024*1024];
  UINT64 bufLength = 1024*1024;
  UINT64 total = 0;
  
  while( length > total ) {
    UINT64 chunk = lemur_compat::min<UINT64>( bufLength, length - total );

    in.read( buffer, position + total, chunk );
    out.write( buffer, total, chunk );
    
    total += chunk;
  }
  
  delete[] buffer;
}
Пример #3
0
static void zlib_read_document( z_stream_s& stream, indri::file::File& infile, UINT64 offset, indri::utility::Buffer& outputBuffer ) {
  // read in data from the file until the stream ends
  // split up the data as necessary
  // decompress positional info

  // read some data
  char inputBuffer[INPUT_BUFFER_SIZE];
  outputBuffer.grow( INPUT_BUFFER_SIZE );
  outputBuffer.write( sizeof(indri::api::ParsedDocument) );

  stream.avail_in = 0;
  stream.avail_out = 0;
  
  while(true) {
    if( !stream.avail_in ) {
      UINT64 readSize = infile.read( inputBuffer, offset, sizeof inputBuffer );
      offset += readSize; 
      
      stream.avail_in = readSize;
      stream.next_in = (Bytef*) inputBuffer;
    }

    stream.avail_out = outputBuffer.size() - outputBuffer.position();
    stream.next_out = (Bytef*) outputBuffer.write( outputBuffer.size() - outputBuffer.position() );

    int result = inflate( &stream, Z_NO_FLUSH );
    outputBuffer.unwrite( stream.avail_out );

    if( result == Z_STREAM_END ) {
      result = inflate( &stream, Z_FINISH );
      
      if( result < 0 )
        LEMUR_THROW( result, "Something bad happened while trying to finish decompressing a document." );

      inflateEnd( &stream );
      break;
    }

    if( result < 0 ) {
      LEMUR_THROW( result, "Something bad happened while trying to decompress a document." );
    }

    if( stream.avail_out == 0 ) {
      outputBuffer.grow();
    }
  }
}