inline
void quantize(ngram_type& ngram, OStream& os, LogProbs& logprobs, Hashed& hashed, Counts& counts, Codemap& codemap, Codebook& codebook, int order, int shard)
{
  hashed.clear();
  counts.clear();
  codemap.clear();
  
  const size_type pos_first = ngram.index[shard].offsets[order - 1];
  const size_type pos_last  = ngram.index[shard].offsets[order];
  
  for (size_type pos = pos_first; pos < pos_last; ++ pos)
    ++ hashed[logprobs(pos, order)];

  counts.insert(hashed.begin(), hashed.end());
  hashed.clear();
  
  expgram::Quantizer::quantize(ngram, counts, codebook, codemap);
  
  for (size_type pos = pos_first; pos < pos_last; ++ pos) {
    typename Codemap::const_iterator citer = codemap.find(logprobs(pos, order));
    if (citer == codemap.end())
      throw std::runtime_error("no codemap?");
	
    os.write((char*) &(citer->second), sizeof(quantized_type));
  }
}