void auxAppend(IPhrase& head, const IPhrase& tail)
{
  head.reserve(head.size()+tail.size());
  for(size_t i = 0; i < tail.size(); ++i) {
    head.push_back(tail[i]);
  }
}
IPhrase LexicalReorderingTableTree::MakeTableKey(const Phrase& f,
    const Phrase& e) const
{
  IPhrase key;
  std::vector<std::string> keyPart;
  if(!m_FactorsF.empty()) {
    for(size_t i = 0; i < f.GetSize(); ++i) {
      /* old code
        std::string s = f.GetWord(i).ToString(m_FactorsF);
        keyPart.push_back(s.substr(0,s.size()-1));
        */
      keyPart.push_back(f.GetWord(i).GetString(m_FactorsF, false));
    }
    auxAppend(key, m_Table->ConvertPhrase(keyPart, SourceVocId));
    keyPart.clear();
  }
  if(!m_FactorsE.empty()) {
    if(!key.empty()) {
      key.push_back(PrefixTreeMap::MagicWord);
    }
    for(size_t i = 0; i < e.GetSize(); ++i) {
      /* old code
        std::string s = e.GetWord(i).ToString(m_FactorsE);
        keyPart.push_back(s.substr(0,s.size()-1));
        */
      keyPart.push_back(e.GetWord(i).GetString(m_FactorsE, false));
    }
    auxAppend(key, m_Table->ConvertPhrase(keyPart,TargetVocId));
    //keyPart.clear();
  }
  return key;
};
bool LexicalReorderingTableTree::Create(std::istream& inFile,
                                        const std::string& outFileName)
{
  std::string line;
  //TRACE_ERR("Entering Create...\n");
  std::string
  ofn(outFileName+".binlexr.srctree"),
      oft(outFileName+".binlexr.tgtdata"),
      ofi(outFileName+".binlexr.idx"),
      ofsv(outFileName+".binlexr.voc0"),
      oftv(outFileName+".binlexr.voc1");


  FILE *os = fOpen(ofn.c_str(),"wb");
  FILE *ot = fOpen(oft.c_str(),"wb");

  //TRACE_ERR("opend files....\n");

  typedef PrefixTreeSA<LabelId,OFF_T> PSA;
  PSA *psa = new PSA;
  PSA::setDefault(InvalidOffT);
  WordVoc* voc[3];

  LabelId currFirstWord = InvalidLabelId;
  IPhrase currKey;

  Candidates         cands;
  std::vector<OFF_T> vo;
  size_t lnc = 0;
  size_t numTokens    = 0;
  size_t numKeyTokens = 0;
  while(getline(inFile, line)) {
    ++lnc;
    if(0 == lnc % 10000) {
      TRACE_ERR(".");
    }
    IPhrase key;
    Scores   score;

    std::vector<std::string> tokens = TokenizeMultiCharSeparator(line, "|||");
    std::string w;
    if(1 == lnc) {
      //do some init stuff in the first line
      numTokens = tokens.size();
      if(tokens.size() == 2) { //f ||| score
        numKeyTokens = 1;
        voc[0] = new WordVoc();
        voc[1] = 0;
      } else if(3 == tokens.size() || 4 == tokens.size()) { //either f ||| e ||| score or f ||| e ||| c ||| score
        numKeyTokens = 2;
        voc[0] = new WordVoc(); //f voc
        voc[1] = new WordVoc(); //e voc
        voc[2] = voc[1];        //c & e share voc
      }
    } else {
      //sanity check ALL lines must have same number of tokens
      CHECK(numTokens == tokens.size());
    }
    size_t phrase = 0;
    for(; phrase < numKeyTokens; ++phrase) {
      //conditioned on more than just f... need |||
      if(phrase >=1) {
        key.push_back(PrefixTreeMap::MagicWord);
      }
      std::istringstream is(tokens[phrase]);
      while(is >> w) {
        key.push_back(voc[phrase]->add(w));
      }
    }
    //collect all non key phrases, i.e. c
    std::vector<IPhrase> tgt_phrases;
    tgt_phrases.resize(numTokens - numKeyTokens - 1);
    for(size_t j = 0; j < tgt_phrases.size(); ++j, ++phrase) {
      std::istringstream is(tokens[numKeyTokens + j]);
      while(is >> w) {
        tgt_phrases[j].push_back(voc[phrase]->add(w));
      }
    }
    //last token is score
    std::istringstream is(tokens[numTokens-1]);
    while(is >> w) {
      score.push_back(atof(w.c_str()));
    }
    //transform score now...
    std::transform(score.begin(),score.end(),score.begin(),TransformScore);
    std::transform(score.begin(),score.end(),score.begin(),FloorScore);
    std::vector<Scores> scores;
    scores.push_back(score);

    if(key.empty()) {
      TRACE_ERR("WARNING: empty source phrase in line '"<<line<<"'\n");
      continue;
    }
    //first time inits
    if(currFirstWord == InvalidLabelId) {
      currFirstWord = key[0];
    }
    if(currKey.empty()) {
      currKey = key;
      //insert key into tree
      CHECK(psa);
      PSA::Data& d = psa->insert(key);
      if(d == InvalidOffT) {
        d = fTell(ot);
      } else {
        TRACE_ERR("ERROR: source phrase already inserted (A)!\nline(" << lnc << "): '" << line << "\n");
        return false;
      }
    }
    if(currKey != key) {
      //ok new key
      currKey = key;
      //a) write cands for old key
      cands.writeBin(ot);
      cands.clear();
      //b) check if we need to move on to new tree root
      if(key[0] != currFirstWord) {
        // write key prefix tree to file and clear
        PTF pf;
        if(currFirstWord >= vo.size()) {
          vo.resize(currFirstWord+1,InvalidOffT);
        }
        vo[currFirstWord] = fTell(os);
        pf.create(*psa, os);
        // clear
        delete psa;
        psa = new PSA;
        currFirstWord = key[0];
      }
      //c) insert key into tree
      CHECK(psa);
      PSA::Data& d = psa->insert(key);
      if(d == InvalidOffT) {
        d = fTell(ot);
      } else {
        TRACE_ERR("ERROR: source phrase already inserted (A)!\nline(" << lnc << "): '" << line << "\n");
        return false;
      }
    }
    cands.push_back(GenericCandidate(tgt_phrases, scores));
  }
  if (lnc == 0) {
    TRACE_ERR("ERROR: empty lexicalised reordering file\n" << std::endl);
    return false;
  }
  //flush remainders
  cands.writeBin(ot);
  cands.clear();
  //process last currFirstWord
  PTF pf;
  if(currFirstWord >= vo.size()) {
    vo.resize(currFirstWord+1,InvalidOffT);
  }
  vo[currFirstWord] = fTell(os);
  pf.create(*psa,os);
  delete psa;
  psa=0;

  fClose(os);
  fClose(ot);
  /*
  std::vector<size_t> inv;
  for(size_t i = 0; i < vo.size(); ++i){
    if(vo[i] == InvalidOffT){
      inv.push_back(i);
    }
  }
  if(inv.size()) {
    TRACE_ERR("WARNING: there are src voc entries with no phrase "
        "translation: count "<<inv.size()<<"\n"
        "There exists phrase translations for "<<vo.size()-inv.size()
        <<" entries\n");
  }
  */
  FILE *oi = fOpen(ofi.c_str(),"wb");
  fWriteVector(oi,vo);
  fClose(oi);

  if(voc[0]) {
    voc[0]->Write(ofsv);
    delete voc[0];
  }
  if(voc[1]) {
    voc[1]->Write(oftv);
    delete voc[1];
  }
  return true;
}