translation/tools/mosesdecoder/misc/addLexROtoPT.cpp

#include <iostream>
#include <string>

#include "moses/Phrase.h"
#include "moses/FactorCollection.h"
#include "moses/Timer.h"
#include "moses/InputFileStream.h"
#include "moses/TranslationModel/CompactPT/BlockHashIndex.h"
#include "moses/TranslationModel/CompactPT/CanonicalHuffman.h"
#include "moses/TranslationModel/CompactPT/StringVector.h"

using namespace Moses;
using namespace std;

Timer timer;

FactorList m_factorsF, m_factorsE, m_factorsC;

BlockHashIndex m_hash(10, 16);
size_t m_numScoreComponent;
bool m_multipleScoreTrees;
bool m_inMemory = false;

typedef CanonicalHuffman<float> ScoreTree;
std::vector<ScoreTree*> m_scoreTrees;

StringVector<unsigned char, unsigned long, MmapAllocator>  m_scoresMapped;
StringVector<unsigned char, unsigned long, std::allocator> m_scoresMemory;

////////////////////////////////////////////////////////////////////////////////////
void Load(const string &filePath)
{
  std::FILE* pFile = std::fopen(filePath.c_str(), "r");
  UTIL_THROW_IF2(pFile == NULL, "File " << filePath << " could not be opened");

  //if(m_inMemory)
  m_hash.Load(pFile);
  //else
  //m_hash.LoadIndex(pFile);

  size_t read = 0;
  read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile);
  read += std::fread(&m_multipleScoreTrees,
                     sizeof(m_multipleScoreTrees), 1, pFile);

  if(m_multipleScoreTrees) {
    m_scoreTrees.resize(m_numScoreComponent);
    for(size_t i = 0; i < m_numScoreComponent; i++)
      m_scoreTrees[i] = new CanonicalHuffman<float>(pFile);
  } else {
    m_scoreTrees.resize(1);
    m_scoreTrees[0] = new CanonicalHuffman<float>(pFile);
  }

  if(m_inMemory)
    m_scoresMemory.load(pFile, false);
  else
    m_scoresMapped.load(pFile, true);

}

////////////////////////////////////////////////////////////////////////////////////

std::string
MakeKey(const std::string& f,
        const std::string& e,
        const std::string& c)
{
  std::string key;
  if(!f.empty()) key += f;
  if(!m_factorsE.empty()) {
    if(!key.empty()) key += " ||| ";
    key += e;
  }
  if(!m_factorsC.empty()) {
    if(!key.empty()) key += " ||| ";
    key += c;
  }
  key += " ||| ";
  return key;
}

////////////////////////////////////////////////////////////////////////////////////

std::vector<float>
GetScore(const std::string& f, const std::string& e, const std::string& c)
{
  std::string key;
  std::vector<float> probs;

  key = MakeKey(f, e, c);

  size_t index = m_hash[key];
  if(m_hash.GetSize() != index) {
    std::string scoresString;
    if(m_inMemory)
      scoresString = m_scoresMemory[index].str();
    else
      scoresString = m_scoresMapped[index].str();


    BitWrapper<> bitStream(scoresString);
    for(size_t i = 0; i < m_numScoreComponent; i++) {
      float prob = m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream);
      prob = exp(prob);
      probs.push_back(prob);
    }

    return probs;
  } else {
    // return empty vector;
  }

  return probs;
}

////////////////////////////////////////////////////////////////////////////////////

int main(int argc, char** argv)
{
  string ptPath(argv[1]);
  string roPath(argv[2]);

  // lex reordering model
  m_factorsF.push_back(0);
  m_factorsE.push_back(0);

  Load(roPath);

  // phrase table
  InputFileStream ptStrm(ptPath);

  string line;
  while (getline(ptStrm, line)) {
    //cerr << line << endl;
    std::vector<std::string> columns(7);
    std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
    assert(toks.size() >= 2);

    for (size_t i = 0; i < toks.size(); ++i) {
      columns[i] = Trim(toks[i]);
    }

    std::vector<float> scores = GetScore(columns[0], columns[1], "");
    // key-value pairs
    if (scores.size()) {
      if (!columns[6].empty()) {
        columns[6] += " ";
      }
      columns[6] += "{{LexRO ";
      for (size_t i = 0; i < scores.size() - 1; ++i) {
        columns[6] += Moses::SPrint(scores[i]);
        columns[6] += " ";
      }
      columns[6] += Moses::SPrint(scores[scores.size() - 1]);
      columns[6] += "}}";
    }

    // output
    for (size_t i = 0; i < columns.size() - 1; ++i) {
      cout << columns[i] << " ||| ";
    }
    cout << columns[columns.size() - 1] << endl;
  }

}