|
|
|
#include "moses/StaticData.h" |
|
#include "moses/FactorCollection.h" |
|
#include <boost/functional/hash.hpp> |
|
#include "NeuralLMWrapper.h" |
|
#include "neuralLM.h" |
|
|
|
using namespace std; |
|
|
|
namespace Moses |
|
{ |
|
NeuralLMWrapper::NeuralLMWrapper(const std::string &line) |
|
:LanguageModelSingleFactor(line) |
|
{ |
|
ReadParameters(); |
|
} |
|
|
|
|
|
NeuralLMWrapper::~NeuralLMWrapper() |
|
{ |
|
delete m_neuralLM_shared; |
|
} |
|
|
|
|
|
void NeuralLMWrapper::Load(AllOptions::ptr const& opts) |
|
{ |
|
|
|
|
|
FactorCollection &factorCollection = FactorCollection::Instance(); |
|
m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, BOS_); |
|
m_sentenceStartWord[m_factorType] = m_sentenceStart; |
|
m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, EOS_); |
|
m_sentenceEndWord[m_factorType] = m_sentenceEnd; |
|
|
|
m_neuralLM_shared = new nplm::neuralLM(); |
|
m_neuralLM_shared->read(m_filePath); |
|
m_neuralLM_shared->premultiply(); |
|
|
|
m_neuralLM_shared->set_cache(1000000); |
|
|
|
m_unk = m_neuralLM_shared->lookup_word("<unk>"); |
|
|
|
UTIL_THROW_IF2(m_nGramOrder != m_neuralLM_shared->get_order(), |
|
"Wrong order of neuralLM: LM has " << m_neuralLM_shared->get_order() << ", but Moses expects " << m_nGramOrder); |
|
|
|
} |
|
|
|
|
|
LMResult NeuralLMWrapper::GetValue(const vector<const Word*> &contextFactor, State* finalState) const |
|
{ |
|
|
|
if (!m_neuralLM.get()) { |
|
m_neuralLM.reset(new nplm::neuralLM(*m_neuralLM_shared)); |
|
|
|
m_neuralLM->set_cache(1000000); |
|
} |
|
|
|
vector<int> words(contextFactor.size()); |
|
const size_t n = contextFactor.size(); |
|
for (size_t i=0; i<n; i++) { |
|
const Word* word = contextFactor[i]; |
|
const Factor* factor = word->GetFactor(m_factorType); |
|
const std::string string = factor->GetString().as_string(); |
|
int neuralLM_wordID = m_neuralLM->lookup_word(string); |
|
words[i] = neuralLM_wordID; |
|
} |
|
|
|
|
|
size_t hashCode = 0; |
|
for (size_t i=1; i<n; ++i) { |
|
boost::hash_combine(hashCode, words[i]); |
|
} |
|
|
|
double value = m_neuralLM->lookup_ngram(words); |
|
|
|
|
|
LMResult ret; |
|
ret.score = FloorScore(value); |
|
ret.unknown = (words.back() == m_unk); |
|
|
|
(*finalState) = (State*) hashCode; |
|
|
|
return ret; |
|
} |
|
|
|
} |
|
|
|
|
|
|