|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#pragma once |
|
|
|
|
|
#include "../FF/StatefulFeatureFunction.h" |
|
|
#include "../TypeDef.h" |
|
|
#include "../InMemoryTrie/InMemoryTrie.h" |
|
|
#include "../legacy/Factor.h" |
|
|
#include "../legacy/Util2.h" |
|
|
|
|
|
namespace Moses2 |
|
|
{ |
|
|
|
|
|
|
|
|
struct LMScores { |
|
|
LMScores() { |
|
|
} |
|
|
|
|
|
LMScores(const LMScores ©) : |
|
|
prob(copy.prob), backoff(copy.backoff) { |
|
|
} |
|
|
|
|
|
LMScores(float inProb, float inBackoff) : |
|
|
prob(inProb), backoff(inBackoff) { |
|
|
} |
|
|
|
|
|
void Debug(std::ostream &out, const System &system) const { |
|
|
out << "(" << prob << "," << backoff << ")" << std::flush; |
|
|
} |
|
|
|
|
|
float prob, backoff; |
|
|
}; |
|
|
|
|
|
|
|
|
class LanguageModel: public StatefulFeatureFunction |
|
|
{ |
|
|
public: |
|
|
LanguageModel(size_t startInd, const std::string &line); |
|
|
virtual ~LanguageModel(); |
|
|
|
|
|
virtual void Load(System &system); |
|
|
|
|
|
virtual void SetParameter(const std::string& key, const std::string& value); |
|
|
|
|
|
virtual FFState* BlankState(MemPool &pool, const System &sys) const; |
|
|
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr, |
|
|
const InputType &input, const Hypothesis &hypo) const; |
|
|
|
|
|
virtual void |
|
|
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source, |
|
|
const TargetPhraseImpl &targetPhrase, Scores &scores, |
|
|
SCORE &estimatedScore) const; |
|
|
|
|
|
virtual void |
|
|
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source, |
|
|
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, |
|
|
SCORE &estimatedScore) const; |
|
|
|
|
|
virtual void EvaluateWhenApplied(const ManagerBase &mgr, |
|
|
const Hypothesis &hypo, const FFState &prevState, Scores &scores, |
|
|
FFState &state) const; |
|
|
|
|
|
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr, |
|
|
const SCFG::Hypothesis &hypo, int featureID, Scores &scores, |
|
|
FFState &state) const; |
|
|
|
|
|
protected: |
|
|
std::string m_path; |
|
|
FactorType m_factorType; |
|
|
size_t m_order; |
|
|
|
|
|
InMemoryTrie<const Factor*, LMScores> m_root; |
|
|
SCORE m_oov; |
|
|
const Factor *m_bos; |
|
|
const Factor *m_eos; |
|
|
|
|
|
void ShiftOrPush(std::vector<const Factor*> &context, |
|
|
const Factor *factor) const; |
|
|
std::pair<SCORE, void*> Score( |
|
|
const std::vector<const Factor*> &context) const; |
|
|
SCORE BackoffScore(const std::vector<const Factor*> &context) const; |
|
|
|
|
|
void DebugContext(const std::vector<const Factor*> &context) const; |
|
|
}; |
|
|
|
|
|
} |
|
|
|
|
|
|
