|
#include "util/exception.hh" |
|
#include "moses/FF/Factory.h" |
|
#include "moses/StaticData.h" |
|
|
|
#include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h" |
|
#include "moses/TranslationModel/PhraseDictionaryMemory.h" |
|
#include "moses/TranslationModel/PhraseDictionaryMultiModel.h" |
|
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h" |
|
#include "moses/TranslationModel/PhraseDictionaryGroup.h" |
|
#include "moses/TranslationModel/PhraseDictionaryScope3.h" |
|
#include "moses/TranslationModel/PhraseDictionaryTransliteration.h" |
|
#include "moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h" |
|
#include "moses/TranslationModel/PhraseDictionaryCache.h" |
|
|
|
#include "moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h" |
|
#include "moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h" |
|
#include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h" |
|
#include "moses/TranslationModel/ProbingPT.h" |
|
#include "moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h" |
|
#include "moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h" |
|
|
|
#include "moses/FF/LexicalReordering/LexicalReordering.h" |
|
|
|
#include "moses/FF/BleuScoreFeature.h" |
|
#include "moses/FF/TargetWordInsertionFeature.h" |
|
#include "moses/FF/SourceWordDeletionFeature.h" |
|
#include "moses/FF/GlobalLexicalModel.h" |
|
#include "moses/FF/GlobalLexicalModelUnlimited.h" |
|
#include "moses/FF/Model1Feature.h" |
|
#include "moses/FF/UnknownWordPenaltyProducer.h" |
|
#include "moses/FF/WordTranslationFeature.h" |
|
#include "moses/FF/TargetBigramFeature.h" |
|
#include "moses/FF/TargetNgramFeature.h" |
|
#include "moses/FF/PhraseBoundaryFeature.h" |
|
#include "moses/FF/PhraseDistanceFeature.h" |
|
#include "moses/FF/PhrasePairFeature.h" |
|
#include "moses/FF/RulePairUnlexicalizedSource.h" |
|
#include "moses/FF/PhraseLengthFeature.h" |
|
#include "moses/FF/DistortionScoreProducer.h" |
|
#include "moses/FF/SparseHieroReorderingFeature.h" |
|
#include "moses/FF/WordPenaltyProducer.h" |
|
#include "moses/FF/InputFeature.h" |
|
#include "moses/FF/PhrasePenalty.h" |
|
#include "moses/FF/OSM-Feature/OpSequenceModel.h" |
|
#include "moses/FF/Dsg-Feature/DsgModel.h" |
|
#include "moses/FF/ControlRecombination.h" |
|
#include "moses/FF/ConstrainedDecoding.h" |
|
#include "moses/FF/SoftSourceSyntacticConstraintsFeature.h" |
|
#include "moses/FF/TargetConstituentAdjacencyFeature.h" |
|
#include "moses/FF/TargetPreferencesFeature.h" |
|
#include "moses/FF/CoveredReferenceFeature.h" |
|
#include "moses/FF/TreeStructureFeature.h" |
|
#include "moses/FF/SoftMatchingFeature.h" |
|
#include "moses/FF/DynamicCacheBasedLanguageModel.h" |
|
#include "moses/FF/SourceGHKMTreeInputMatchFeature.h" |
|
#include "moses/FF/HyperParameterAsWeight.h" |
|
#include "moses/FF/SetSourcePhrase.h" |
|
#include "moses/FF/PhraseOrientationFeature.h" |
|
#include "moses/FF/UnalignedWordCountFeature.h" |
|
#include "CountNonTerms.h" |
|
#include "ReferenceComparison.h" |
|
#include "RuleScope.h" |
|
#include "MaxSpanFreeNonTermSource.h" |
|
#include "NieceTerminal.h" |
|
#include "SpanLength.h" |
|
#include "SyntaxRHS.h" |
|
#include "DeleteRules.h" |
|
|
|
#include "moses/FF/ExampleStatelessFF.h" |
|
#include "moses/FF/ExampleStatefulFF.h" |
|
#include "moses/LM/ExampleLM.h" |
|
#include "moses/FF/ExampleTranslationOptionListFeature.h" |
|
#include "moses/LM/BilingualLM.h" |
|
#include "moses/TranslationModel/ExamplePT.h" |
|
#include "moses/Syntax/InputWeightFF.h" |
|
#include "moses/Syntax/RuleTableFF.h" |
|
|
|
#include "moses/LM/InMemoryPerSentenceOnDemandLM.h" |
|
#include "moses/FF/EditOps.h" |
|
#include "moses/FF/CorrectionPattern.h" |
|
|
|
#ifdef HAVE_VW |
|
#include "moses/FF/VW/VW.h" |
|
#include "moses/FF/VW/VWFeatureContextBigrams.h" |
|
#include "moses/FF/VW/VWFeatureContextBilingual.h" |
|
#include "moses/FF/VW/VWFeatureContextWindow.h" |
|
#include "moses/FF/VW/VWFeatureSourceBagOfWords.h" |
|
#include "moses/FF/VW/VWFeatureSourceBigrams.h" |
|
#include "moses/FF/VW/VWFeatureSourceIndicator.h" |
|
#include "moses/FF/VW/VWFeatureSourcePhraseInternal.h" |
|
#include "moses/FF/VW/VWFeatureSourceSenseWindow.h" |
|
#include "moses/FF/VW/VWFeatureSourceWindow.h" |
|
#include "moses/FF/VW/VWFeatureTargetBigrams.h" |
|
#include "moses/FF/VW/VWFeatureTargetIndicator.h" |
|
#include "moses/FF/VW/VWFeatureSourceExternalFeatures.h" |
|
#include "moses/FF/VW/VWFeatureTargetPhraseInternal.h" |
|
#include "moses/FF/VW/VWFeatureTargetPhraseScores.h" |
|
#endif |
|
|
|
#ifdef HAVE_CMPH |
|
#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h" |
|
#endif |
|
#ifdef PT_UG |
|
#include "moses/TranslationModel/UG/mmsapt.h" |
|
#endif |
|
|
|
#include "moses/LM/Ken.h" |
|
#include "moses/LM/Reloading.h" |
|
#ifdef LM_IRST |
|
#include "moses/LM/IRST.h" |
|
#endif |
|
|
|
#ifdef LM_SRI |
|
#include "moses/LM/SRI.h" |
|
#endif |
|
|
|
#ifdef LM_MAXENT_SRI |
|
#include "moses/LM/MaxEntSRI.h" |
|
#endif |
|
|
|
#ifdef LM_RAND |
|
#include "moses/LM/Rand.h" |
|
#endif |
|
|
|
#ifdef HAVE_SYNLM |
|
#include "moses/SyntacticLanguageModel.h" |
|
#endif |
|
|
|
#ifdef LM_NEURAL |
|
#include "moses/LM/NeuralLMWrapper.h" |
|
#include "moses/LM/RDLM.h" |
|
#include "moses/LM/bilingual-lm/BiLM_NPLM.h" |
|
#endif |
|
|
|
#ifdef LM_DALM |
|
#include "moses/LM/DALMWrapper.h" |
|
#endif |
|
|
|
#ifdef LM_OXLM |
|
#include "moses/LM/oxlm/OxLM.h" |
|
#include "moses/LM/oxlm/SourceOxLM.h" |
|
#endif |
|
|
|
#include "util/exception.hh" |
|
|
|
#include <vector> |
|
|
|
namespace Moses |
|
{ |
|
|
|
class FeatureFactory |
|
{ |
|
public: |
|
virtual ~FeatureFactory() {} |
|
|
|
virtual void Create(const std::string &line) = 0; |
|
|
|
protected: |
|
template <class F> static void DefaultSetup(F *feature); |
|
|
|
FeatureFactory() {} |
|
}; |
|
|
|
template <class F> |
|
void |
|
FeatureFactory |
|
::DefaultSetup(F *feature) |
|
{ |
|
FeatureFunction::Register(feature); |
|
|
|
StaticData &static_data = StaticData::InstanceNonConst(); |
|
const std::string &featureName = feature->GetScoreProducerDescription(); |
|
std::vector<float> weights = static_data.GetParameter()->GetWeights(featureName); |
|
|
|
|
|
if (feature->GetNumScoreComponents()) { |
|
if (weights.size() == 0) { |
|
weights = feature->DefaultWeights(); |
|
if (weights.size() == 0) { |
|
TRACE_ERR("WARNING: No weights specified in config file for FF " |
|
<< featureName << ". This FF does not supply default values.\n" |
|
<< "WARNING: Auto-initializing all weights for this FF to 1.0"); |
|
weights.assign(feature->GetNumScoreComponents(),1.0); |
|
} else { |
|
VERBOSE(2,"WARNING: No weights specified in config file for FF " |
|
<< featureName << ". Using default values supplied by FF."); |
|
} |
|
} |
|
UTIL_THROW_IF2(weights.size() != feature->GetNumScoreComponents(), |
|
"FATAL ERROR: Mismatch in number of features and number " |
|
<< "of weights for Feature Function " << featureName |
|
<< " (features: " << feature->GetNumScoreComponents() |
|
<< " vs. weights: " << weights.size() << ")"); |
|
static_data.SetWeights(feature, weights); |
|
} else if (feature->IsTuneable()) |
|
static_data.SetWeights(feature, weights); |
|
} |
|
|
|
namespace |
|
{ |
|
|
|
template <class F> |
|
class DefaultFeatureFactory : public FeatureFactory |
|
{ |
|
public: |
|
void Create(const std::string &line) { |
|
DefaultSetup(new F(line)); |
|
} |
|
}; |
|
|
|
class KenFactory : public FeatureFactory |
|
{ |
|
public: |
|
void Create(const std::string &line) { |
|
DefaultSetup(ConstructKenLM(line)); |
|
} |
|
}; |
|
|
|
class ReloadingFactory : public FeatureFactory |
|
{ |
|
public: |
|
void Create(const std::string &line) { |
|
DefaultSetup(ConstructReloadingLM(line)); |
|
} |
|
}; |
|
|
|
} |
|
|
|
FeatureRegistry::FeatureRegistry() |
|
{ |
|
|
|
#define MOSES_FNAME(name) Add(#name, new DefaultFeatureFactory< name >()); |
|
|
|
#define MOSES_FNAME2(name, type) Add(name, new DefaultFeatureFactory< type >()); |
|
|
|
MOSES_FNAME2("PhraseDictionaryBinary", PhraseDictionaryTreeAdaptor); |
|
MOSES_FNAME(PhraseDictionaryOnDisk); |
|
MOSES_FNAME(PhraseDictionaryMemory); |
|
MOSES_FNAME(PhraseDictionaryScope3); |
|
MOSES_FNAME(PhraseDictionaryMultiModel); |
|
MOSES_FNAME(PhraseDictionaryMultiModelCounts); |
|
MOSES_FNAME(PhraseDictionaryGroup); |
|
MOSES_FNAME(PhraseDictionaryALSuffixArray); |
|
|
|
MOSES_FNAME(PhraseDictionaryTransliteration); |
|
MOSES_FNAME(PhraseDictionaryDynamicCacheBased); |
|
MOSES_FNAME(PhraseDictionaryCache); |
|
MOSES_FNAME(PhraseDictionaryFuzzyMatch); |
|
MOSES_FNAME(ProbingPT); |
|
MOSES_FNAME(PhraseDictionaryMemoryPerSentence); |
|
MOSES_FNAME(PhraseDictionaryMemoryPerSentenceOnDemand); |
|
MOSES_FNAME2("RuleTable", Syntax::RuleTableFF); |
|
MOSES_FNAME2("SyntaxInputWeight", Syntax::InputWeightFF); |
|
|
|
MOSES_FNAME(GlobalLexicalModel); |
|
|
|
MOSES_FNAME(Model1Feature); |
|
MOSES_FNAME(SourceWordDeletionFeature); |
|
MOSES_FNAME(TargetWordInsertionFeature); |
|
MOSES_FNAME(PhraseBoundaryFeature); |
|
MOSES_FNAME(PhraseDistanceFeature); |
|
MOSES_FNAME(PhraseLengthFeature); |
|
MOSES_FNAME(WordTranslationFeature); |
|
MOSES_FNAME(TargetBigramFeature); |
|
MOSES_FNAME(TargetNgramFeature); |
|
MOSES_FNAME(PhrasePairFeature); |
|
MOSES_FNAME(RulePairUnlexicalizedSource); |
|
MOSES_FNAME(LexicalReordering); |
|
MOSES_FNAME2("Generation", GenerationDictionary); |
|
MOSES_FNAME(BleuScoreFeature); |
|
MOSES_FNAME2("Distortion", DistortionScoreProducer); |
|
MOSES_FNAME2("WordPenalty", WordPenaltyProducer); |
|
MOSES_FNAME(InputFeature); |
|
MOSES_FNAME(OpSequenceModel); |
|
MOSES_FNAME(DesegModel); |
|
MOSES_FNAME(PhrasePenalty); |
|
MOSES_FNAME2("UnknownWordPenalty", UnknownWordPenaltyProducer); |
|
MOSES_FNAME(ControlRecombination); |
|
MOSES_FNAME(ConstrainedDecoding); |
|
MOSES_FNAME(CoveredReferenceFeature); |
|
MOSES_FNAME(SourceGHKMTreeInputMatchFeature); |
|
MOSES_FNAME(SoftSourceSyntacticConstraintsFeature); |
|
MOSES_FNAME(TargetConstituentAdjacencyFeature); |
|
MOSES_FNAME(TargetPreferencesFeature); |
|
MOSES_FNAME(TreeStructureFeature); |
|
MOSES_FNAME(SoftMatchingFeature); |
|
MOSES_FNAME(DynamicCacheBasedLanguageModel); |
|
MOSES_FNAME(HyperParameterAsWeight); |
|
MOSES_FNAME(SetSourcePhrase); |
|
MOSES_FNAME(CountNonTerms); |
|
MOSES_FNAME(ReferenceComparison); |
|
MOSES_FNAME(RuleScope); |
|
MOSES_FNAME(MaxSpanFreeNonTermSource); |
|
MOSES_FNAME(NieceTerminal); |
|
MOSES_FNAME(SparseHieroReorderingFeature); |
|
MOSES_FNAME(SpanLength); |
|
MOSES_FNAME(SyntaxRHS); |
|
MOSES_FNAME(PhraseOrientationFeature); |
|
MOSES_FNAME(UnalignedWordCountFeature); |
|
MOSES_FNAME(DeleteRules); |
|
|
|
MOSES_FNAME(ExampleStatelessFF); |
|
MOSES_FNAME(ExampleStatefulFF); |
|
MOSES_FNAME(ExampleLM); |
|
MOSES_FNAME(ExampleTranslationOptionListFeature); |
|
MOSES_FNAME(ExamplePT); |
|
|
|
MOSES_FNAME(InMemoryPerSentenceOnDemandLM); |
|
MOSES_FNAME(EditOps); |
|
MOSES_FNAME(CorrectionPattern); |
|
|
|
#ifdef HAVE_VW |
|
MOSES_FNAME(VW); |
|
MOSES_FNAME(VWFeatureContextBigrams); |
|
MOSES_FNAME(VWFeatureContextBilingual); |
|
MOSES_FNAME(VWFeatureContextWindow); |
|
MOSES_FNAME(VWFeatureSourceBagOfWords); |
|
MOSES_FNAME(VWFeatureSourceBigrams); |
|
MOSES_FNAME(VWFeatureSourceIndicator); |
|
MOSES_FNAME(VWFeatureSourcePhraseInternal); |
|
MOSES_FNAME(VWFeatureSourceSenseWindow); |
|
MOSES_FNAME(VWFeatureSourceWindow); |
|
MOSES_FNAME(VWFeatureTargetBigrams); |
|
MOSES_FNAME(VWFeatureTargetPhraseInternal); |
|
MOSES_FNAME(VWFeatureTargetIndicator); |
|
MOSES_FNAME(VWFeatureSourceExternalFeatures); |
|
MOSES_FNAME(VWFeatureTargetPhraseScores); |
|
#endif |
|
|
|
#ifdef HAVE_CMPH |
|
MOSES_FNAME(PhraseDictionaryCompact); |
|
#endif |
|
#ifdef PT_UG |
|
MOSES_FNAME(Mmsapt); |
|
MOSES_FNAME2("PhraseDictionaryBitextSampling",Mmsapt); |
|
#endif |
|
|
|
#ifdef HAVE_SYNLM |
|
MOSES_FNAME(SyntacticLanguageModel); |
|
#endif |
|
#ifdef LM_IRST |
|
MOSES_FNAME2("IRSTLM", LanguageModelIRST); |
|
#endif |
|
#ifdef LM_SRI |
|
MOSES_FNAME2("SRILM", LanguageModelSRI); |
|
#endif |
|
#ifdef LM_MAXENT_SRI |
|
MOSES_FNAME2("MaxEntLM", LanguageModelMaxEntSRI); |
|
#endif |
|
#ifdef LM_RAND |
|
MOSES_FNAME2("RANDLM", LanguageModelRandLM); |
|
#endif |
|
#ifdef LM_NEURAL |
|
MOSES_FNAME2("NeuralLM", NeuralLMWrapper); |
|
MOSES_FNAME(RDLM); |
|
MOSES_FNAME2("BilingualNPLM", BilingualLM_NPLM); |
|
#endif |
|
#ifdef LM_DALM |
|
MOSES_FNAME2("DALM", LanguageModelDALM); |
|
#endif |
|
#ifdef LM_OXLM |
|
MOSES_FNAME2("OxLM", OxLM<oxlm::LM>); |
|
MOSES_FNAME2("OxFactoredLM", OxLM<oxlm::FactoredLM>); |
|
MOSES_FNAME2("OxFactoredMaxentLM", OxLM<oxlm::FactoredMaxentLM>); |
|
MOSES_FNAME2("OxSourceFactoredLM", SourceOxLM); |
|
MOSES_FNAME2("OxTreeLM", OxLM<oxlm::FactoredTreeLM>); |
|
#endif |
|
Add("ReloadingLM", new ReloadingFactory()); |
|
Add("KENLM", new KenFactory()); |
|
} |
|
|
|
FeatureRegistry::~FeatureRegistry() |
|
{ |
|
} |
|
|
|
void FeatureRegistry::Add(const std::string &name, FeatureFactory *factory) |
|
{ |
|
std::pair<std::string, boost::shared_ptr<FeatureFactory> > to_ins(name, boost::shared_ptr<FeatureFactory>(factory)); |
|
UTIL_THROW_IF2(!registry_.insert(to_ins).second, "Duplicate feature name " << name); |
|
} |
|
|
|
namespace |
|
{ |
|
class UnknownFeatureException : public util::Exception {}; |
|
} |
|
|
|
void FeatureRegistry::Construct(const std::string &name, const std::string &line) |
|
{ |
|
Map::iterator i = registry_.find(name); |
|
UTIL_THROW_IF(i == registry_.end(), UnknownFeatureException, "Feature name " << name << " is not registered."); |
|
i->second->Create(line); |
|
} |
|
|
|
void FeatureRegistry::PrintFF() const |
|
{ |
|
std::vector<std::string> ffs; |
|
std::cerr << "Available feature functions:" << std::endl; |
|
Map::const_iterator iter; |
|
for (iter = registry_.begin(); iter != registry_.end(); ++iter) { |
|
const std::string &ffName = iter->first; |
|
ffs.push_back(ffName); |
|
} |
|
|
|
std::vector<std::string>::const_iterator iterVec; |
|
std::sort(ffs.begin(), ffs.end()); |
|
for (iterVec = ffs.begin(); iterVec != ffs.end(); ++iterVec) { |
|
const std::string &ffName = *iterVec; |
|
std::cerr << ffName << " "; |
|
} |
|
|
|
std::cerr << std::endl; |
|
} |
|
|
|
} |
|
|
