Hugging Face's logo

sakharamg
/
NMTKD

Model card Files Community
NMTKD
File size: 5,161 Bytes 
158b61b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
// $Id$

#ifndef moses_DynamicCacheBasedLanguageModel_h
#define moses_DynamicCacheBasedLanguageModel_h

#include "moses/Util.h"
#include "FeatureFunction.h"

#ifdef WITH_THREADS
#include <boost/thread/shared_mutex.hpp>
#include <boost/thread/locks.hpp>
#endif

typedef std::pair<int, float> decaying_cache_value_t;
typedef std::map<std::string, decaying_cache_value_t > decaying_cache_t;

#define CBLM_QUERY_TYPE_UNDEFINED (-1)
#define CBLM_QUERY_TYPE_ALLSUBSTRINGS 0
#define CBLM_QUERY_TYPE_WHOLESTRING 1

#define CBLM_SCORE_TYPE_UNDEFINED (-1)
#define CBLM_SCORE_TYPE_HYPERBOLA 0
#define CBLM_SCORE_TYPE_POWER 1
#define CBLM_SCORE_TYPE_EXPONENTIAL 2
#define CBLM_SCORE_TYPE_COSINE 3
#define CBLM_SCORE_TYPE_HYPERBOLA_REWARD 10
#define CBLM_SCORE_TYPE_POWER_REWARD 11
#define CBLM_SCORE_TYPE_EXPONENTIAL_REWARD 12
#define PI 3.14159265

namespace Moses
{

class Range;

/** Calculates score for the Dynamic Cache-Based pseudo LM
 */
class DynamicCacheBasedLanguageModel : public StatelessFeatureFunction
{
  // data structure for the cache;
  // the key is the word and the value is the decaying score
  decaying_cache_t m_cache;
  size_t m_query_type; //way of querying the cache
  size_t m_score_type; //way of scoring entries of the cache
  std::string m_initfiles; // vector of files loaded in the initialization phase
  std::string m_name; // internal name to identify this instance of the Cache-based pseudo LM
  float m_lower_score; //lower_bound_score for no match
  bool m_constant; //flag for setting a non-decaying cache
  std::vector<float> precomputedScores;
  unsigned int m_maxAge;

#ifdef WITH_THREADS
  //multiple readers - single writer lock
  mutable boost::shared_mutex m_cacheLock;
#endif

  float decaying_score(unsigned int age);
  void SetPreComputedScores();
  float GetPreComputedScores(const unsigned int age);

  float Evaluate_Whole_String( const TargetPhrase&) const;
  float Evaluate_All_Substrings( const TargetPhrase&) const;

  void Decay();
  void Update(std::vector<std::string> words, int age);

  void ClearEntries(std::vector<std::string> entries);

  void Execute(std::vector<std::string> commands);
  void Execute_Single_Command(std::string command);

  void Load_Multiple_Files(std::vector<std::string> files);
  void Load_Single_File(const std::string file);

  void Insert(std::vector<std::string> ngrams);

//  void EvaluateInIsolation(const Phrase&, const TargetPhrase&, ScoreComponentCollection&, ScoreComponentCollection& ) const;
  void Print() const;

protected:
  static DynamicCacheBasedLanguageModel* s_instance;
  static std::map< const std::string, DynamicCacheBasedLanguageModel* > s_instance_map;

public:
  DynamicCacheBasedLanguageModel(const std::string &line);
  ~DynamicCacheBasedLanguageModel();

  inline const std::string GetName() {
    return m_name;
  };
  inline void SetName(const std::string name) {
    m_name = name;
  }

  static const DynamicCacheBasedLanguageModel* Instance(const std::string& name) {
    if (s_instance_map.find(name) == s_instance_map.end()) {
      return NULL;
    }
    return s_instance_map[name];
  }

  static DynamicCacheBasedLanguageModel* InstanceNonConst(const std::string& name) {
    if (s_instance_map.find(name) == s_instance_map.end()) {
      return NULL;
    }
    return s_instance_map[name];
  }



  static const DynamicCacheBasedLanguageModel& Instance() {
    return *s_instance;
  }
  static DynamicCacheBasedLanguageModel& InstanceNonConst() {
    return *s_instance;
  }

  bool IsUseable(const FactorMask &mask) const {
    return true;
  }

  void Load(AllOptions::ptr const& opts);
  void Load(const std::string filestr);
  void Execute(std::string command);
  void SetParameter(const std::string& key, const std::string& value);
  void ExecuteDlt(std::map<std::string, std::string> dlt_meta);

  void ClearEntries(std::string &entries);
  void Insert(std::string &entries);
  void Clear();

  virtual void EvaluateInIsolation(const Phrase &source
                                   , const TargetPhrase &targetPhrase
                                   , ScoreComponentCollection &scoreBreakdown
                                   , ScoreComponentCollection &estimatedScores) const;

  void EvaluateWithSourceContext(const InputType &input
                                 , const InputPath &inputPath
                                 , const TargetPhrase &targetPhrase
                                 , const StackVec *stackVec
                                 , ScoreComponentCollection &scoreBreakdown
                                 , ScoreComponentCollection *estimatedScores = NULL) const {
  }

  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
      , const TranslationOptionList &translationOptionList) const {
  }

  void EvaluateWhenApplied(const Hypothesis& hypo,
                           ScoreComponentCollection* accumulator) const {
  }

  void EvaluateWhenApplied(const ChartHypothesis &hypo,
                           ScoreComponentCollection* accumulator) const {
  }

  void SetQueryType(size_t type);
  void SetScoreType(size_t type);
  void SetMaxAge(unsigned int age);
};

}

#endif