Hugging Face's logo

sakharamg
/
NMTKD

Model card Files Community
NMTKD
File size: 4,356 Bytes 
158b61b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
// vim:tabstop=2
#include "PhraseDictionaryMemoryPerSentenceOnDemand.h"
#include <sstream>

using namespace std;

namespace Moses
{
PhraseDictionaryMemoryPerSentenceOnDemand::PhraseDictionaryMemoryPerSentenceOnDemand(const std::string &line)
  : PhraseDictionary(line, true), m_valuesAreProbabilities(true)
{
  ReadParameters();
}

void PhraseDictionaryMemoryPerSentenceOnDemand::Load(AllOptions::ptr const& opts)
{
  m_options = opts;
  SetFeaturesToApply();

  // don't load anything. Load when we have the input
}


TargetPhraseCollection::shared_ptr PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const
{

  Coll &coll = GetColl();

  return coll[source];

}


void PhraseDictionaryMemoryPerSentenceOnDemand::InitializeForInput(ttasksptr const& ttask)
{
  Coll &coll = GetColl();
  coll.clear();

  VERBOSE(2, "Initializing PhraseDictionaryMemoryPerSentenceOnDemand " << m_description << "\n");

  // The context scope object for this translation task
  //     contains a map of translation task-specific data
  boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();

  // The key to the map is this object
  void const* key = static_cast<void const*>(this);

  // The value stored in the map is a string representing a phrase table
  boost::shared_ptr<string> value = contextScope->get<string>(key);

  // Create a stream to read the phrase table data
  stringstream strme(*(value.get()));

  // Read the phrase table data, one line at a time
  string line;
  while (getline(strme, line)) {

    VERBOSE(3, "\t" << line);

    vector<string> toks = TokenizeMultiCharSeparator(line, "|||");
    Phrase source;
    source.CreateFromString(Input, m_input, toks[0], NULL);

    TargetPhrase *target = new TargetPhrase(this);
    target->CreateFromString(Output, m_output, toks[1], NULL);

    // score for this phrase table
    vector<float> scores = Tokenize<float>(toks[2]);
    if (m_valuesAreProbabilities) {
      std::transform(scores.begin(), scores.end(), scores.begin(),TransformScore);
      std::transform(scores.begin(), scores.end(), scores.begin(),FloorScore);
    }
    target->GetScoreBreakdown().PlusEquals(this, scores);

    // score of all other ff when this rule is being loaded
    target->EvaluateInIsolation(source, GetFeaturesToApply());

    // add to coll
    TargetPhraseCollection::shared_ptr &tpsPtr = coll[source];
    TargetPhraseCollection *tps = tpsPtr.get();
    if (tps == NULL) {
      tps = new TargetPhraseCollection();
      tpsPtr.reset(tps);
    }
    tps->Add(target);
  }
}

void PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
{
  InputPathList::const_iterator iter;
  for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
    InputPath &inputPath = **iter;
    const Phrase &source = inputPath.GetPhrase();

    Coll &coll = GetColl();
    Coll::const_iterator iter = coll.find(source);
    if (iter == coll.end()) {
      TargetPhraseCollection::shared_ptr tprPtr;
      inputPath.SetTargetPhrases(*this, tprPtr, NULL);
    } else {
      const TargetPhraseCollection::shared_ptr &tprPtr = iter->second;
      inputPath.SetTargetPhrases(*this, tprPtr, NULL);
    }
  }
}


ChartRuleLookupManager* PhraseDictionaryMemoryPerSentenceOnDemand::CreateRuleLookupManager(const ChartParser &parser,
    const ChartCellCollectionBase &cellCollection,
    std::size_t /*maxChartSpan*/)
{
  abort();
}

PhraseDictionaryMemoryPerSentenceOnDemand::Coll &PhraseDictionaryMemoryPerSentenceOnDemand::GetColl() const
{
  Coll *coll;
  coll = m_coll.get();
  if (coll == NULL) {
    coll = new Coll;
    m_coll.reset(coll);
  }
  assert(coll);
  return *coll;
}

void
PhraseDictionaryMemoryPerSentenceOnDemand::SetParameter(const std::string& key, const std::string& value)
{
  if (key == "path") {
    UTIL_THROW(util::Exception, "PhraseDictionaryMemoryPerSentenceOnDemand does not support key \"path\".");
  } else if (key == "valuesAreProbabilities") {
    m_valuesAreProbabilities = Scan<bool>(value);
  } else {
    PhraseDictionary::SetParameter(key, value);
  }
}


TO_STRING_BODY(PhraseDictionaryMemoryPerSentenceOnDemand);

// friend
ostream& operator<<(ostream& out, const PhraseDictionaryMemoryPerSentenceOnDemand& phraseDict)
{
  return out;
}

}