File size: 2,667 Bytes
158b61b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
/*
* CHRFScorer.h
*
* Created on: Dec 28, 2016
* Author: [email protected]
*/
#pragma once
#ifndef MERT_CHRFSCORER_H_
#define MERT_CHRFSCORER_H_
#include <fstream>
#include <string>
#include <vector>
#include <set>
#include <boost/shared_ptr.hpp>
#include "Ngram.h"
#include "Reference.h"
#include "ScopedVector.h"
#include "ScoreData.h"
#include "StatisticsBasedScorer.h"
#include "Types.h"
namespace MosesTuning {
const size_t CHRFNgramOrder = 6;
class CHRFScorer : public StatisticsBasedScorer{
public:
enum ReferenceLengthType {
AVERAGE,
CLOSEST,
SHORTEST
};
explicit CHRFScorer(const std::string& config = "");
~CHRFScorer();
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles);
virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry);
virtual statscore_t calculateScore(const std::vector<ScoreStatsType>& comps) const;
virtual std::size_t NumberOfScores() const {
return 3*CHRFNgramOrder + 2;
}
void CalcCHRFStats(const Reference& ref, const std::string& text, ScoreStats& entry) const;
int CalcReferenceLength(const Reference& ref, std::size_t length) const;
ReferenceLengthType GetReferenceLengthType() const {
return m_ref_length_type;
}
void SetReferenceLengthType(ReferenceLengthType type) {
m_ref_length_type = type;
}
const std::vector<Reference*>& GetReferences() const {
return m_references.get();
}
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const {
return totals[CHRFNgramOrder*3+2];
}
/**
* Count the ngrams of each type, up to the given length in the input line.
*/
size_t CountNgrams(const std::string& line, NgramCounts& counts, unsigned int n, bool is_testing=false) const;
void DumpCounts(std::ostream* os, const NgramCounts& counts) const;
// NOTE: this function is also used for unit testing.
bool OpenReferenceStream(std::istream* is, std::size_t file_id);
void ProcessReferenceLine(const std::string& line, Reference* ref) const;
bool GetNextReferenceFromStreams(std::vector<boost::shared_ptr<std::ifstream> >& referenceStreams, Reference& ref) const;
protected:
ReferenceLengthType m_ref_length_type;
// reference translations.
ScopedVector<Reference> m_references;
// no copying allowed
CHRFScorer(const CHRFScorer&);
CHRFScorer& operator=(const CHRFScorer&);
private:
float m_beta;
float m_smooth;
// data extracted from reference files
std::vector<std::size_t> m_ref_lengths;
std::vector<std::multiset<int> > m_ref_tokens;
};
} /* namespace MosesTuning */
#endif /* MERT_CHRFSCORER_H_ */
|