File size: 2,043 Bytes
158b61b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
#include "SplitPointFileParser.h"
#include <istream>
#include <string>
#include "util/string_piece.hh"
#include "util/tokenize_piece.hh"
#include "syntax-common/exception.h"
namespace MosesTraining
{
namespace Syntax
{
namespace PostprocessEgretForests
{
SplitPointFileParser::SplitPointFileParser()
: m_input(0)
{
}
SplitPointFileParser::SplitPointFileParser(std::istream &input)
: m_input(&input)
{
++(*this);
}
SplitPointFileParser &SplitPointFileParser::operator++()
{
if (!m_input) {
return *this;
}
m_entry.splitPoints.clear();
if (!std::getline(*m_input, m_tmpLine)) {
m_input = 0;
return *this;
}
ParseLine(m_tmpLine, m_entry.splitPoints);
return *this;
}
void SplitPointFileParser::ParseLine(const std::string &line,
std::vector<SplitPoint> &splitPoints)
{
std::string tmp;
const util::AnyCharacter delimiter(" \t");
for (util::TokenIter<util::AnyCharacter, true> p(line, delimiter); p; ++p) {
splitPoints.resize(splitPoints.size()+1);
SplitPoint &splitPoint = splitPoints.back();
std::size_t pos = p->find(',');
StringPiece sp = p->substr(0, pos);
sp.CopyToString(&tmp);
splitPoint.tokenPos = std::atoi(tmp.c_str());
std::size_t begin = pos+1;
pos = p->find(',', begin);
sp = p->substr(begin, pos-begin);
sp.CopyToString(&tmp);
splitPoint.charPos = std::atoi(tmp.c_str());
sp = p->substr(pos+1);
sp.CopyToString(&splitPoint.connector);
if (splitPoint.connector.size() > 1) {
throw Exception("multi-character connectors not currently supported");
}
}
}
bool operator==(const SplitPointFileParser &lhs,
const SplitPointFileParser &rhs)
{
// TODO Is this right? Compare values of istreams if non-zero?
return lhs.m_input == rhs.m_input;
}
bool operator!=(const SplitPointFileParser &lhs,
const SplitPointFileParser &rhs)
{
return !(lhs == rhs);
}
} // namespace PostprocessEgretForests
} // namespace Syntax
} // namespace MosesTraining
|