|
|
#include "TreeFragmentTokenizer.h" |
|
|
|
|
|
#include <cctype> |
|
|
|
|
|
namespace Moses |
|
|
{ |
|
|
namespace Syntax |
|
|
{ |
|
|
namespace F2S |
|
|
{ |
|
|
|
|
|
TreeFragmentToken::TreeFragmentToken(TreeFragmentTokenType t, |
|
|
StringPiece v, std::size_t p) |
|
|
: type(t) |
|
|
, value(v) |
|
|
, pos(p) |
|
|
{ |
|
|
} |
|
|
|
|
|
TreeFragmentTokenizer::TreeFragmentTokenizer() |
|
|
: value_(TreeFragmentToken_EOS, "", -1) |
|
|
{ |
|
|
} |
|
|
|
|
|
TreeFragmentTokenizer::TreeFragmentTokenizer(const StringPiece &s) |
|
|
: str_(s) |
|
|
, value_(TreeFragmentToken_EOS, "", -1) |
|
|
, iter_(s.begin()) |
|
|
, end_(s.end()) |
|
|
, pos_(0) |
|
|
{ |
|
|
++(*this); |
|
|
} |
|
|
|
|
|
TreeFragmentTokenizer &TreeFragmentTokenizer::operator++() |
|
|
{ |
|
|
while (iter_ != end_ && (*iter_ == ' ' || *iter_ == '\t')) { |
|
|
++iter_; |
|
|
++pos_; |
|
|
} |
|
|
|
|
|
if (iter_ == end_) { |
|
|
value_ = TreeFragmentToken(TreeFragmentToken_EOS, "", pos_); |
|
|
return *this; |
|
|
} |
|
|
|
|
|
if (*iter_ == '[') { |
|
|
value_ = TreeFragmentToken(TreeFragmentToken_LSB, "[", pos_); |
|
|
++iter_; |
|
|
++pos_; |
|
|
} else if (*iter_ == ']') { |
|
|
value_ = TreeFragmentToken(TreeFragmentToken_RSB, "]", pos_); |
|
|
++iter_; |
|
|
++pos_; |
|
|
} else { |
|
|
std::size_t start = pos_; |
|
|
while (true) { |
|
|
++iter_; |
|
|
++pos_; |
|
|
if (iter_ == end_ || *iter_ == ' ' || *iter_ == '\t') { |
|
|
break; |
|
|
} |
|
|
if (*iter_ == '[' || *iter_ == ']') { |
|
|
break; |
|
|
} |
|
|
} |
|
|
StringPiece word = str_.substr(start, pos_-start); |
|
|
value_ = TreeFragmentToken(TreeFragmentToken_WORD, word, start); |
|
|
} |
|
|
|
|
|
return *this; |
|
|
} |
|
|
|
|
|
TreeFragmentTokenizer TreeFragmentTokenizer::operator++(int) |
|
|
{ |
|
|
TreeFragmentTokenizer tmp(*this); |
|
|
++*this; |
|
|
return tmp; |
|
|
} |
|
|
|
|
|
bool operator==(const TreeFragmentTokenizer &lhs, |
|
|
const TreeFragmentTokenizer &rhs) |
|
|
{ |
|
|
if (lhs.value_.type == TreeFragmentToken_EOS || |
|
|
rhs.value_.type == TreeFragmentToken_EOS) { |
|
|
return lhs.value_.type == TreeFragmentToken_EOS && |
|
|
rhs.value_.type == TreeFragmentToken_EOS; |
|
|
} |
|
|
return lhs.iter_ == rhs.iter_; |
|
|
} |
|
|
|
|
|
bool operator!=(const TreeFragmentTokenizer &lhs, |
|
|
const TreeFragmentTokenizer &rhs) |
|
|
{ |
|
|
return !(lhs == rhs); |
|
|
} |
|
|
|
|
|
} |
|
|
} |
|
|
} |
|
|
|
