Spaces:
Sleeping
Sleeping
File size: 2,770 Bytes
ffaa9fc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
#
# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
# Use of this file is governed by the BSD 3-clause license that
# can be found in the LICENSE.txt file in the project root.
#/
#
# This class extends {@link BufferedTokenStream} with functionality to filter
# token streams to tokens on a particular channel (tokens where
# {@link Token#getChannel} returns a particular value).
#
# <p>
# This token stream provides access to all tokens by index or when calling
# methods like {@link #getText}. The channel filtering is only used for code
# accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and
# {@link #LB}.</p>
#
# <p>
# By default, tokens are placed on the default channel
# ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the
# {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to
# call {@link Lexer#setChannel}.
# </p>
#
# <p>
# Note: lexer rules which use the {@code ->skip} lexer command or call
# {@link Lexer#skip} do not produce tokens at all, so input text matched by
# such a rule will not be available as part of the token stream, regardless of
# channel.</p>
#/
from antlr4.BufferedTokenStream import BufferedTokenStream
from antlr4.Lexer import Lexer
from antlr4.Token import Token
class CommonTokenStream(BufferedTokenStream):
__slots__ = 'channel'
def __init__(self, lexer:Lexer, channel:int=Token.DEFAULT_CHANNEL):
super().__init__(lexer)
self.channel = channel
def adjustSeekIndex(self, i:int):
return self.nextTokenOnChannel(i, self.channel)
def LB(self, k:int):
if k==0 or (self.index-k)<0:
return None
i = self.index
n = 1
# find k good tokens looking backwards
while n <= k:
# skip off-channel tokens
i = self.previousTokenOnChannel(i - 1, self.channel)
n += 1
if i < 0:
return None
return self.tokens[i]
def LT(self, k:int):
self.lazyInit()
if k == 0:
return None
if k < 0:
return self.LB(-k)
i = self.index
n = 1 # we know tokens[pos] is a good one
# find k good tokens
while n < k:
# skip off-channel tokens, but make sure to not look past EOF
if self.sync(i + 1):
i = self.nextTokenOnChannel(i + 1, self.channel)
n += 1
return self.tokens[i]
# Count EOF just once.#/
def getNumberOfOnChannelTokens(self):
n = 0
self.fill()
for i in range(0, len(self.tokens)):
t = self.tokens[i]
if t.channel==self.channel:
n += 1
if t.type==Token.EOF:
break
return n
|