python3-antlr3 3.5.2-4 / usr / lib / python3 / dist-packages / antlr3 / tokens.py

"""ANTLR3 runtime package"""

# begin[licence]
#
# [The "BSD licence"]
# Copyright (c) 2005-2012 Terence Parr
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# end[licence]

from .constants import DEFAULT_CHANNEL, EOF, INVALID_TOKEN_TYPE

############################################################################
#
# basic token interface
#
############################################################################

class Token(object):
    """@brief Abstract token baseclass."""

    TOKEN_NAMES_MAP = None

    @classmethod
    def registerTokenNamesMap(cls, tokenNamesMap):
        """@brief Store a mapping from token type to token name.
        
        This enables token.typeName to give something more meaningful
        than, e.g., '6'.
        """
        cls.TOKEN_NAMES_MAP = tokenNamesMap
        cls.TOKEN_NAMES_MAP[EOF] = "EOF"

    def __init__(self, type=None, channel=DEFAULT_CHANNEL, text=None,
                 index=-1, line=0, charPositionInLine=-1, input=None):
        # We use -1 for index and charPositionInLine as an invalid index
        self._type = type
        self._channel = channel
        self._text = text
        self._index = index
        self._line = 0
        self._charPositionInLine = charPositionInLine
        self.input = input

    # To override a property, you'll need to override both the getter and setter.
    @property
    def text(self):
        return self._text

    @text.setter
    def text(self, value):
        self._text = value


    @property
    def type(self):
        return self._type

    @type.setter
    def type(self, value):
        self._type = value

    # For compatibility
    def getType(self):
        return self._type

    @property
    def typeName(self):
        if self.TOKEN_NAMES_MAP:
            return self.TOKEN_NAMES_MAP.get(self._type, "INVALID_TOKEN_TYPE")
        else:
            return str(self._type)
    
    @property
    def line(self):
        """Lines are numbered 1..n."""
        return self._line

    @line.setter
    def line(self, value):
        self._line = value


    @property
    def charPositionInLine(self):
        """Columns are numbered 0..n-1."""
        return self._charPositionInLine

    @charPositionInLine.setter
    def charPositionInLine(self, pos):
        self._charPositionInLine = pos


    @property
    def channel(self):
        return self._channel

    @channel.setter
    def channel(self, value):
        self._channel = value


    @property
    def index(self):
        """
        An index from 0..n-1 of the token object in the input stream.
        This must be valid in order to use the ANTLRWorks debugger.
        """
        return self._index

    @index.setter
    def index(self, value):
        self._index = value


    def getInputStream(self):
        """@brief From what character stream was this token created.

        You don't have to implement but it's nice to know where a Token
        comes from if you have include files etc... on the input."""

        raise NotImplementedError

    def setInputStream(self, input):
        """@brief From what character stream was this token created.

        You don't have to implement but it's nice to know where a Token
        comes from if you have include files etc... on the input."""

        raise NotImplementedError


############################################################################
#
# token implementations
#
# Token
# +- CommonToken
# \- ClassicToken
#
############################################################################

class CommonToken(Token):
    """@brief Basic token implementation.

    This implementation does not copy the text from the input stream upon
    creation, but keeps start/stop pointers into the stream to avoid
    unnecessary copy operations.

    """

    def __init__(self, type=None, channel=DEFAULT_CHANNEL, text=None,
                 input=None, start=None, stop=None, oldToken=None):

        if oldToken:
            super().__init__(oldToken.type, oldToken.channel, oldToken.text,
                             oldToken.index, oldToken.line,
                             oldToken.charPositionInLine, oldToken.input)
            if isinstance(oldToken, CommonToken):
                self.start = oldToken.start
                self.stop = oldToken.stop
            else:
                self.start = start
                self.stop = stop

        else:
            super().__init__(type=type, channel=channel, input=input)

            # We need to be able to change the text once in a while.  If
            # this is non-null, then getText should return this.  Note that
            # start/stop are not affected by changing this.
            self._text = text

            # The char position into the input buffer where this token starts
            self.start = start

            # The char position into the input buffer where this token stops
            # This is the index of the last char, *not* the index after it!
            self.stop = stop


    @property
    def text(self):
        # Could be the empty string, and we want to return that.
        if self._text is not None:
            return self._text

        if not self.input:
            return None

        if self.start < self.input.size() and self.stop < self.input.size():
            return self.input.substring(self.start, self.stop)

        return '<EOF>'

    @text.setter
    def text(self, value):
        """
        Override the text for this token.  getText() will return this text
        rather than pulling from the buffer.  Note that this does not mean
        that start/stop indexes are not valid.  It means that that input
        was converted to a new string in the token object.
        """
        self._text = value


    def getInputStream(self):
        return self.input

    def setInputStream(self, input):
        self.input = input


    def __str__(self):
        if self.type == EOF:
            return "<EOF>"

        channelStr = ""
        if self.channel > 0:
            channelStr = ",channel=" + str(self.channel)

        txt = self.text
        if txt:
            # Put 2 backslashes in front of each character
            txt = txt.replace("\n", r"\\n")
            txt = txt.replace("\r", r"\\r")
            txt = txt.replace("\t", r"\\t")
        else:
            txt = "<no text>"

        return ("[@{0.index},{0.start}:{0.stop}={txt!r},"
                "<{0.typeName}>{channelStr},"
                "{0.line}:{0.charPositionInLine}]"
                .format(self, txt=txt, channelStr=channelStr))


class ClassicToken(Token):
    """@brief Alternative token implementation.

    A Token object like we'd use in ANTLR 2.x; has an actual string created
    and associated with this object.  These objects are needed for imaginary
    tree nodes that have payload objects.  We need to create a Token object
    that has a string; the tree node will point at this token.  CommonToken
    has indexes into a char stream and hence cannot be used to introduce
    new strings.
    """

    def __init__(self, type=None, text=None, channel=DEFAULT_CHANNEL,
                 oldToken=None):
        if oldToken:
            super().__init__(type=oldToken.type, channel=oldToken.channel,
                             text=oldToken.text, line=oldToken.line,
                             charPositionInLine=oldToken.charPositionInLine)

        else:
            super().__init__(type=type, channel=channel, text=text,
                             index=None, line=None, charPositionInLine=None)


    def getInputStream(self):
        return None

    def setInputStream(self, input):
        pass


    def toString(self):
        channelStr = ""
        if self.channel > 0:
            channelStr = ",channel=" + str(self.channel)

        txt = self.text
        if not txt:
            txt = "<no text>"

        return ("[@{0.index!r},{txt!r},<{0.type!r}>{channelStr},"
                "{0.line!r}:{0.charPositionInLine!r}]"
                .format(self, txt=txt, channelStr=channelStr))

    __str__ = toString
    __repr__ = toString


INVALID_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE)

# In an action, a lexer rule can set token to this SKIP_TOKEN and ANTLR
# will avoid creating a token for this symbol and try to fetch another.
SKIP_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE)