Skip to content

A
arpeggio-gm
Commit 40c1b0a9 authored by Igor Dejanovic's avatar Igor Dejanovic
Browse files
Options

Fixing PEP8 violations.

parent 577f2986
Hide whitespace changes
Inline Side-by-side
Showing with 227 additions and 181 deletions
arpeggio/__init__.py
View file @ 40c1b0a9
...@@ -16,37 +16,47 @@ import logging ...@@ -16,37 +16,47 @@ import logging
logger = logging.getLogger('arpeggio') logger = logging.getLogger('arpeggio')
DEFAULT_WS='\t\n\r ' DEFAULT_WS = '\t\n\r '
class ArpeggioError(Exception): class ArpeggioError(Exception):
'''Base class for arpeggio errors.''' '''Base class for arpeggio errors.'''
def __init__(self, message): def __init__(self, message):
self.message = message self.message = message
def __str__(self): def __str__(self):
return repr(self.message) return repr(self.message)
class GrammarError(ArpeggioError): class GrammarError(ArpeggioError):
''' '''
Error raised during parser building phase used to indicate error in the grammar Error raised during parser building phase used to indicate error in the
definition. grammar definition.
''' '''
class SemanticError(ArpeggioError): class SemanticError(ArpeggioError):
''' '''
Error raised during the phase of semantic analisys used to indicate semantic Error raised during the phase of semantic analysis used to indicate
error. semantic error.
''' '''
class NoMatch(Exception): class NoMatch(Exception):
''' '''
Exception raised by the Match classes during parsing to indicate that the Exception raised by the Match classes during parsing to indicate that the
match is not successful. match is not successful.
''' '''
def __init__(self, value, position, parser): def __init__(self, value, position, parser):
self.value = value self.value = value
self.position = position # Position in the input stream where error occured
# Position in the input stream where error occured
self.position = position
self.parser = parser self.parser = parser
self._up = True # By default when NoMatch is thrown we will go up the Parse Model Tree.
# By default when NoMatch is thrown we will go up the Parse Model Tree.
self._up = True
def flatten(_iterable): def flatten(_iterable):
...@@ -59,31 +69,36 @@ def flatten(_iterable): ...@@ -59,31 +69,36 @@ def flatten(_iterable):
result.append(e) result.append(e)
return result return result
# --------------------------------------------------------- # ---------------------------------------------------------
# Parser Model (PEG Abstract Semantic Graph) elements # Parser Model (PEG Abstract Semantic Graph) elements
class ParsingExpression(object): class ParsingExpression(object):
""" """
Represents node of the Parser Model. Represents node of the Parser Model.
Root parser expression node will create non-terminal parser tree node while non-root Root parser expression node will create non-terminal parser tree node while
node will create list of terminals and non-terminals. non-root node will create list of terminals and non-terminals.
""" """
def __init__(self, rule=None, root=False, nodes=None): def __init__(self, rule=None, root=False, nodes=None):
''' '''
@param rule - the name of the parser rule if this is the root of the parser rule. @param rule - the name of the parser rule if this is the root of the
@param root - Does this parser expression represents the root of the parser rule? parser rule.
The root parser rule will create non-terminal node of the @param root - Does this parser expression represents the
parse tree during parsing. root of the parser rule?
The root parser rule will create non-terminal node of
the parse tree during parsing.
@param nodes - list of child parser expression nodes. @param nodes - list of child parser expression nodes.
''' '''
# Memoization. Every node cache the parsing results for the given input positions. # Memoization. Every node cache the parsing results for the given input
self.result_cache = {} # position -> parse tree # positions.
self.result_cache = {} # position -> parse tree
self.nodes = nodes self.nodes = nodes
if nodes is None: if nodes is None:
self.nodes = [] # child expressions self.nodes = [] # child expressions
self.rule = rule self.rule = rule
self.root = root self.root = root
@property @property
def desc(self): def desc(self):
return self.name return self.name
...@@ -93,98 +108,100 @@ class ParsingExpression(object): ...@@ -93,98 +108,100 @@ class ParsingExpression(object):
if self.root: if self.root:
return "%s(%s)" % (self.__class__.__name__, self.rule) return "%s(%s)" % (self.__class__.__name__, self.rule)
else: else:
return self.__class__.__name__ return self.__class__.__name__
@property @property
def id(self): def id(self):
if self.root: if self.root:
return self.rule return self.rule
else: else:
return id(self) return id(self)
def clear_cache(self, processed=None): def clear_cache(self, processed=None):
''' '''
Clears memoization cache. Should be called on input change. Clears memoization cache. Should be called on input change.
Args: Args:
processed (set): Set of processed nodes to prevent infinite loops. processed (set): Set of processed nodes to prevent infinite loops.
''' '''
self.result_cache = {} self.result_cache = {}
if not processed: if not processed:
processed = set() processed = set()
for node in self.nodes: for node in self.nodes:
if node not in processed: if node not in processed:
processed.add(node) processed.add(node)
node.clear_cache(processed) node.clear_cache(processed)
def _parse_intro(self, parser): def _parse_intro(self, parser):
logger.debug("Parsing %s" % self.name) logger.debug("Parsing %s" % self.name)
results = []
parser._skip_ws() parser._skip_ws()
self.c_pos = parser.position self.c_pos = parser.position
def parse(self, parser): def parse(self, parser):
self._parse_intro(parser) self._parse_intro(parser)
#Memoization. #Memoization.
#If this position is already parsed by this parser expression than use #If this position is already parsed by this parser expression than use
#the result #the result
if self.result_cache.has_key(self.c_pos): if self.c_pos in self.result_cache:
logger.debug("Result for [%s, %s] founded in result_cache." % (self, self.c_pos)) logger.debug("Result for [%s, %s] founded in result_cache." %
result, new_pos = self.result_cache[self.c_pos] (self, self.c_pos))
result, new_pos = self.result_cache[self.c_pos]
parser.position = new_pos parser.position = new_pos
return result return result
# We are descending down # We are descending down
if parser.nm: if parser.nm:
parser.nm._up = False parser.nm._up = False
result = self._parse(parser) result = self._parse(parser)
if result: if result:
if parser.reduce_tree: if parser.reduce_tree:
if isinstance(result,list): if isinstance(result, list):
if self.root: if self.root:
result = flatten(result) result = flatten(result)
if len(result)>1: if len(result) > 1:
result = NonTerminal(self.rule, self.c_pos, result) result = NonTerminal(self.rule, self.c_pos, result)
else: else:
result = result[0] result = result[0]
else: else:
if self.root: if self.root:
result = NonTerminal(self.rule, self.c_pos, result) result = NonTerminal(self.rule, self.c_pos, result)
# Result caching for use by memoization. # Result caching for use by memoization.
self.result_cache[self.c_pos] = (result, parser.position) self.result_cache[self.c_pos] = (result, parser.position)
return result return result
#TODO: _nm_change_rule should be called from every parser expression parse #TODO: _nm_change_rule should be called from every parser expression parse
# method that can potentialy be the root parser rule. # method that can potentialy be the root parser rule.
def _nm_change_rule(self, nm, parser): def _nm_change_rule(self, nm, parser):
''' '''
Change rule for the given NoMatch object to a more generic if Change rule for the given NoMatch object to a more generic if
we did not consume any input and we are moving up the parser model tree. we did not consume any input and we are moving up the parser model
Used to report most generic language element expected at the place of tree. Used to report most generic language element expected at the
the NoMatch exception. place of the NoMatch exception.
''' '''
if self.root and self.c_pos == nm.position and nm._up: if self.root and self.c_pos == nm.position and nm._up:
nm.value = self.rule nm.value = self.rule
class Sequence(ParsingExpression): class Sequence(ParsingExpression):
''' '''
Will match sequence of parser expressions in exact order they are defined. Will match sequence of parser expressions in exact order they are defined.
''' '''
def __init__(self, elements=None, rule=None, root=False, nodes=None): def __init__(self, elements=None, rule=None, root=False, nodes=None):
''' '''
@param elements - list used as a stageing structure for python based grammar definition. @param elements - list used as a staging structure for python based
Used in _from_python for building nodes list of child parser expressions. grammar definition. Used in _from_python for building nodes list of
child parser expressions.
''' '''
super(Sequence, self).__init__(rule, root, nodes) super(Sequence, self).__init__(rule, root, nodes)
self.elements = elements self.elements = elements
def _parse(self, parser): def _parse(self, parser):
results = [] results = []
try: try:
...@@ -195,13 +212,13 @@ class Sequence(ParsingExpression): ...@@ -195,13 +212,13 @@ class Sequence(ParsingExpression):
except NoMatch, m: except NoMatch, m:
self._nm_change_rule(m, parser) self._nm_change_rule(m, parser)
raise raise
return results return results
class OrderedChoice(Sequence): class OrderedChoice(Sequence):
''' '''
Will match one of the parser expressions specified. Parser will try to Will match one of the parser expressions specified. Parser will try to
match expressions in the order they are defined. match expressions in the order they are defined.
''' '''
def _parse(self, parser): def _parse(self, parser):
...@@ -212,17 +229,17 @@ class OrderedChoice(Sequence): ...@@ -212,17 +229,17 @@ class OrderedChoice(Sequence):
result = e.parse(parser) result = e.parse(parser)
match = True match = True
except NoMatch, m: except NoMatch, m:
parser.position = self.c_pos # Backtracking parser.position = self.c_pos # Backtracking
self._nm_change_rule(m, parser) self._nm_change_rule(m, parser)
else: else:
break break
if not match: if not match:
parser.position = self.c_pos # Backtracking parser.position = self.c_pos # Backtracking
raise parser.nm raise parser.nm
return result return result
class Repetition(ParsingExpression): class Repetition(ParsingExpression):
''' '''
...@@ -230,10 +247,10 @@ class Repetition(ParsingExpression): ...@@ -230,10 +247,10 @@ class Repetition(ParsingExpression):
''' '''
def __init__(self, *elements, **kwargs): def __init__(self, *elements, **kwargs):
super(Repetition, self).__init__(None) super(Repetition, self).__init__(None)
if len(elements)==1: if len(elements) == 1:
elements = elements[0] elements = elements[0]
self.elements = elements self.elements = elements
nodes = kwargs.get('nodes', []) nodes = kwargs.get('nodes', [])
if not hasattr(nodes, '__iter__'): if not hasattr(nodes, '__iter__'):
nodes = [nodes] nodes = [nodes]
...@@ -250,15 +267,16 @@ class Optional(Repetition): ...@@ -250,15 +267,16 @@ class Optional(Repetition):
try: try:
result = self.nodes[0].parse(parser) result = self.nodes[0].parse(parser)
except NoMatch: except NoMatch:
parser.position = self.c_pos # Backtracking parser.position = self.c_pos # Backtracking
pass pass
return result return result
class ZeroOrMore(Repetition): class ZeroOrMore(Repetition):
''' '''
ZeroOrMore will try to match parser expression specified zero or more times. ZeroOrMore will try to match parser expression specified zero or more
It will never fail. times. It will never fail.
''' '''
def _parse(self, parser): def _parse(self, parser):
results = [] results = []
...@@ -267,11 +285,12 @@ class ZeroOrMore(Repetition): ...@@ -267,11 +285,12 @@ class ZeroOrMore(Repetition):
self.c_pos = parser.position self.c_pos = parser.position
results.append(self.nodes[0].parse(parser)) results.append(self.nodes[0].parse(parser))
except NoMatch: except NoMatch:
parser.position = self.c_pos # Backtracking parser.position = self.c_pos # Backtracking
break break
return results return results
class OneOrMore(Repetition): class OneOrMore(Repetition):
''' '''
OneOrMore will try to match parser expression specified one or more times. OneOrMore will try to match parser expression specified one or more times.
...@@ -285,21 +304,22 @@ class OneOrMore(Repetition): ...@@ -285,21 +304,22 @@ class OneOrMore(Repetition):
results.append(self.nodes[0].parse(parser)) results.append(self.nodes[0].parse(parser))
first = True first = True
except NoMatch: except NoMatch:
parser.position = self.c_pos # Backtracking parser.position = self.c_pos # Backtracking
if not first: if not first:
raise raise
break break
return results return results
class SyntaxPredicate(ParsingExpression): class SyntaxPredicate(ParsingExpression):
''' '''
Base class for all syntax predicates (and, not). Base class for all syntax predicates (and, not).
Predicates are parser expressions that will do the match but will not consume Predicates are parser expressions that will do the match but will not
any input. consume any input.
''' '''
def __init__(self, *elements, **kwargs): def __init__(self, *elements, **kwargs):
if len(elements)==1: if len(elements) == 1:
elements = elements[0] elements = elements[0]
self.elements = elements self.elements = elements
...@@ -310,9 +330,11 @@ class SyntaxPredicate(ParsingExpression): ...@@ -310,9 +330,11 @@ class SyntaxPredicate(ParsingExpression):
super(SyntaxPredicate, self).__init__(None) super(SyntaxPredicate, self).__init__(None)
class And(SyntaxPredicate): class And(SyntaxPredicate):
''' '''
This predicate will succeed if the specified expression matches current input. This predicate will succeed if the specified expression matches current
input.
''' '''
def _parse(self, parser): def _parse(self, parser):
for e in self.nodes: for e in self.nodes:
...@@ -320,13 +342,14 @@ class And(SyntaxPredicate): ...@@ -320,13 +342,14 @@ class And(SyntaxPredicate):
e.parse(parser) e.parse(parser)
except NoMatch: except NoMatch:
parser.position = self.c_pos parser.position = self.c_pos
raise raise
parser.position = self.c_pos parser.position = self.c_pos
class Not(SyntaxPredicate): class Not(SyntaxPredicate):
''' '''
This predicate will succeed if the specified expression doesn't match current input. This predicate will succeed if the specified expression doesn't match
current input.
''' '''
def _parse(self, parser): def _parse(self, parser):
for e in self.nodes: for e in self.nodes:
...@@ -338,23 +361,24 @@ class Not(SyntaxPredicate): ...@@ -338,23 +361,24 @@ class Not(SyntaxPredicate):
parser.position = self.c_pos parser.position = self.c_pos
parser._nm_raise(self.name, self.c_pos, parser) parser._nm_raise(self.name, self.c_pos, parser)
class Match(ParsingExpression): class Match(ParsingExpression):
''' '''
Base class for all classes that will try to match something from the input. Base class for all classes that will try to match something from the input.
''' '''
def __init__(self, rule, root=False): def __init__(self, rule, root=False):
super(Match,self).__init__(rule, root) super(Match, self).__init__(rule, root)
@property @property
def name(self): def name(self):
return "%s(%s)" % (self.__class__.__name__, self.to_match) return "%s(%s)" % (self.__class__.__name__, self.to_match)
def parse(self, parser): def parse(self, parser):
self._parse_intro(parser) self._parse_intro(parser)
if parser._in_parse_comment: if parser._in_parse_comment:
return self._parse(parser) return self._parse(parser)
comments = [] comments = []
try: try:
match = self._parse(parser) match = self._parse(parser)
except NoMatch, nm: except NoMatch, nm:
# If not matched try to match comment # If not matched try to match comment
...@@ -362,26 +386,27 @@ class Match(ParsingExpression): ...@@ -362,26 +386,27 @@ class Match(ParsingExpression):
# handle comments. # handle comments.
if parser.comments_model: if parser.comments_model:
try: try:
parser._in_parse_comment = True parser._in_parse_comment = True
while True: while True:
comments.append(parser.comments_model.parse(parser)) comments.append(parser.comments_model.parse(parser))
parser._skip_ws() parser._skip_ws()
except NoMatch: except NoMatch:
# If comment match successfull try terminal match again # If comment match successfull try terminal match again
if comments: if comments:
match = self._parse(parser) match = self._parse(parser)
match.comments = NonTerminal('comment', self.c_pos, comments) match.comments = NonTerminal('comment', self.c_pos,
comments)
else: else:
parser._nm_raise(nm) parser._nm_raise(nm)
finally: finally:
parser._in_parse_comment = False parser._in_parse_comment = False
else: else:
parser._nm_raise(nm) parser._nm_raise(nm)
return match return match
class RegExMatch(Match): class RegExMatch(Match):
''' '''
This Match class will perform input matching based on Regular Expressions. This Match class will perform input matching based on Regular Expressions.
...@@ -402,10 +427,13 @@ class RegExMatch(Match): ...@@ -402,10 +427,13 @@ class RegExMatch(Match):
if m: if m:
parser.position += len(m.group()) parser.position += len(m.group())
logger.debug("Match %s at %d" % (m.group(), self.c_pos)) logger.debug("Match %s at %d" % (m.group(), self.c_pos))
return Terminal(self.rule if self.root else '', self.c_pos, m.group()) return Terminal(self.rule if self.root else '', self.c_pos,
m.group())
else: else:
logger.debug("NoMatch at %d" % self.c_pos) logger.debug("NoMatch at %d" % self.c_pos)
parser._nm_raise(self.root if self.root else self.name, self.c_pos, parser) parser._nm_raise(self.root if self.root else self.name, self.c_pos,
parser)
class StrMatch(Match): class StrMatch(Match):
''' '''
...@@ -422,7 +450,8 @@ class StrMatch(Match): ...@@ -422,7 +450,8 @@ class StrMatch(Match):
if parser.input[parser.position:].startswith(self.to_match): if parser.input[parser.position:].startswith(self.to_match):
parser.position += len(self.to_match) parser.position += len(self.to_match)
logger.debug("Match %s at %d" % (self.to_match, self.c_pos)) logger.debug("Match %s at %d" % (self.to_match, self.c_pos))
return Terminal(self.rule if self.root else '', self.c_pos, self.to_match) return Terminal(self.rule if self.root else '', self.c_pos,
self.to_match)
else: else:
logger.debug("NoMatch at %d" % self.c_pos) logger.debug("NoMatch at %d" % self.c_pos)
parser._nm_raise(self.to_match, self.c_pos, parser) parser._nm_raise(self.to_match, self.c_pos, parser)
...@@ -433,7 +462,7 @@ class StrMatch(Match): ...@@ -433,7 +462,7 @@ class StrMatch(Match):
def __eq__(self, other): def __eq__(self, other):
return self.to_match == str(other) return self.to_match == str(other)
# HACK: Kwd class is a bit hackish. Need to find a better way to # HACK: Kwd class is a bit hackish. Need to find a better way to
# introduce different classes of string tokens. # introduce different classes of string tokens.
class Kwd(StrMatch): class Kwd(StrMatch):
...@@ -443,9 +472,10 @@ class Kwd(StrMatch): ...@@ -443,9 +472,10 @@ class Kwd(StrMatch):
def __init__(self, to_match): def __init__(self, to_match):
super(Kwd, self).__init__(to_match, rule=None) super(Kwd, self).__init__(to_match, rule=None)
self.to_match = to_match self.to_match = to_match
self.root = True self.root = True
self.rule = 'keyword' self.rule = 'keyword'
class EndOfFile(Match): class EndOfFile(Match):
''' '''
Match class that will succeed in case end of input is reached. Match class that will succeed in case end of input is reached.
...@@ -463,13 +493,13 @@ class EndOfFile(Match): ...@@ -463,13 +493,13 @@ class EndOfFile(Match):
else: else:
logger.debug("EOF not matched.") logger.debug("EOF not matched.")
parser._nm_raise(self.name, self.c_pos, parser) parser._nm_raise(self.name, self.c_pos, parser)
def EOF(): return EndOfFile() def EOF(): return EndOfFile()
# --------------------------------------------------------- # ---------------------------------------------------------
#--------------------------------------------------- #---------------------------------------------------
# Parse Tree node classes # Parse Tree node classes
...@@ -480,41 +510,45 @@ class ParseTreeNode(object): ...@@ -480,41 +510,45 @@ class ParseTreeNode(object):
''' '''
def __init__(self, type, position, error): def __init__(self, type, position, error):
''' '''
@param type - the name of the rule that created this node or empty string in case @param type - the name of the rule that created this node or empty
this node is created by a non-root parser model node. string in case this node is created by a non-root
@param position - position in the input stream where match occured. parser model node.
@param error - is this a false parse tree node created during error recovery? @param position - position in the input stream where match occurred.
@param error - is this a false parse tree node created during error
recovery?
''' '''
self.type = type self.type = type
self.position = position self.position = position
self.error = error self.error = error
self.comments = None self.comments = None
@property @property
def name(self): def name(self):
return "%s [%s]" % (self.type, self.position) return "%s [%s]" % (self.type, self.position)
class Terminal(ParseTreeNode): class Terminal(ParseTreeNode):
''' '''
Leaf node of the Parse Tree. Represents matched string. Leaf node of the Parse Tree. Represents matched string.
''' '''
def __init__(self, type, position, value, error=False): def __init__(self, type, position, value, error=False):
''' '''
@param value - matched string or missing token name in case of an error node. @param value - matched string or missing token name in case of an error
node.
''' '''
super(Terminal, self).__init__(type, position, error) super(Terminal, self).__init__(type, position, error)
self.value = value self.value = value
@property @property
def desc(self): def desc(self):
return "%s \'%s\' [%s]" % (self.type, self.value, self.position) return "%s \'%s\' [%s]" % (self.type, self.value, self.position)
def __str__(self): def __str__(self):
return self.value return self.value
def __eq__(self, other): def __eq__(self, other):
return str(self)==str(other) return str(self) == str(other)
class NonTerminal(ParseTreeNode): class NonTerminal(ParseTreeNode):
''' '''
...@@ -526,116 +560,120 @@ class NonTerminal(ParseTreeNode): ...@@ -526,116 +560,120 @@ class NonTerminal(ParseTreeNode):
''' '''
super(NonTerminal, self).__init__(type, position, error) super(NonTerminal, self).__init__(type, position, error)
self.nodes = flatten([nodes]) self.nodes = flatten([nodes])
@property @property
def desc(self): def desc(self):
return self.name return self.name
# ---------------------------------------------------- # ----------------------------------------------------
# Semantic Actions # Semantic Actions
# #
class SemanticAction(object): class SemanticAction(object):
''' '''
Semantic actions are executed during semantic analysis. They are in charge Semantic actions are executed during semantic analysis. They are in charge
of producing Abstract Semantic Graph (ASG) out of the parse tree. of producing Abstract Semantic Graph (ASG) out of the parse tree.
Every non-terminal and terminal can have semantic action defined which will be Every non-terminal and terminal can have semantic action defined which will
triggered during semantic analisys. be triggered during semantic analysis.
Semantic action triggering is separated in two passes. first_pass method is required Semantic action triggering is separated in two passes. first_pass method is
and the method called second_pass is optional and will be called if exists after required and the method called second_pass is optional and will be called
the first pass. Second pass can be used for forward referencing, if exists after the first pass. Second pass can be used for forward
e.g. linking to the declaration registered in the first pass stage. referencing, e.g. linking to the declaration registered in the first pass
stage.
''' '''
def first_pass(self, parser, node, nodes): def first_pass(self, parser, node, nodes):
''' '''
Called in the first pass of tree walk. Called in the first pass of tree walk.
''' '''
raise NotImplementedError() raise NotImplementedError()
# ---------------------------------------------------- # ----------------------------------------------------
# Parsers # Parsers
class Parser(object): class Parser(object):
def __init__(self, skipws=True, ws=DEFAULT_WS, reduce_tree=False): def __init__(self, skipws=True, ws=DEFAULT_WS, reduce_tree=False):
''' '''
@skipws - if True whitespaces will not be part of parse tree. @skipws - if True whitespaces will not be part of parse tree.
@ws - rule for matching ws @ws - rule for matching ws
@reduce_tree - if true nonterminals with single child will be eliminated. @reduce_tree - if true nonterminals with single child will be
eliminated.
''' '''
self.skipws = skipws self.skipws = skipws
self.ws = ws self.ws = ws
self.reduce_tree = reduce_tree self.reduce_tree = reduce_tree
self.comments_model = None self.comments_model = None
self.sem_actions = {} self.sem_actions = {}
self.parse_tree = None self.parse_tree = None
self._in_parse_comment = False self._in_parse_comment = False
def parse(self, _input): def parse(self, _input):
self.position = 0 # Input position self.position = 0 # Input position
self.nm_pos = 0 # Position for last NoMatch exception self.nm_pos = 0 # Position for last NoMatch exception
self.nm = None # Last NoMatch exception self.nm = None # Last NoMatch exception
self.line_ends = [] self.line_ends = []
self.input = _input self.input = _input
self.parser_model.clear_cache() self.parser_model.clear_cache()
self.parse_tree = self._parse() self.parse_tree = self._parse()
return self.parse_tree return self.parse_tree
def getASG(self, sem_actions=None): def getASG(self, sem_actions=None):
''' '''
Creates Abstract Semantic Graph (ASG) from the parse tree. Creates Abstract Semantic Graph (ASG) from the parse tree.
@param sem_actions - semantic actions dictionary to use for semantic analysis. @param sem_actions - semantic actions dictionary to use for semantic
Rule names are the keys and semantic action objects are values. analysis. Rule names are the keys and semantic
action objects are values.
''' '''
if not self.parse_tree: if not self.parse_tree:
raise Exception("Parse tree is empty. You did call parse(), didn't you?") raise Exception("Parse tree is empty. You did call parse(), didn't you?")
if sem_actions is None: if sem_actions is None:
if not self.sem_actions: if not self.sem_actions:
raise Exception("Semantic actions not defined.") raise Exception("Semantic actions not defined.")
else: else:
sem_actions = self.sem_actions sem_actions = self.sem_actions
if type(sem_actions) is not dict: if type(sem_actions) is not dict:
raise Exception("Semantic actions parameter must be a dictionary.") raise Exception("Semantic actions parameter must be a dictionary.")
for_second_pass = [] for_second_pass = []
def tree_walk(node): def tree_walk(node):
''' '''
Walking the parse tree and calling first_pass for every registered semantic Walking the parse tree and calling first_pass for every registered
actions and creating list of object that needs to be called in the second pass. semantic actions and creating list of object that needs to be
called in the second pass.
''' '''
nodes = [] nodes = []
if isinstance(node, NonTerminal): if isinstance(node, NonTerminal):
for n in node.nodes: for n in node.nodes:
nodes.append(tree_walk(n)) nodes.append(tree_walk(n))
if sem_actions.has_key(node.type): if node.type in sem_actions:
retval = sem_actions[node.type].first_pass(self, node, nodes) retval = sem_actions[node.type].first_pass(self, node, nodes)
if hasattr(sem_actions[node.type], "second_pass"): if hasattr(sem_actions[node.type], "second_pass"):
for_second_pass.append((node.type,retval)) for_second_pass.append((node.type, retval))
else: else:
if isinstance(node, NonTerminal): if isinstance(node, NonTerminal):
retval = NonTerminal(node.type, node.position, nodes) retval = NonTerminal(node.type, node.position, nodes)
else: else:
retval = node retval = node
return retval return retval
logger.debug("ASG: First pass") logger.debug("ASG: First pass")
asg = tree_walk(self.parse_tree) asg = tree_walk(self.parse_tree)
logger.debug("ASG: Second pass") logger.debug("ASG: Second pass")
# Second pass # Second pass
for sa_name, asg_node in for_second_pass: for sa_name, asg_node in for_second_pass:
sem_actions[sa_name].second_pass(self, asg_node) sem_actions[sa_name].second_pass(self, asg_node)
return asg return asg
def pos_to_linecol(self, pos): def pos_to_linecol(self, pos):
''' '''
Calculate (line, column) tuple for the given position in the stream. Calculate (line, column) tuple for the given position in the stream.
...@@ -646,26 +684,28 @@ class Parser(object): ...@@ -646,26 +684,28 @@ class Parser(object):
self.line_ends.append(self.input.index("\n")) self.line_ends.append(self.input.index("\n"))
while True: while True:
try: try:
self.line_ends.append(self.input.index("\n", self.line_ends[-1]+1)) self.line_ends.append(
self.input.index("\n", self.line_ends[-1] + 1))
except ValueError: except ValueError:
break break
except ValueError: except ValueError:
pass pass
line = bisect.bisect_left(self.line_ends, pos) line = bisect.bisect_left(self.line_ends, pos)
col = pos col = pos
if line > 0: if line > 0:
col -= self.line_ends[line-1] col -= self.line_ends[line - 1]
if self.input[self.line_ends[line-1]] in '\n\r': if self.input[self.line_ends[line - 1]] in '\n\r':
col -= 1 col -= 1
return line+1, col+1 return line + 1, col + 1
def _skip_ws(self): def _skip_ws(self):
''' '''
Skiping whitespace characters. Skiping whitespace characters.
''' '''
if self.skipws: if self.skipws:
while self.position<len(self.input) and self.input[self.position] in self.ws: while self.position < len(self.input) and \
self.input[self.position] in self.ws:
self.position += 1 self.position += 1
def _skip_comments(self): def _skip_comments(self):
...@@ -678,12 +718,12 @@ class Parser(object): ...@@ -678,12 +718,12 @@ class Parser(object):
def _nm_raise(self, *args): def _nm_raise(self, *args):
''' '''
Register new NoMatch object if the input is consumed Register new NoMatch object if the input is consumed
from the last NoMatch and raise last NoMatch from the last NoMatch and raise last NoMatch
@param args - NoMatch instance or value, position, parser @param args - NoMatch instance or value, position, parser
''' '''
if not self._in_parse_comment: if not self._in_parse_comment:
if len(args)==1 and isinstance(args[0], NoMatch): if len(args) == 1 and isinstance(args[0], NoMatch):
if self.nm is None or args[0].position > self.nm.position: if self.nm is None or args[0].position > self.nm.position:
self.nm = args[0] self.nm = args[0]
else: else:
...@@ -691,19 +731,20 @@ class Parser(object): ...@@ -691,19 +731,20 @@ class Parser(object):
if self.nm is None or position > self.nm.position: if self.nm is None or position > self.nm.position:
self.nm = NoMatch(value, position, parser) self.nm = NoMatch(value, position, parser)
raise self.nm raise self.nm
class ParserPython(Parser): class ParserPython(Parser):
def __init__(self, language_def, comment_def=None, skipws=True, ws=DEFAULT_WS, \ def __init__(self, language_def, comment_def=None, skipws=True,
reduce_tree=False): ws=DEFAULT_WS, reduce_tree=False):
super(ParserPython, self).__init__(skipws, ws, reduce_tree) super(ParserPython, self).__init__(skipws, ws, reduce_tree)
# PEG Abstract Syntax Graph # PEG Abstract Syntax Graph
self.parser_model = self._from_python(language_def) self.parser_model = self._from_python(language_def)
self.comments_model = self._from_python(comment_def) if comment_def else None self.comments_model = self._from_python(comment_def) \
if comment_def else None
# Comments should be optional and there can be more of them # Comments should be optional and there can be more of them
if self.comments_model: # and not isinstance(self.comments_model, ZeroOrMore): if self.comments_model: # and not isinstance(self.comments_model, ZeroOrMore):
self.comments_model.root = True self.comments_model.root = True
self.comments_model.rule = comment_def.__name__ self.comments_model.rule = comment_def.__name__
...@@ -712,74 +753,80 @@ class ParserPython(Parser): ...@@ -712,74 +753,80 @@ class ParserPython(Parser):
def _from_python(self, expression): def _from_python(self, expression):
""" """
Create parser model from the definition given in the form of python functions returning Create parser model from the definition given in the form of python
lists, tuples, callables, strings and ParsingExpression objects. functions returning lists, tuples, callables, strings and
ParsingExpression objects.
@returns - Parser Model (PEG Abstract Semantic Graph) @returns - Parser Model (PEG Abstract Semantic Graph)
""" """
__rule_cache = {"EndOfFile": EndOfFile()} __rule_cache = {"EndOfFile": EndOfFile()}
__for_resolving = [] # Expressions that needs crossref resolvnih __for_resolving = [] # Expressions that needs crossref resolvnih
self.__cross_refs = 0 self.__cross_refs = 0
class CrossRef(object): class CrossRef(object):
def __init__(self, rule_name): def __init__(self, rule_name):
self.rule_name = rule_name self.rule_name = rule_name
def inner_from_python(expression): def inner_from_python(expression):
retval = None retval = None
if callable(expression): # Is this expression a parser rule? if callable(expression): # Is this expression a parser rule?
rule = expression.__name__ rule = expression.__name__
if __rule_cache.has_key(rule): if rule in __rule_cache:
logger.debug("Rule %s founded in cache." % rule) logger.debug("Rule %s founded in cache." % rule)
if isinstance(__rule_cache.get(rule), CrossRef): if isinstance(__rule_cache.get(rule), CrossRef):
self.__cross_refs += 1 self.__cross_refs += 1
logger.debug("CrossRef usage: %s" % __rule_cache.get(rule).rule_name) logger.debug("CrossRef usage: %s" %
__rule_cache.get(rule).rule_name)
return __rule_cache.get(rule) return __rule_cache.get(rule)
expression_expression = expression() expression_expression = expression()
if callable(expression_expression): if callable(expression_expression):
raise GrammarError( raise GrammarError(
"Rule element can't be just another rule in '%s'." % rule) "Rule element can't be just another rule in '%s'." %
rule)
# Semantic action for the rule # Semantic action for the rule
if hasattr(expression, "sem"): if hasattr(expression, "sem"):
self.sem_actions[rule] = expression.sem self.sem_actions[rule] = expression.sem
# Register rule cross-ref to support recursion # Register rule cross-ref to support recursion
__rule_cache[rule] = CrossRef(rule) __rule_cache[rule] = CrossRef(rule)
retval = inner_from_python(expression()) retval = inner_from_python(expression())
retval.rule = rule retval.rule = rule
retval.root = True retval.root = True
# Update cache # Update cache
__rule_cache[rule] = retval __rule_cache[rule] = retval
logger.debug("New rule: %s -> %s" % (rule, retval.__class__.__name__)) logger.debug("New rule: %s -> %s" %
(rule, retval.__class__.__name__))
elif isinstance(expression, Match): elif isinstance(expression, Match):
retval = expression retval = expression
elif isinstance(expression, Repetition) or isinstance(expression, SyntaxPredicate): elif isinstance(expression, Repetition) or \
isinstance(expression, SyntaxPredicate):
retval = expression retval = expression
retval.nodes.append(inner_from_python(retval.elements)) retval.nodes.append(inner_from_python(retval.elements))
if any((isinstance(x, CrossRef) for x in retval.nodes)): if any((isinstance(x, CrossRef) for x in retval.nodes)):
__for_resolving.append(retval) __for_resolving.append(retval)
elif type(expression) in [list, tuple]: elif type(expression) in [list, tuple]:
if type(expression) is list: if type(expression) is list:
retval = OrderedChoice(expression) retval = OrderedChoice(expression)
else: else:
retval = Sequence(expression) retval = Sequence(expression)
retval.nodes = [inner_from_python(e) for e in expression] retval.nodes = [inner_from_python(e) for e in expression]
if any((isinstance(x, CrossRef) for x in retval.nodes)): if any((isinstance(x, CrossRef) for x in retval.nodes)):
__for_resolving.append(retval) __for_resolving.append(retval)
elif type(expression) is str: elif type(expression) is str:
retval = StrMatch(expression) retval = StrMatch(expression)
else: else:
raise GrammarError("Unrecognized grammar element '%s' in rule %s." % (str(expression), rule)) raise GrammarError("Unrecognized grammar element '%s'." %
str(expression))
return retval return retval
# Cross-ref resolving # Cross-ref resolving
...@@ -789,7 +836,7 @@ class ParserPython(Parser): ...@@ -789,7 +836,7 @@ class ParserPython(Parser):
if isinstance(node, CrossRef): if isinstance(node, CrossRef):
self.__cross_refs -= 1 self.__cross_refs -= 1
e.nodes[i] = __rule_cache[node.rule_name] e.nodes[i] = __rule_cache[node.rule_name]
parser_model = inner_from_python(expression) parser_model = inner_from_python(expression)
resolve() resolve()
assert self.__cross_refs == 0, "Not all crossrefs are resolved!" assert self.__cross_refs == 0, "Not all crossrefs are resolved!"
...@@ -797,4 +844,3 @@ class ParserPython(Parser): ...@@ -797,4 +844,3 @@ class ParserPython(Parser):
def errors(self): def errors(self):
pass pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment