Skip to content

A
arpeggio-gm
Commit ecf34778 authored by Igor Dejanovic's avatar Igor Dejanovic
Browse files
Options

Changed PTNode.rule->rule_name. rule is now a reference to ParsingExpression.

parent 1c76b784
Hide whitespace changes
Inline Side-by-side
Showing with 80 additions and 74 deletions
arpeggio/__init__.py
View file @ ecf34778
......@@ -51,13 +51,13 @@ class NoMatch(Exception):
match is not successful.
Args:
rule (str): A name of the rule.
rule_name (str): A name of the rule.
position (int): A position in the input stream where exception
occurred.
parser (Parser): An instance of a parser.
"""
def __init__(self, rule, position, parser):
self.rule = rule
def __init__(self, rule_name, position, parser):
self.rule_name = rule_name
self.position = position
self.parser = parser
......@@ -65,7 +65,7 @@ class NoMatch(Exception):
self._up = True
def __str__(self):
return "Expected '{}' at position {} => '{}'.".format(self.rule,
return "Expected '{}' at position {} => '{}'.".format(self.rule_name,
str(self.parser.pos_to_linecol(self.position)),
self.parser.context(position=self.position))
......@@ -94,7 +94,7 @@ class ParsingExpression(object):
elements: A list (or other python object) used as a staging structure
for python based grammar definition. Used in _from_python for
building nodes list of child parser expressions.
rule (str): The name of the parser rule if this is the root rule.
rule_name (str): The name of the parser rule if this is the root rule.
root (bool): Does this parser expression represents the
root of the parser rule? The root parser rule will create
non-terminal node of the parse tree during parsing.
......@@ -106,7 +106,7 @@ class ParsingExpression(object):
elements = elements[0]
self.elements = elements
self.rule = kwargs.get('rule')
self.rule_name = kwargs.get('rule_name', '')
self.root = kwargs.get('root', False)
nodes = kwargs.get('nodes', [])
......@@ -125,14 +125,14 @@ class ParsingExpression(object):
@property
def name(self):
if self.root:
return "%s=%s" % (self.rule, self.__class__.__name__)
return "%s=%s" % (self.rule_name, self.__class__.__name__)
else:
return self.__class__.__name__
@property
def id(self):
if self.root:
return self.rule
return self.rule_name
else:
return id(self)
......@@ -241,9 +241,9 @@ class ParsingExpression(object):
if len(result) == 1:
result = result[0]
else:
result = NonTerminal(self.rule, c_pos, result)
result = NonTerminal(self, c_pos, result)
else:
result = NonTerminal(self.rule, c_pos, result)
result = NonTerminal(self, c_pos, result)
# Result caching for use by memoization.
self.result_cache[c_pos] = (result, parser.position)
......@@ -260,7 +260,7 @@ class ParsingExpression(object):
place of the NoMatch exception.
"""
if self.root and parser.position == nm.position and nm._up:
nm.rule = self.rule
nm.rule_name = self.rule_name
class Sequence(ParsingExpression):
......@@ -446,7 +446,7 @@ class Combine(Decorator):
results = flatten(results)
# Create terminal from result
return Terminal(self.rule if self.root else '', c_pos, \
return Terminal(self, c_pos, \
"".join([str(result) for result in results]))
except NoMatch:
parser.position = c_pos # Backtracking
......@@ -461,13 +461,13 @@ class Match(ParsingExpression):
"""
Base class for all classes that will try to match something from the input.
"""
def __init__(self, rule, root=False):
super(Match, self).__init__(rule=rule, root=root)
def __init__(self, rule_name, root=False):
super(Match, self).__init__(rule_name=rule_name, root=root)
@property
def name(self):
if self.root:
return "%s=%s(%s)" % (self.rule, self.__class__.__name__, self.to_match)
return "%s=%s(%s)" % (self.rule_name, self.__class__.__name__, self.to_match)
else:
return "%s(%s)" % (self.__class__.__name__, self.to_match)
......@@ -495,8 +495,8 @@ class RegExMatch(Match):
Default is None to support propagation from global parser setting.
'''
def __init__(self, to_match, rule=None, root=False, ignore_case=None):
super(RegExMatch, self).__init__(rule, root)
def __init__(self, to_match, rule_name='', root=False, ignore_case=None):
super(RegExMatch, self).__init__(rule_name, root)
self.to_match = to_match
self.ignore_case = ignore_case
......@@ -517,8 +517,7 @@ class RegExMatch(Match):
print("++ Match '%s' at %d => '%s'" % (m.group(), \
c_pos, parser.context(len(m.group()))))
parser.position += len(m.group())
return Terminal(self.rule if self.root else '', c_pos,
m.group())
return Terminal(self, c_pos, m.group())
else:
if parser.debug:
print("-- NoMatch at {}".format(c_pos))
......@@ -534,8 +533,8 @@ class StrMatch(Match):
ignore_case(bool): If case insensitive match is needed.
Default is None to support propagation from global parser setting.
"""
def __init__(self, to_match, rule=None, root=False, ignore_case=None):
super(StrMatch, self).__init__(rule, root)
def __init__(self, to_match, rule_name='', root=False, ignore_case=None):
super(StrMatch, self).__init__(rule_name, root)
self.to_match = to_match
self.ignore_case = ignore_case
......@@ -557,8 +556,7 @@ class StrMatch(Match):
# If this match is inside sequence than mark for suppression
suppress = type(parser._last_pexpression) is Sequence
return Terminal(self.rule if self.root else '', c_pos,
self.to_match, suppress=suppress)
return Terminal(self, c_pos, self.to_match, suppress=suppress)
else:
if parser.debug:
print("-- NoMatch at {}".format(c_pos))
......@@ -584,15 +582,15 @@ class Kwd(StrMatch):
super(Kwd, self).__init__(to_match, rule=None)
self.to_match = to_match
self.root = True
self.rule = 'keyword'
self.rule_name = 'keyword'
class EndOfFile(Match):
"""
The Match class that will succeed in case end of input is reached.
"""
def __init__(self, rule=None):
super(EndOfFile, self).__init__(rule)
def __init__(self):
super(EndOfFile, self).__init__("EOF")
@property
def name(self):
......@@ -601,7 +599,7 @@ class EndOfFile(Match):
def _parse(self, parser):
c_pos = parser.position
if len(parser.input) == c_pos:
return Terminal('EOF', c_pos, '', suppress=True)
return Terminal(EOF(), c_pos, '', suppress=True)
else:
if parser.debug:
print("!! EOF not matched.")
......@@ -623,8 +621,9 @@ class ParseTreeNode(object):
The node can be terminal(the leaf of the parse tree) or non-terminal.
Attributes:
rule (str): The name of the rule that created this node or empty
string in case this node is created by a non-root pexpression.
rule (ParsingExpression): The rule that created this node.
rule_name (str): The name of the rule that created this node if root rule
or empty string otherwise.
position (int): A position in the input stream where the match
occurred.
error (bool): Is this a false parse tree node created during error
......@@ -632,14 +631,17 @@ class ParseTreeNode(object):
comments : A parse tree of comment(s) attached to this node.
"""
def __init__(self, rule, position, error):
assert rule
assert rule.rule_name is not None
self.rule = rule
self.rule_name = rule.rule_name
self.position = position
self.error = error
self.comments = None
@property
def name(self):
return "%s [%s]" % (self.rule, self.position)
return "%s [%s]" % (self.rule_name, self.position)
class Terminal(ParseTreeNode):
......@@ -647,7 +649,7 @@ class Terminal(ParseTreeNode):
Leaf node of the Parse Tree. Represents matched string.
Attributes:
rule (str): The name of the rule that created this terminal.
rule (ParsingExpression): The rule that created this terminal.
position (int): A position in the input stream where match occurred.
value (str): Matched string at the given position or missing token
name in the case of an error node.
......@@ -662,9 +664,9 @@ class Terminal(ParseTreeNode):
@property
def desc(self):
if self.value:
return "%s '%s' [%s]" % (self.rule, self.value, self.position)
return "%s '%s' [%s]" % (self.rule_name, self.value, self.position)
else:
return "%s [%s]" % (self.rule, self.position)
return "%s [%s]" % (self.rule_name, self.position)
def __str__(self):
return self.value
......@@ -720,7 +722,8 @@ class NonTerminal(ParseTreeNode, list):
this node rule.
"""
# Prevent infinite recursion
if rule_name == '_expr_cache':
if rule_name in ['_expr_cache', '_filtered', 'rule', 'rule_name',
'position', 'append', 'extend']:
raise AttributeError
# First check the cache
......@@ -731,19 +734,22 @@ class NonTerminal(ParseTreeNode, list):
# with the given rule name and create new NonTerminal
# and cache it for later access.
nodes = []
rule = None
for n in self:
if self._filtered:
# For filtered NT rule_name is a rule on
# each of its children
for m in n:
if m.rule == rule_name:
if m.rule_name == rule_name:
nodes.append(m)
rule = m.rule
else:
if n.rule == rule_name:
if n.rule_name == rule_name:
nodes.append(n)
rule = n.rule
# For expression NonTerminals instances position does not have any sense.
result = NonTerminal(rule=rule_name, position=None, nodes=nodes, _filtered=True)
result = NonTerminal(rule=rule, position=None, nodes=nodes, _filtered=True)
self._expr_cache[rule_name] = result
return result
......@@ -894,9 +900,9 @@ class Parser(object):
# visualization
if self.debug:
from arpeggio.export import PTDOTExporter
root_rule = self.parse_tree.rule
root_rule_name = self.parse_tree.rule_name
PTDOTExporter().exportFile(self.parse_tree,
"{}_parse_tree.dot".format(root_rule))
"{}_parse_tree.dot".format(root_rule_name))
return self.parse_tree
def getASG(self, sem_actions=None, defaults=True):
......@@ -940,7 +946,7 @@ class Parser(object):
for n in node:
child = tree_walk(n)
if child is not None:
children.append_result(n.rule, child)
children.append_result(n.rule_name, child)
if self.debug:
print("Processing ", node.name, "= '", str(node),
......@@ -949,15 +955,15 @@ class Parser(object):
for i, a in enumerate(children):
print ("\t%d:" % (i + 1), unicode(a), "type:", type(a).__name__)
if node.rule in sem_actions:
sem_action = sem_actions[node.rule]
if node.rule_name in sem_actions:
sem_action = sem_actions[node.rule_name]
if type(sem_action) is types.FunctionType:
retval = sem_action(self, node, children)
else:
retval = sem_action.first_pass(self, node, children)
if hasattr(sem_action, "second_pass"):
for_second_pass.append((node.rule, retval))
for_second_pass.append((node.rule_name, retval))
if self.debug:
print("\tApplying semantic action ", type(sem_action))
......@@ -1071,9 +1077,9 @@ class Parser(object):
if self.nm is None or args[0].position > self.nm.position:
self.nm = args[0]
else:
rule, position, parser = args
rule_name, position, parser = args
if self.nm is None or position > self.nm.position:
self.nm = NoMatch(rule, position, parser)
self.nm = NoMatch(rule_name, position, parser)
raise self.nm
......@@ -1106,7 +1112,7 @@ class ParserPython(Parser):
# Comments should be optional and there can be more of them
if self.comments_model: # and not isinstance(self.comments_model, ZeroOrMore):
self.comments_model.root = True
self.comments_model.rule = comment_def.__name__
self.comments_model.rule_name = comment_def.__name__
def _parse(self):
return self.parser_model.parse(self)
......@@ -1127,11 +1133,11 @@ class ParserPython(Parser):
def inner_from_python(expression):
retval = None
if type(expression) == types.FunctionType: # Is this expression a parser rule?
rule = expression.__name__
if rule in __rule_cache:
c_rule = __rule_cache.get(rule)
rule_name = expression.__name__
if rule_name in __rule_cache:
c_rule = __rule_cache.get(rule_name)
if self.debug:
print("Rule {} founded in cache.".format(rule))
print("Rule {} founded in cache.".format(rule_name))
if isinstance(c_rule, CrossRef):
self.__cross_refs += 1
if self.debug:
......@@ -1141,10 +1147,10 @@ class ParserPython(Parser):
# Semantic action for the rule
if hasattr(expression, "sem"):
self.sem_actions[rule] = expression.sem
self.sem_actions[rule_name] = expression.sem
# Register rule cross-ref to support recursion
__rule_cache[rule] = CrossRef(rule)
__rule_cache[rule_name] = CrossRef(rule_name)
curr_expr = expression
while type(curr_expr) is types.FunctionType:
......@@ -1152,14 +1158,14 @@ class ParserPython(Parser):
# go into until non-function is returned.
curr_expr = curr_expr()
retval = inner_from_python(curr_expr)
retval.rule = rule
retval.rule_name = rule_name
retval.root = True
# Update cache
__rule_cache[rule] = retval
__rule_cache[rule_name] = retval
if self.debug:
print("New rule: {} -> {}"
.format(rule, retval.__class__.__name__))
.format(rule_name, retval.__class__.__name__))
elif isinstance(expression, StrMatch):
if expression.ignore_case is None:
......
arpeggio/peg.py
View file @ ecf34778
......@@ -81,14 +81,14 @@ class PEGSemanticAction(SemanticAction):
# If resolved rule hasn't got the same name it
# should be cloned and preserved in the peg_rules cache
if resolved_rule.rule != n.rule_name:
if resolved_rule.rule_name != n.rule_name:
resolved_rule = copy.copy(resolved_rule)
resolved_rule.rule = n.rule_name
parser.peg_rules[resolved_rule.rule] = resolved_rule
resolved_rule.rule_name = n.rule_name
parser.peg_rules[resolved_rule.rule_name] = resolved_rule
if parser.debug:
print("Resolving: cloned to {} = > {}"\
.format(resolved_rule.rule, resolved_rule.name))
.format(resolved_rule.rule_name, resolved_rule.name))
node.nodes[i] = resolved_rule
......@@ -111,7 +111,7 @@ class SemRule(PEGSemanticAction):
retval = Sequence(nodes=children[1:])
else:
retval = children[1]
retval.rule = rule_name
retval.rule_name = rule_name
retval.root = True
if not hasattr(parser, "peg_rules"):
......@@ -230,14 +230,14 @@ class ParserPEG(Parser):
# visualization
if self.debug:
from arpeggio.export import PMDOTExporter
root_rule = self.parser_model.rule
root_rule = self.parser_model.rule_name
PMDOTExporter().exportFile(self.parser_model,
"{}_peg_parser_model.dot".format(root_rule))
# Comments should be optional and there can be more of them
if self.comments_model: # and not isinstance(self.comments_model, ZeroOrMore):
self.comments_model.root = True
self.comments_model.rule = comment_rule_name
self.comments_model.rule_name = comment_rule_name
def _parse(self):
return self.parser_model.parse(self)
......
tests/unit/test_peg_parser.py
View file @ ecf34778
......@@ -24,7 +24,7 @@ def test_construct_parser():
parser = ParserPEG(grammar, 'calc')
assert parser.parser_model.rule == 'calc'
assert parser.parser_model.rule_name == 'calc'
assert isinstance(parser.parser_model, Sequence)
assert parser.parser_model.nodes[0].name == 'OneOrMore'
......
tests/unit/test_ptnode_navigation_expressions.py
View file @ ecf34778
......@@ -34,9 +34,9 @@ def test_lookup_single():
assert isinstance(result, ParseTreeNode)
assert isinstance(result.bar, NonTerminal)
# dot access
assert result.bar.rule == 'bar'
assert result.bar.rule_name == 'bar'
# Index access
assert result[1].rule == 'bar'
assert result[1].rule_name == 'bar'
# There are six children from result
assert len(result) == 6
......@@ -52,7 +52,7 @@ def test_lookup_single():
# For example this returns all bum from all bar in result
assert len(result.bar.bum) == 2
# Verify that proper bum are returned
assert result.bar.bum[0].rule == 'bum'
assert result.bar.bum[0].rule_name == 'bum'
assert result.bar.bum[1].position == 18
# Access to terminal
......
tests/unit/test_python_parser.py
View file @ ecf34778
......@@ -29,7 +29,7 @@ def test_pp_construction():
'''
parser = ParserPython(calc)
assert parser.parser_model.rule == 'calc'
assert parser.parser_model.rule_name == 'calc'
assert isinstance(parser.parser_model, Sequence)
assert parser.parser_model.nodes[0].desc == 'OneOrMore'
......
tests/unit/test_reduce_tree.py
View file @ ecf34778
......@@ -31,21 +31,21 @@ def test_reduce_tree():
# PTDOTExporter().exportFile(result, 'test_reduce_tree_pt.dot')
assert result[0].rule == 'first'
assert result[0].rule_name == 'first'
assert isinstance(result[0], NonTerminal)
assert result[3].rule == 'first'
assert result[0][0].rule == 'fourth'
assert result[3].rule_name == 'first'
assert result[0][0].rule_name == 'fourth'
# Check reduction for direct OrderedChoice
assert result[2][0].rule == 'third'
assert result[2][0].rule_name == 'third'
parser = ParserPython(grammar, reduce_tree=True)
result = parser.parse(input)
# PTDOTExporter().exportFile(result, 'test_reduce_tree_pt.dot')
assert result[0].rule == 'fourth'
assert result[0].rule_name == 'fourth'
assert isinstance(result[0], Terminal)
assert result[3].rule == 'fourth'
assert result[3].rule_name == 'fourth'
# Check reduction for direct OrderedChoice
assert result[2][0].rule == 'third_str'
assert result[2][0].rule_name == 'third_str'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment