textX meta-language based on Xtext. WIP.

arpeggio/textx.py

+#######################################################################
+# Name: textx.py
+# Purpose: Implementation of textX language in Arpeggio.
+# Author: Igor R. Dejanovic <igor DOT dejanovic AT gmail DOT com>
+# Copyright: (c) 2014 Igor R. Dejanovic <igor DOT dejanovic AT gmail DOT com>
+# License: MIT License
+#
+# The idea for this language is shamelessly stolen from the Xtext language
+# but there are some subtle differences in both syntax and semantics.
+# To make things clear I have named this language textX ;)
+#######################################################################
+
+from collections import namedtuple
+
+from arpeggio import StrMatch, Optional, ZeroOrMore, OneOrMore, Sequence,\
+    OrderedChoice, RegExMatch, EOF,\
+    SemanticAction,ParserPython, Combine, Parser, SemanticActionSingleChild,\
+    SemanticActionBodyWithBraces
+from arpeggio.export import PMDOTExporter, PTDOTExporter
+from arpeggio import RegExMatch as _
+
+
+# textX grammar
+def textx_model():          return ZeroOrMore(rule), EOF
+def rule():                 return [metaclass, enum]
+def enum():                 return enum_kwd, ident, ':', enum_literal,\
+                                    ZeroOrMore("|", enum_literal), ';'
+def enum_literal():         return ident, '=', str_match
+def metaclass():            return metaclass_name, ":", choice, ';'
+def metaclass_name():       return ident
+
+def choice():               return sequence, ZeroOrMore("|", sequence)
+def sequence():             return OneOrMore(expr)
+
+def expr():                 return [assignment, terminal_match, rule_match,
+                                    bracketed_choice],\
+                                    Optional(repeat_operator)
+def bracketed_choice():     return '(', choice, ')'
+def repeat_operator():      return ['*', '?', '+']
+
+# Assignment
+def assignment():           return attribute, assignment_op, assignment_rhs
+def attribute():            return ident
+def assignment_op():        return ["=", "*=", "+=", "?="]
+def assignment_rhs():       return [rule_ref, list_match, terminal_match]
+
+# Match
+def match():                return [terminal_match, list_match, rule_ref]
+def terminal_match():       return [str_match, re_match]
+def str_match():            return [("'", _(r"((\\')|[^'])*"),"'"),\
+                                    ('"', _(r'((\\")|[^"])*'),'"')]
+def re_match():             return "/", _(r"((\\/)|[^/])*"), "/"
+
+def list_match():           return "{", rule_ref, Optional(list_separator), '}'
+def list_separator():       return terminal_match
+
+# Rule reference
+def rule_ref():             return [rule_match, rule_link]
+def rule_match():           return ident
+def rule_link():            return '[', rule_choice, ']'
+def rule_choice():          return rule_name, ZeroOrMore('|', rule_name)
+def rule_name():            return ident
+
+def ident():                return _(r'\w+')
+def enum_kwd():             return 'enum'
+
+# Comments
+def comment():              return [comment_line, comment_block]
+def comment_line():         return _(r'//.*$')
+def comment_block():        return _(r'/\*(.|\n)*?\*/')
+
+
+
+class RuleMatchCrossRef(object):
+    """Helper class used for cross reference resolving."""
+    def __init__(self, rule_name, position):
+        self.rule_name = rule_name
+        self.position = position
+
+
+class TextXSemanticError(Exception):
+    pass
+
+
+# TextX semantic actions
+class TextXModelSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+
+        class TextXLanguageParser(Parser):
+            """
+            Parser created from textual textX language description.
+            Semantic actions for this parser will construct object
+            graph representing model on the given language.
+            """
+            def __init__(self, *args, **kwargs):
+                super(TextXLanguageParser, self).__init__(*args, **kwargs)
+                self.parser_model = Sequence(nodes=children[:], rule='model', root=True)
+                self.comments_model = parser._peg_rules.get('__comment', None)
+
+            def _parse(self):
+                return self.parser_model.parse(self)
+
+        textx_parser = TextXLanguageParser()
+
+        textx_parser._metaclasses = parser._metaclasses
+        textx_parser._peg_rules = parser._peg_rules
+
+        return textx_parser
+
+    def second_pass(self, parser, textx_parser):
+        """Cross reference resolving for parser model."""
+
+        resolved_set = set()
+
+        def resolve(node):
+            """Recursively resolve peg rule references."""
+            if node in resolved_set or not hasattr(node, 'nodes'):
+                return
+
+            resolved_set.add(node)
+
+            def _inner_resolve(rule):
+                if type(rule) == RuleMatchCrossRef:
+                    if rule.rule_name in textx_parser._peg_rules:
+                        rule_name = rule.rule_name
+                        rule = textx_parser._peg_rules[rule.rule_name]
+
+                        # Cached rule may be crossref also.
+                        while type(rule) == RuleMatchCrossRef:
+                            rule_name = rule.rule_name
+                            rule = _inner_resolve(rule)
+                            # Rewrite rule in the rules map
+                            textx_parser._peg_rules[rule_name] = rule
+
+                return rule
+
+            for idx, rule in enumerate(node.nodes):
+                # If crossref resolve
+                if type(rule) == RuleMatchCrossRef:
+                    rule = _inner_resolve(rule)
+                    node.nodes[idx] = _inner_resolve(rule)
+
+                # If still unresolved raise exception
+                if type(rule) == RuleMatchCrossRef:
+                    raise TextXSemanticError('Unexisting rule "{}" at position {}.'\
+                            .format(rule.rule_name, parser.pos_to_linecol(rule.position)))
+
+                # Depth-First processing
+                resolve(rule)
+
+        resolve(textx_parser.parser_model)
+
+        return textx_parser
+
+
+class MetaClassSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        rule_name, rule = children
+
+        # Do some name mangling for comment rule
+        # to prevent refererencing from other rules
+        if rule_name.lower() == "comment":
+            rule_name = "__comment"
+
+        parser._peg_rules[rule_name] = rule
+        return rule
+
+
+class MetaClassNameSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        class Meta(object):
+            """Dynamic metaclass."""
+            pass
+        name = str(node)
+        cls = Meta
+        cls.__name__ = name
+        # TODO: Attributes and inheritance
+        parser._metaclasses[name] = cls
+        parser._current_metaclass = cls
+
+        # First rule will be the root of the meta-model
+        if not parser.root_rule_name:
+            parser.root_rule_name = name
+
+        return name
+
+
+class SequenceSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        return Sequence(nodes=children[:])
+
+
+class ChoiceSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        return OrderedChoice(nodes=children[:])
+
+
+class AssignmentSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        #TODO: Register assignment on metaclass
+        return children[2]
+
+
+class ExprSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        if children[1] == '?':
+            return Optional(nodes=[children[0]])
+        elif children[1] == '*':
+            return ZeroOrMore(nodes=[children[0]])
+        elif children[1] == '+':
+            return OneOrMore(nodes=[children[0]])
+        else:
+            TextXSemanticError('Unknown repetition operand "{}" at {}'\
+                    .format(children[1], str(parser.pos_to_linecol(node[1].position))))
+
+
+class StrMatchSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        return StrMatch(children[0], ignore_case=parser.ignore_case)
+
+
+class REMatchSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        to_match = children[0]
+        print("TOMATCH:", to_match)
+        regex = RegExMatch(to_match, ignore_case=parser.ignore_case)
+        regex.compile()
+        return regex
+
+
+class RuleMatchSA(SemanticAction):
+    def first_pass(self, parser, node, children):
+        return RuleMatchCrossRef(str(node), node.position)
+
+
+
+textx_model.sem = TextXModelSA()
+metaclass.sem = MetaClassSA()
+metaclass_name.sem = MetaClassNameSA()
+sequence.sem = SequenceSA()
+choice.sem = ChoiceSA()
+bracketed_choice.sem = SemanticActionSingleChild()
+expr.sem = ExprSA()
+str_match.sem = StrMatchSA()
+re_match.sem = REMatchSA()
+rule_match.sem = RuleMatchSA()
+
+
+def get_parser(language_def, ignore_case=True, debug=False):
+    # First create parser for TextX descriptions
+    parser = ParserPython(textx_model, comment_def=comment,
+            ignore_case=ignore_case, reduce_tree=True, debug=debug)
+
+    # This is used during parser construction phase.
+    parser._metaclasses = {}
+    parser._peg_rules = {
+            # Special rules - primitive types
+            'ID': _(r'[^\d\W]\w*\b'),
+            'INT': _(r'[-+]?[0-9]+'),
+            'FLOAT': _(r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?'),
+            'STRING': _(r'("[^"]*")|(\'[^\']*\')')
+            }
+    for regex in parser._peg_rules.values():
+        regex.compile()
+    parser._current_metaclass = None
+    parser.root_rule_name = None
+
+    # Parse language description with TextX parser
+    parse_tree = parser.parse(language_def)
+
+    # Construct new parser based on the given language description.
+    # This parser will have semantic actions in place to create
+    # object model from the textx textual representations.
+    lang_parser = parser.getASG()
+
+    if debug:
+        # Create dot file for debuging purposes
+        PMDOTExporter().exportFile(lang_parser.parser_model,\
+                "{}_parser_model.dot".format(parser.root_rule_name))
+
+    return lang_parser
+
+

examples/textx/dommlite.tx

+
+DOMMLiteModel:
+	"model" name=ID (shortDesc=STRING)? (longDesc=STRING)?
+	((dataTypes+=UserDataType)|
+	(constraintTypes+=ConstraintType))*
+	packages*=Package;
+
+PackageElement:
+	Package|Classifier;
+	
+Classifier:
+	Entity|Service|ValueObject|Exception|DataType|ConstraintType;
+	
+DataType:
+	BuildInDataType|UserDataType|Enumeration;
+
+ConstraintType:
+	ValidatorType|TagType;
+
+ValidatorType:
+	UserValidatorType|BuildInValidatorType;
+
+TagType:
+	UserTagType|BuildInTagType;
+
+Feature:
+	Property|Operation;
+
+TypedElement:
+	Feature|Parameter;
+
+NamedElement:
+	DOMMLiteModel|PackageElement|FeatureCompartment|OperationCompartment|EnumerationLiteral|TypedElement;
+
+
+/* Ugradeni elementi modela ne bi trebalo da se definisu u modelu vec trebaju biti automatski prisutni u svakom modelu*/
+BuildInDataType:
+	"buildinDataType" name=ID 
+	(shortDesc=STRING)? (longDesc=STRING)?;
+	
+BuildInValidatorType:
+	"buildinValidator" name=ID 
+		("("(parameters+=ConstraintTypeParameter)?(","parameters+=ConstraintTypeParameter)* ")")? 
+		("appliesTo" 	
+			((appliesToEntity?="_entity") |
+			(appliesToProperty?="_prop") | 
+			(appliesToParameter?="_param") | 
+			(appliesToOperation?="_op") |
+			(appliesToService?="_service") | 
+			(appliesToValueObject?="_valueObject"))*)? 
+	(shortDesc=STRING)? (longDesc=STRING)?;
+	
+BuildInTagType:
+	"buildinTagType" name=ID 
+		("("(parameters+=ConstraintTypeParameter)?(","parameters+=ConstraintTypeParameter)* ")")? 
+		("appliesTo" 	
+			((appliesToEntity?="_entity") |
+			(appliesToProperty?="_prop") | 
+			(appliesToParameter?="_param") | 
+			(appliesToOperation?="_op") |
+			(appliesToService?="_service") | 
+			(appliesToValueObject?="_valueObject"))*)? 
+	(shortDesc=STRING)? (longDesc=STRING)?;
+
+
+UserDataType:
+	"dataType" name=ID (shortDesc=STRING)? (longDesc=STRING)?;
+
+UserTagType:
+	"tagType" name=ID 
+		("("(parameters+=ConstraintTypeParameter)?(","parameters+=ConstraintTypeParameter)* ")")? 
+		("appliesTo" 	
+			((appliesToEntity?="_entity") |
+			(appliesToProperty?="_prop") | 
+			(appliesToParameter?="_param") | 
+			(appliesToOperation?="_op") |
+			(appliesToService?="_service") | 
+			(appliesToValueObject?="_valueObject"))*)? 
+		(shortDesc=STRING)? (longDesc=STRING)?;
+
+UserValidatorType:
+	"validatorType" name=ID 
+		("("(parameters+=ConstraintTypeParameter)?(","parameters+=ConstraintTypeParameter)* ")")? 
+		("appliesTo" 	
+			((appliesToEntity?="_entity") |
+			(appliesToProperty?="_prop") | 
+			(appliesToParameter?="_param") | 
+			(appliesToOperation?="_op") |
+			(appliesToService?="_service") | 
+			(appliesToValueObject?="_valueObject"))*)? 
+		(shortDesc=STRING)? (longDesc=STRING)?;
+
+Enum ConstraintTypeParameter:
+	string="_string"|
+	int="_int" | 
+	ref="_ref" | 
+	ellipsis="..."
+;
+
+Package:
+	"package" name=ID (shortDesc=STRING)? (longDesc=STRING)? "{"
+		(packageElements+=PackageElement)*
+	"}";
+	
+Entity:
+	"entity" name=ID
+		("extends" extends=[Entity])?
+		("depends" depends+=[Service] ("," depends+=[Service])*)? 
+		(shortDesc=STRING)? (longDesc=STRING)? 
+	"{"
+		key=BusinessKey
+		("repr" repr+=ReprParameter ("+" repr+=ReprParameter)*)?
+		("[" constraints+=ConstraintSpec ("," constraints+=ConstraintSpec)* "]")?
+		(features+=Feature)* 		
+		(featureCompartments+=FeatureCompartment)*
+	"}";
+
+ReprParameter:
+	ReprParameterStr|ReprParameterRef;
+ReprParameterStr:
+	paramValue=STRING;
+ReprParameterRef:
+	paramValue=[Property];
+
+
+BusinessKey:
+	"key" "{"
+		(properties+=Property)+
+	"}";
+
+ConstraintSpec:
+	 type=[ConstraintType] ("(" (parameters+=ConstraintParameter)?  ("," parameters+=ConstraintParameter)* ")")? 
+	
+;
+
+ConstraintParameter:
+	ConstraintIntParameter|ConstraintRefParameter|ConstraintStrParameter
+;
+
+ConstraintStrParameter:
+	paramValue=STRING
+;
+
+ConstraintIntParameter:
+	paramValue=INT
+;
+	
+ConstraintRefParameter:
+	paramValue = [Property] 
+;
+
+Exception:
+	"exception" name=ID (shortDesc=STRING)? (longDesc=STRING)? "{"
+		(properties+=Property)* 		
+	"}";
+
+FeatureCompartment:
+	"compartment" name=ID (shortDesc=STRING)? (longDesc=STRING)? "{"
+		(features+=Feature)* 
+	"}";
+	
+	
+Service:
+	"service" name=ID
+	("extends" extends=[Service])?
+	("depends" depends+=[Service] ("," depends+=[Service])*)?
+	(shortDesc=STRING)? (longDesc=STRING)? 
+	"{"
+		("[" constraints+=ConstraintSpec ("," constraints+=ConstraintSpec)* "]")?
+		(operations+=Operation)*
+		(operationCompartmens+=OperationCompartment)*
+	"}";	
+	
+OperationCompartment:
+	"compartment" name=ID (shortDesc=STRING)? (longDesc=STRING)? "{"
+		(operations+=Operation)* 
+	"}";
+	
+	
+ValueObject:
+	"valueObject" name=ID
+	("extends" extends=[ValueObject])?
+	("depends" depends+=[Entity] ("," depends+=[Entity])*)? 
+	(shortDesc=STRING)? (longDesc=STRING)? 
+	"{"
+		("[" constraints+=ConstraintSpec ("," constraints+=ConstraintSpec)* "]")?
+		(properties+=Property)* 
+	"}";
+	
+Operation:
+	"op" ( (ordered?="ordered") |
+		(unique?="unique") | 
+		(required?="required"))*
+		type=[Classifier] (many?="[" (multiplicity=INT)? "]")? 
+		name=ID "("(parameters+=Parameter)?(","parameters+=Parameter)* ")"
+		("throws" exceptions+=[Exception] ("," exceptions+=[Exception])?)?
+		("[" constraints+=ConstraintSpec ("," constraints+=ConstraintSpec)* "]")?
+		(shortDesc=STRING)? (longDesc=STRING)?;
+
+/*
+	Atribut i referenca su objedinjeni u jedinstvenom konceptu property
+*/
+Property:
+	"prop" 
+		( (ordered?="ordered") |
+		(unique?="unique") | 
+		(readonly?="readonly") | 
+		(required?="required") )* 
+		(containment?="+")? 
+		type=[Classifier] (many?="[" (multiplicity=INT)? "]")?  name=ID 
+		("<>" oppositeEnd=[Property])? 
+		("[" constraints+=ConstraintSpec ("," constraints+=ConstraintSpec)* "]")?
+		(shortDesc=STRING)? (longDesc=STRING)? 
+		;
+
+
+Parameter:
+	( (ordered?="ordered") |
+	  (unique?="unique") | 
+	  (required?="required") )*
+	type=[Classifier] (many?="[" (multiplicity=INT)? "]")?  name=ID  
+	("[" constraints+=ConstraintSpec ("," constraints+=ConstraintSpec)* "]")?
+	(shortDesc=STRING)? (longDesc=STRING)?
+; 
+
+Enumeration:
+	"enum" name=ID (shortDesc=STRING)? (longDesc=STRING)? "{"
+		(literals+=EnumerationLiteral)+
+	"}";
+	
+EnumerationLiteral:
+	name=ID value=STRING (shortDesc=STRING)? (longDesc=STRING)?;
+
+
+
+// Enum DataType:
+//	Str="string"|Integer="int"|Boolean="bool"|DateTime="datetime"|Void="void"|Float="float";
+	

examples/textx/experiment.pf

+# This is a reference example file that demonstrates all
+# features of pyflies language
+
+
+# base abstract test
+# Used to specify timings for each test in the
+# experiment
+test base {
+  tmin 3000
+  tmax 5000
+  twait 3000
+}
+
+# test definition for simple visual
+# this test is non-abstract (defines visual simple test type)
+# but will not be used directly in experiment definition.
+# It will be used as a base for all visual simple tests
+# in this experiment.
+test mytest:visual_simple {
+  shape square
+  size large
+  color white
+}
+
+# Inheritance.
+# This test definition is non-abstract.
+# It inherits definition from base and mytest
+# therefore it does not need to specify test type or
+# timing settings.
+test mytest_3 < base, mytest {
+  trials 3
+}
+
+# This test is same as previous but defines
+# a 30 trials to run.
+test mytest_30< base, mytest {
+  trials 30
+}
+
+# Experiment consist of a sequence of blocks
+# (tests, introductions, sequences and randomize).
+experiment {
+
+  # subject block will collect data about
+  # the subject under experiment.
+  # A user will be presented with the GUI form to
+  # fill it. Experimenter may fill this on behalf of the subject.
+  # The form of the each subject line is:
+  # Variable_name, type, optional label
+  # Type can be: str, int, float or enumerated list
+  subject {
+    full_name str "First and last name"
+    age int
+    gender [Male, Female] "Gender"
+  }
+
+  # This is here just to show that test may be
+  # put under experiment directly.
+  # If the tests run in sequence (without randomization)
+  # this is an easiest way to define it.
+  test mytest_3
+
+  randomize {
+    # Following two sequences will 
+    # be randomized.
+
+    # Sequence block will be performed in
+    # the order of definition.
+    sequence {
+      # This is an introduction to the following test.
+      # Introduction is text between ======== separators.
+      # Separator consist of at least 3 '=' characters.
+      ================
+      Welcome
+      -------
+      In the following test you will be presented with a blank screen
+      where a white square will appear after a few seconds.
+      You should react by pressing the space bar as fast as you can when
+      the square appears.
+
+      This is a practice test with 3 trials.
+      Afterwards the real test will run with 30 trials.
+
+      Press space bar
+      ================
+
+      # This test will run in practice mode.
+      # This mode tells experiment runner
+      # not to collect any sample data.
+      test mytest_3 practice
+
+      ================
+      Real test with 30 trials
+      ------------------------
+      Press space bar
+      ================
+      # This is the real test run. Samples will be collected.
+      Test mytest_30
+
+    }
+
+    # This is here just to show that randomize block should contain
+    # more than 1 block. Those block runs will be randomized.
+    sequence {
+
+      test mytest_3 practice
+
+      =======
+      This is heading
+      ---------------
+      This is just to show another introduction screen.
+      =======
+
+      test mytest_30
+    }
+
+  }
+
+
+}

examples/textx/pyflies.tx

+/*
+  This is a TextX definition of pyFlies DSL for Reaction Time test
+  experiments definition.
+  Author: Igor R. Dejanovic <igor DOT dejanovic AT gmail DOT com>
+  Copyright: (c) 2014 Igor R. Dejanovic <igor DOT dejanovic AT gmail DOT com>
+  License: MIT License
+*/
+
+PyFliesModel:
+  TestDef* Experiment
+;
+
+TestDef:
+  "test" name=ID (":" test_type=TestType)? ( "<" inherits*={[TestDef] ","} )? "{"
+    TestParam*
+  "}"
+;
+
+TestType:
+  name=ID
+;
+
+TestParam:
+  name=ID TestParamValue label=TestParamLabel?
+;
+
+TestParamValue:
+  (value=ID|FLOAT|INT|STRING)|( '[' value+={ID ","} ']')
+;
+
+TestParamLabel:
+  label = STRING
+;
+
+
+Block:
+  Intro|Test|Sequence|Randomize
+;
+
+Experiment:
+  'experiment' '{'
+    blocks=Block*
+  '}'
+;
+
+Sequence:
+  'sequence' '{'
+    blocks=Block*
+  '}'
+;
+
+Randomize:
+  'randomize' '{'
+    blocks=Block*
+  '}'
+;
+
+Test:
+  'test' type=[TestDef] practice?="practice"
+;
+
+Intro:
+  /====*/
+  content=/([^=][^=]?[^=]?)*/
+  /====*/
+;
+
+// Special rule for comments
+Comment:
+  /#.*$/
+;
+

examples/textx/textx.py

+#######################################################################
+# Name: textx.py
+# Purpose: Demonstration of textX meta-language.
+# Author: Igor R. Dejanovic <igor DOT dejanovic AT gmail DOT com>
+# Copyright: (c) 2014 Igor R. Dejanovic <igor DOT dejanovic AT gmail DOT com>
+# License: MIT License
+#######################################################################
+
+from arpeggio.textx import get_parser
+
+def main(debug=False):
+
+    # Load textX description of the language
+    with open('pyflies.tx', 'r') as f:
+        language_def = f.read()
+
+    # Create parser for the new lanuage
+    parser = get_parser(language_def, debug=debug)
+
+    # Parse pyflies example
+    with open('experiment.pf') as f:
+        pyflies_input = f.read()
+    parse_tree = parser.parse(pyflies_input)
+
+    result = parser.getASG()
+
+if __name__ == "__main__":
+    # In debug mode dot (graphviz) files for parser model
+    # and parse tree will be created for visualization.
+    # Checkout current folder for .dot files.
+    main(debug=True)
+