Module margo_parser.tokenizer.margo_transformer
Expand source code
from lark import Lark, Transformer, Tree, Token
import json
import yaml
class MargoTransformer(Transformer):
def block(self, b):
statements = list(filter(lambda s: s != ["ENDBLOCK"], b))
# every other token should be an endblock
assert len(statements) == len(b) / 2
# a block is made up only of statements
for s in statements:
assert s["TYPE"] == "STATEMENT"
assert type(s["BODY"]) == list
assert len(s["BODY"]) < 2
nonempty_statements = list(filter(lambda s: len(s["BODY"]) > 0, statements))
unwrapped_statements = list(map(lambda x: x["BODY"][0], nonempty_statements))
return {
"SYNTAX": "MARGO",
"TYPE": "BLOCK",
"VERSION": "0",
"BODY": unwrapped_statements,
}
def directive(self, d):
assert len(d) == 1
name = str(d[0])
return {"TYPE": "DIRECTIVE", "NAME": name}
def evf_assignment(self, c):
assert len(c) == 3
key = c[0]
lang = c[1].lower().strip()
body = c[2]
parsed = False
value = None
if lang == "raw":
value = body
parsed = True
if lang == "margo":
value = body
parsed = True
if lang == "yaml":
try:
value = yaml.load(body, Loader=yaml.FullLoader)
parsed = True
except Exception as e:
print(f"Error parsing YAML {body}: {e}")
if lang == "json":
try:
value = json.loads(body)
parsed = True
except Exception as e:
print(f"Error parsing JSON '{body}': {e}")
pass
return {
"TYPE": "DECLARATION",
"LANGUAGE": lang,
"PARSED": parsed,
"BODY": body,
"NAME": key,
"VALUE": value,
}
def statement(self, s: Tree):
# # Statements should contain two items, an expression and an endblock
# assert len(s) == 2
# assert s[1] == ["ENDBLOCK"]
assert type(s) == list
return {"TYPE": "STATEMENT", "BODY": s}
def expression(self, e: Tree):
return {"TYPE": "EXPRESSION", "BODY": e}
def ENDBLOCK(self, eb):
return ["ENDBLOCK"]
def string(self, s):
return s[0]
def builtin(self, s):
assert len(s) == 1
return {"TYPE": "BUILTIN", "BODY": s[0]}
# def view_statement(self, v):
# (keys) = v
# return {"NAME": "view", "VIEW_LIST": v}
def IGNORE_CELL(self, _):
return {"NAME": "IGNORE_CELL"}
def function(self, f):
return ["FUNCTION", f]
def function_name(self, fn):
return str(fn[0])
def argument_list(self, al):
(s, *vals) = al
return vals
def mvf_assignment(self, kvp):
(k, *vals) = kvp
return self.evf_assignment([k, "json", json.dumps(vals)]) # key
def KEY(self, k):
return str(k)
def false(self, _):
return False
def true(self, _):
return True
def null(self, _):
return None
def QSTRING(self, s):
if len(str(s)) < 1:
return ""
return str(s)[1:-1]
def value(self, v):
return v[0]
def number(self, n):
(n,) = n
try:
return int(n.value)
except BaseException:
pass
try:
return float(n.value)
except BaseException:
pass
Classes
class MargoTransformer (visit_tokens=True)
-
Transformers visit each node of the tree, and run the appropriate method on it according to the node's data.
Calls its methods (provided by user via inheritance) according to
tree.data
. The returned value replaces the old one in the structure.They work bottom-up (or depth-first), starting with the leaves and ending at the root of the tree. Transformers can be used to implement map & reduce patterns. Because nodes are reduced from leaf to root, at any point the callbacks may assume the children have already been transformed (if applicable).
Transformer
can do anythingVisitor
can do, but because it reconstructs the tree, it is slightly less efficient. It can be used to implement map or reduce patterns.All these classes implement the transformer interface:
Transformer
- Recursively transforms the tree. This is the one you probably want.Transformer_InPlace
- Non-recursive. Changes the tree in-place instead of returning new instancesTransformer_InPlaceRecursive
- Recursive. Changes the tree in-place instead of returning new instances
Parameters
visit_tokens: By default, transformers only visit rules. visit_tokens=True will tell
Transformer
to visit tokens as well. This is a slightly slower alternative to lexer_callbacks but it's easier to maintain and works for all algorithms (even when there isn't a lexer).Expand source code
class MargoTransformer(Transformer): def block(self, b): statements = list(filter(lambda s: s != ["ENDBLOCK"], b)) # every other token should be an endblock assert len(statements) == len(b) / 2 # a block is made up only of statements for s in statements: assert s["TYPE"] == "STATEMENT" assert type(s["BODY"]) == list assert len(s["BODY"]) < 2 nonempty_statements = list(filter(lambda s: len(s["BODY"]) > 0, statements)) unwrapped_statements = list(map(lambda x: x["BODY"][0], nonempty_statements)) return { "SYNTAX": "MARGO", "TYPE": "BLOCK", "VERSION": "0", "BODY": unwrapped_statements, } def directive(self, d): assert len(d) == 1 name = str(d[0]) return {"TYPE": "DIRECTIVE", "NAME": name} def evf_assignment(self, c): assert len(c) == 3 key = c[0] lang = c[1].lower().strip() body = c[2] parsed = False value = None if lang == "raw": value = body parsed = True if lang == "margo": value = body parsed = True if lang == "yaml": try: value = yaml.load(body, Loader=yaml.FullLoader) parsed = True except Exception as e: print(f"Error parsing YAML {body}: {e}") if lang == "json": try: value = json.loads(body) parsed = True except Exception as e: print(f"Error parsing JSON '{body}': {e}") pass return { "TYPE": "DECLARATION", "LANGUAGE": lang, "PARSED": parsed, "BODY": body, "NAME": key, "VALUE": value, } def statement(self, s: Tree): # # Statements should contain two items, an expression and an endblock # assert len(s) == 2 # assert s[1] == ["ENDBLOCK"] assert type(s) == list return {"TYPE": "STATEMENT", "BODY": s} def expression(self, e: Tree): return {"TYPE": "EXPRESSION", "BODY": e} def ENDBLOCK(self, eb): return ["ENDBLOCK"] def string(self, s): return s[0] def builtin(self, s): assert len(s) == 1 return {"TYPE": "BUILTIN", "BODY": s[0]} # def view_statement(self, v): # (keys) = v # return {"NAME": "view", "VIEW_LIST": v} def IGNORE_CELL(self, _): return {"NAME": "IGNORE_CELL"} def function(self, f): return ["FUNCTION", f] def function_name(self, fn): return str(fn[0]) def argument_list(self, al): (s, *vals) = al return vals def mvf_assignment(self, kvp): (k, *vals) = kvp return self.evf_assignment([k, "json", json.dumps(vals)]) # key def KEY(self, k): return str(k) def false(self, _): return False def true(self, _): return True def null(self, _): return None def QSTRING(self, s): if len(str(s)) < 1: return "" return str(s)[1:-1] def value(self, v): return v[0] def number(self, n): (n,) = n try: return int(n.value) except BaseException: pass try: return float(n.value) except BaseException: pass
Ancestors
- lark.visitors.Transformer
- lark.visitors._Decoratable
Methods
def ENDBLOCK(self, eb)
-
Expand source code
def ENDBLOCK(self, eb): return ["ENDBLOCK"]
def IGNORE_CELL(self, _)
-
Expand source code
def IGNORE_CELL(self, _): return {"NAME": "IGNORE_CELL"}
def KEY(self, k)
-
Expand source code
def KEY(self, k): return str(k)
def QSTRING(self, s)
-
Expand source code
def QSTRING(self, s): if len(str(s)) < 1: return "" return str(s)[1:-1]
def argument_list(self, al)
-
Expand source code
def argument_list(self, al): (s, *vals) = al return vals
def block(self, b)
-
Expand source code
def block(self, b): statements = list(filter(lambda s: s != ["ENDBLOCK"], b)) # every other token should be an endblock assert len(statements) == len(b) / 2 # a block is made up only of statements for s in statements: assert s["TYPE"] == "STATEMENT" assert type(s["BODY"]) == list assert len(s["BODY"]) < 2 nonempty_statements = list(filter(lambda s: len(s["BODY"]) > 0, statements)) unwrapped_statements = list(map(lambda x: x["BODY"][0], nonempty_statements)) return { "SYNTAX": "MARGO", "TYPE": "BLOCK", "VERSION": "0", "BODY": unwrapped_statements, }
def builtin(self, s)
-
Expand source code
def builtin(self, s): assert len(s) == 1 return {"TYPE": "BUILTIN", "BODY": s[0]}
def directive(self, d)
-
Expand source code
def directive(self, d): assert len(d) == 1 name = str(d[0]) return {"TYPE": "DIRECTIVE", "NAME": name}
def evf_assignment(self, c)
-
Expand source code
def evf_assignment(self, c): assert len(c) == 3 key = c[0] lang = c[1].lower().strip() body = c[2] parsed = False value = None if lang == "raw": value = body parsed = True if lang == "margo": value = body parsed = True if lang == "yaml": try: value = yaml.load(body, Loader=yaml.FullLoader) parsed = True except Exception as e: print(f"Error parsing YAML {body}: {e}") if lang == "json": try: value = json.loads(body) parsed = True except Exception as e: print(f"Error parsing JSON '{body}': {e}") pass return { "TYPE": "DECLARATION", "LANGUAGE": lang, "PARSED": parsed, "BODY": body, "NAME": key, "VALUE": value, }
def expression(self, e: lark.tree.Tree)
-
Expand source code
def expression(self, e: Tree): return {"TYPE": "EXPRESSION", "BODY": e}
def false(self, _)
-
Expand source code
def false(self, _): return False
def function(self, f)
-
Expand source code
def function(self, f): return ["FUNCTION", f]
def function_name(self, fn)
-
Expand source code
def function_name(self, fn): return str(fn[0])
def mvf_assignment(self, kvp)
-
Expand source code
def mvf_assignment(self, kvp): (k, *vals) = kvp return self.evf_assignment([k, "json", json.dumps(vals)]) # key
def null(self, _)
-
Expand source code
def null(self, _): return None
def number(self, n)
-
Expand source code
def number(self, n): (n,) = n try: return int(n.value) except BaseException: pass try: return float(n.value) except BaseException: pass
def statement(self, s: lark.tree.Tree)
-
Expand source code
def statement(self, s: Tree): # # Statements should contain two items, an expression and an endblock # assert len(s) == 2 # assert s[1] == ["ENDBLOCK"] assert type(s) == list return {"TYPE": "STATEMENT", "BODY": s}
def string(self, s)
-
Expand source code
def string(self, s): return s[0]
def true(self, _)
-
Expand source code
def true(self, _): return True
def value(self, v)
-
Expand source code
def value(self, v): return v[0]