Module margo_parser.tokenizer.margo_transformer
Lark Transformer for Margo syntax
Expand source code
"""Lark Transformer for Margo syntax"""
from lark import Transformer, Tree
import json
import yaml
class MargoTransformer(Transformer):
"""Lark Transformer for Margo syntax"""
def block(self, b):
"""
*Transform a Margo block*
A Margo block is a sequence of statements separated by endblocks, i.e.:
```margo
{statement-1} ::
{statement-2} ::
{statement-3} ::
```
"""
statements = list(filter(lambda s: s != ["ENDBLOCK"], b))
# every other token should be an endblock, so there should always
# be twice as many items in b as as there are statements
assert len(statements) == len(b) / 2
# a block is made up only of statements
for s in statements:
assert s["TYPE"] == "STATEMENT"
assert type(s["BODY"]) == list
assert len(s["BODY"]) < 2
nonempty_statements = list(
filter(lambda s: len(s["BODY"]) > 0, statements)
)
unwrapped_statements = list(
map(lambda x: x["BODY"][0], nonempty_statements)
)
return {
"SYNTAX": "MARGO",
"TYPE": "BLOCK",
"VERSION": "0",
"BODY": unwrapped_statements,
}
def directive(self, d):
"""
*Transform a Margo directive*
A directive is a statement that makes no assignment, i.e.:
```margo
ignore-cell ::
```
In the above example, `ignore-cell` is the directive statement
and `::` is the endblock.
"""
assert len(d) == 1
name = str(d[0])
return {"TYPE": "DIRECTIVE", "NAME": name}
def evf_assignment(self, c):
"""
*Transform an external value format assignment*
External value formats include:
* JSON
* YAML
* raw (plain text)
Example:
```margo
requirements [yaml]:'
- nbformat
- requests
' ::
```
In the above example:
* `requirements` is the name
* `[yaml]` specifies the format
* The quoted string following the colon is the value
assigned to the name.
"""
assert len(c) == 3
key = c[0]
lang = c[1].lower().strip()
body = c[2]
parsed = False
value = None
if lang == "raw":
value = body
parsed = True
if lang == "margo":
value = body
parsed = True
if lang == "yaml":
try:
value = yaml.load(body, Loader=yaml.FullLoader)
parsed = True
except Exception as e:
print(f"Error parsing YAML {body}: {e}")
if lang == "json":
try:
value = json.loads(body)
parsed = True
except Exception as e:
print(f"Error parsing JSON '{body}': {e}")
pass
return {
"TYPE": "DECLARATION",
"LANGUAGE": lang,
"PARSED": parsed,
"BODY": body,
"NAME": key,
"VALUE": value,
}
def statement(self, s: Tree):
# # Statements should contain two items, an expression and an endblock
# assert len(s) == 2
# assert s[1] == ["ENDBLOCK"]
assert type(s) == list
return {"TYPE": "STATEMENT", "BODY": s}
def expression(self, e: Tree):
return {"TYPE": "EXPRESSION", "BODY": e}
def ENDBLOCK(self, eb):
return ["ENDBLOCK"]
def string(self, s):
return s[0]
def builtin(self, s):
assert len(s) == 1
return {"TYPE": "BUILTIN", "BODY": s[0]}
# def view_statement(self, v):
# (keys) = v
# return {"NAME": "view", "VIEW_LIST": v}
def IGNORE_CELL(self, _):
return {"NAME": "IGNORE_CELL"}
def function(self, f):
return ["FUNCTION", f]
def function_name(self, fn):
return str(fn[0])
def argument_list(self, al):
(s, *vals) = al
return vals
def mvf_assignment(self, kvp):
(k, *vals) = kvp
return self.evf_assignment([k, "json", json.dumps(vals)]) # key
def KEY(self, k):
return str(k)
def false(self, _):
return False
def true(self, _):
return True
def null(self, _):
return None
def QSTRING(self, s):
if len(str(s)) < 1:
return ""
return str(s)[1:-1]
def value(self, v):
return v[0]
def number(self, n):
(n,) = n
try:
return int(n.value)
except BaseException:
pass
try:
return float(n.value)
except BaseException:
pass
Classes
class MargoTransformer (visit_tokens=True)
-
Lark Transformer for Margo syntax
Expand source code
class MargoTransformer(Transformer): """Lark Transformer for Margo syntax""" def block(self, b): """ *Transform a Margo block* A Margo block is a sequence of statements separated by endblocks, i.e.: ```margo {statement-1} :: {statement-2} :: {statement-3} :: ``` """ statements = list(filter(lambda s: s != ["ENDBLOCK"], b)) # every other token should be an endblock, so there should always # be twice as many items in b as as there are statements assert len(statements) == len(b) / 2 # a block is made up only of statements for s in statements: assert s["TYPE"] == "STATEMENT" assert type(s["BODY"]) == list assert len(s["BODY"]) < 2 nonempty_statements = list( filter(lambda s: len(s["BODY"]) > 0, statements) ) unwrapped_statements = list( map(lambda x: x["BODY"][0], nonempty_statements) ) return { "SYNTAX": "MARGO", "TYPE": "BLOCK", "VERSION": "0", "BODY": unwrapped_statements, } def directive(self, d): """ *Transform a Margo directive* A directive is a statement that makes no assignment, i.e.: ```margo ignore-cell :: ``` In the above example, `ignore-cell` is the directive statement and `::` is the endblock. """ assert len(d) == 1 name = str(d[0]) return {"TYPE": "DIRECTIVE", "NAME": name} def evf_assignment(self, c): """ *Transform an external value format assignment* External value formats include: * JSON * YAML * raw (plain text) Example: ```margo requirements [yaml]:' - nbformat - requests ' :: ``` In the above example: * `requirements` is the name * `[yaml]` specifies the format * The quoted string following the colon is the value assigned to the name. """ assert len(c) == 3 key = c[0] lang = c[1].lower().strip() body = c[2] parsed = False value = None if lang == "raw": value = body parsed = True if lang == "margo": value = body parsed = True if lang == "yaml": try: value = yaml.load(body, Loader=yaml.FullLoader) parsed = True except Exception as e: print(f"Error parsing YAML {body}: {e}") if lang == "json": try: value = json.loads(body) parsed = True except Exception as e: print(f"Error parsing JSON '{body}': {e}") pass return { "TYPE": "DECLARATION", "LANGUAGE": lang, "PARSED": parsed, "BODY": body, "NAME": key, "VALUE": value, } def statement(self, s: Tree): # # Statements should contain two items, an expression and an endblock # assert len(s) == 2 # assert s[1] == ["ENDBLOCK"] assert type(s) == list return {"TYPE": "STATEMENT", "BODY": s} def expression(self, e: Tree): return {"TYPE": "EXPRESSION", "BODY": e} def ENDBLOCK(self, eb): return ["ENDBLOCK"] def string(self, s): return s[0] def builtin(self, s): assert len(s) == 1 return {"TYPE": "BUILTIN", "BODY": s[0]} # def view_statement(self, v): # (keys) = v # return {"NAME": "view", "VIEW_LIST": v} def IGNORE_CELL(self, _): return {"NAME": "IGNORE_CELL"} def function(self, f): return ["FUNCTION", f] def function_name(self, fn): return str(fn[0]) def argument_list(self, al): (s, *vals) = al return vals def mvf_assignment(self, kvp): (k, *vals) = kvp return self.evf_assignment([k, "json", json.dumps(vals)]) # key def KEY(self, k): return str(k) def false(self, _): return False def true(self, _): return True def null(self, _): return None def QSTRING(self, s): if len(str(s)) < 1: return "" return str(s)[1:-1] def value(self, v): return v[0] def number(self, n): (n,) = n try: return int(n.value) except BaseException: pass try: return float(n.value) except BaseException: pass
Ancestors
- lark.visitors.Transformer
- lark.visitors._Decoratable
Methods
def ENDBLOCK(self, eb)
-
Expand source code
def ENDBLOCK(self, eb): return ["ENDBLOCK"]
def IGNORE_CELL(self, _)
-
Expand source code
def IGNORE_CELL(self, _): return {"NAME": "IGNORE_CELL"}
def KEY(self, k)
-
Expand source code
def KEY(self, k): return str(k)
def QSTRING(self, s)
-
Expand source code
def QSTRING(self, s): if len(str(s)) < 1: return "" return str(s)[1:-1]
def argument_list(self, al)
-
Expand source code
def argument_list(self, al): (s, *vals) = al return vals
def block(self, b)
-
Transform a Margo block
A Margo block is a sequence of statements separated by endblocks, i.e.:
{statement-1} :: {statement-2} :: {statement-3} ::
Expand source code
def block(self, b): """ *Transform a Margo block* A Margo block is a sequence of statements separated by endblocks, i.e.: ```margo {statement-1} :: {statement-2} :: {statement-3} :: ``` """ statements = list(filter(lambda s: s != ["ENDBLOCK"], b)) # every other token should be an endblock, so there should always # be twice as many items in b as as there are statements assert len(statements) == len(b) / 2 # a block is made up only of statements for s in statements: assert s["TYPE"] == "STATEMENT" assert type(s["BODY"]) == list assert len(s["BODY"]) < 2 nonempty_statements = list( filter(lambda s: len(s["BODY"]) > 0, statements) ) unwrapped_statements = list( map(lambda x: x["BODY"][0], nonempty_statements) ) return { "SYNTAX": "MARGO", "TYPE": "BLOCK", "VERSION": "0", "BODY": unwrapped_statements, }
def builtin(self, s)
-
Expand source code
def builtin(self, s): assert len(s) == 1 return {"TYPE": "BUILTIN", "BODY": s[0]}
def directive(self, d)
-
Transform a Margo directive
A directive is a statement that makes no assignment, i.e.:
ignore-cell ::
In the above example,
ignore-cell
is the directive statement and::
is the endblock.Expand source code
def directive(self, d): """ *Transform a Margo directive* A directive is a statement that makes no assignment, i.e.: ```margo ignore-cell :: ``` In the above example, `ignore-cell` is the directive statement and `::` is the endblock. """ assert len(d) == 1 name = str(d[0]) return {"TYPE": "DIRECTIVE", "NAME": name}
def evf_assignment(self, c)
-
Transform an external value format assignment
External value formats include: * JSON * YAML * raw (plain text)
Example:
requirements [yaml]:' - nbformat - requests ' ::
In the above example: *
requirements
is the name *[yaml]
specifies the format * The quoted string following the colon is the value assigned to the name.Expand source code
def evf_assignment(self, c): """ *Transform an external value format assignment* External value formats include: * JSON * YAML * raw (plain text) Example: ```margo requirements [yaml]:' - nbformat - requests ' :: ``` In the above example: * `requirements` is the name * `[yaml]` specifies the format * The quoted string following the colon is the value assigned to the name. """ assert len(c) == 3 key = c[0] lang = c[1].lower().strip() body = c[2] parsed = False value = None if lang == "raw": value = body parsed = True if lang == "margo": value = body parsed = True if lang == "yaml": try: value = yaml.load(body, Loader=yaml.FullLoader) parsed = True except Exception as e: print(f"Error parsing YAML {body}: {e}") if lang == "json": try: value = json.loads(body) parsed = True except Exception as e: print(f"Error parsing JSON '{body}': {e}") pass return { "TYPE": "DECLARATION", "LANGUAGE": lang, "PARSED": parsed, "BODY": body, "NAME": key, "VALUE": value, }
def expression(self, e: lark.tree.Tree)
-
Expand source code
def expression(self, e: Tree): return {"TYPE": "EXPRESSION", "BODY": e}
def false(self, _)
-
Expand source code
def false(self, _): return False
def function(self, f)
-
Expand source code
def function(self, f): return ["FUNCTION", f]
def function_name(self, fn)
-
Expand source code
def function_name(self, fn): return str(fn[0])
def mvf_assignment(self, kvp)
-
Expand source code
def mvf_assignment(self, kvp): (k, *vals) = kvp return self.evf_assignment([k, "json", json.dumps(vals)]) # key
def null(self, _)
-
Expand source code
def null(self, _): return None
def number(self, n)
-
Expand source code
def number(self, n): (n,) = n try: return int(n.value) except BaseException: pass try: return float(n.value) except BaseException: pass
def statement(self, s: lark.tree.Tree)
-
Expand source code
def statement(self, s: Tree): # # Statements should contain two items, an expression and an endblock # assert len(s) == 2 # assert s[1] == ["ENDBLOCK"] assert type(s) == list return {"TYPE": "STATEMENT", "BODY": s}
def string(self, s)
-
Expand source code
def string(self, s): return s[0]
def true(self, _)
-
Expand source code
def true(self, _): return True
def value(self, v)
-
Expand source code
def value(self, v): return v[0]