Module margo_parser.tokenizer.margo_transformer

Expand source code
from lark import Lark, Transformer, Tree, Token
import json
import yaml


class MargoTransformer(Transformer):
    def block(self, b):

        statements = list(filter(lambda s: s != ["ENDBLOCK"], b))

        # every other token should be an endblock
        assert len(statements) == len(b) / 2

        # a block is made up only of statements
        for s in statements:
            assert s["TYPE"] == "STATEMENT"
            assert type(s["BODY"]) == list
            assert len(s["BODY"]) < 2

        nonempty_statements = list(filter(lambda s: len(s["BODY"]) > 0, statements))

        unwrapped_statements = list(map(lambda x: x["BODY"][0], nonempty_statements))

        return {
            "SYNTAX": "MARGO",
            "TYPE": "BLOCK",
            "VERSION": "0",
            "BODY": unwrapped_statements,
        }

    def directive(self, d):
        assert len(d) == 1
        name = str(d[0])
        return {"TYPE": "DIRECTIVE", "NAME": name}

    def evf_assignment(self, c):
        assert len(c) == 3
        key = c[0]
        lang = c[1].lower().strip()
        body = c[2]
        parsed = False
        value = None

        if lang == "raw":
            value = body
            parsed = True

        if lang == "margo":
            value = body
            parsed = True

        if lang == "yaml":
            try:
                value = yaml.load(body, Loader=yaml.FullLoader)
                parsed = True
            except Exception as e:
                print(f"Error parsing YAML {body}: {e}")

        if lang == "json":
            try:
                value = json.loads(body)
                parsed = True
            except Exception as e:
                print(f"Error parsing JSON '{body}': {e}")
                pass

        return {
            "TYPE": "DECLARATION",
            "LANGUAGE": lang,
            "PARSED": parsed,
            "BODY": body,
            "NAME": key,
            "VALUE": value,
        }

    def statement(self, s: Tree):

        # # Statements should contain two items, an expression and an endblock
        # assert len(s) == 2
        # assert s[1] == ["ENDBLOCK"]
        assert type(s) == list

        return {"TYPE": "STATEMENT", "BODY": s}

    def expression(self, e: Tree):
        return {"TYPE": "EXPRESSION", "BODY": e}

    def ENDBLOCK(self, eb):
        return ["ENDBLOCK"]

    def string(self, s):
        return s[0]

    def builtin(self, s):

        assert len(s) == 1

        return {"TYPE": "BUILTIN", "BODY": s[0]}

    # def view_statement(self, v):
    #     (keys) = v
    #     return {"NAME": "view", "VIEW_LIST": v}

    def IGNORE_CELL(self, _):
        return {"NAME": "IGNORE_CELL"}

    def function(self, f):
        return ["FUNCTION", f]

    def function_name(self, fn):
        return str(fn[0])

    def argument_list(self, al):
        (s, *vals) = al
        
        return vals

    def mvf_assignment(self, kvp):
        (k, *vals) = kvp

        return self.evf_assignment([k, "json", json.dumps(vals)])  # key

    def KEY(self, k):
        return str(k)

    def false(self, _):
        return False

    def true(self, _):
        return True

    def null(self, _):
        return None

    def QSTRING(self, s):
        if len(str(s)) < 1:
            return ""
        return str(s)[1:-1]

    def value(self, v):
        return v[0]

    def number(self, n):
        (n,) = n
        try:
            return int(n.value)
        except BaseException:
            pass
        try:
            return float(n.value)
        except BaseException:
            pass

Classes

class MargoTransformer (visit_tokens=True)

Transformers visit each node of the tree, and run the appropriate method on it according to the node's data.

Calls its methods (provided by user via inheritance) according to tree.data. The returned value replaces the old one in the structure.

They work bottom-up (or depth-first), starting with the leaves and ending at the root of the tree. Transformers can be used to implement map & reduce patterns. Because nodes are reduced from leaf to root, at any point the callbacks may assume the children have already been transformed (if applicable).

Transformer can do anything Visitor can do, but because it reconstructs the tree, it is slightly less efficient. It can be used to implement map or reduce patterns.

All these classes implement the transformer interface:

  • Transformer - Recursively transforms the tree. This is the one you probably want.
  • Transformer_InPlace - Non-recursive. Changes the tree in-place instead of returning new instances
  • Transformer_InPlaceRecursive - Recursive. Changes the tree in-place instead of returning new instances

Parameters

visit_tokens: By default, transformers only visit rules. visit_tokens=True will tell Transformer to visit tokens as well. This is a slightly slower alternative to lexer_callbacks but it's easier to maintain and works for all algorithms (even when there isn't a lexer).

Expand source code
class MargoTransformer(Transformer):
    def block(self, b):

        statements = list(filter(lambda s: s != ["ENDBLOCK"], b))

        # every other token should be an endblock
        assert len(statements) == len(b) / 2

        # a block is made up only of statements
        for s in statements:
            assert s["TYPE"] == "STATEMENT"
            assert type(s["BODY"]) == list
            assert len(s["BODY"]) < 2

        nonempty_statements = list(filter(lambda s: len(s["BODY"]) > 0, statements))

        unwrapped_statements = list(map(lambda x: x["BODY"][0], nonempty_statements))

        return {
            "SYNTAX": "MARGO",
            "TYPE": "BLOCK",
            "VERSION": "0",
            "BODY": unwrapped_statements,
        }

    def directive(self, d):
        assert len(d) == 1
        name = str(d[0])
        return {"TYPE": "DIRECTIVE", "NAME": name}

    def evf_assignment(self, c):
        assert len(c) == 3
        key = c[0]
        lang = c[1].lower().strip()
        body = c[2]
        parsed = False
        value = None

        if lang == "raw":
            value = body
            parsed = True

        if lang == "margo":
            value = body
            parsed = True

        if lang == "yaml":
            try:
                value = yaml.load(body, Loader=yaml.FullLoader)
                parsed = True
            except Exception as e:
                print(f"Error parsing YAML {body}: {e}")

        if lang == "json":
            try:
                value = json.loads(body)
                parsed = True
            except Exception as e:
                print(f"Error parsing JSON '{body}': {e}")
                pass

        return {
            "TYPE": "DECLARATION",
            "LANGUAGE": lang,
            "PARSED": parsed,
            "BODY": body,
            "NAME": key,
            "VALUE": value,
        }

    def statement(self, s: Tree):

        # # Statements should contain two items, an expression and an endblock
        # assert len(s) == 2
        # assert s[1] == ["ENDBLOCK"]
        assert type(s) == list

        return {"TYPE": "STATEMENT", "BODY": s}

    def expression(self, e: Tree):
        return {"TYPE": "EXPRESSION", "BODY": e}

    def ENDBLOCK(self, eb):
        return ["ENDBLOCK"]

    def string(self, s):
        return s[0]

    def builtin(self, s):

        assert len(s) == 1

        return {"TYPE": "BUILTIN", "BODY": s[0]}

    # def view_statement(self, v):
    #     (keys) = v
    #     return {"NAME": "view", "VIEW_LIST": v}

    def IGNORE_CELL(self, _):
        return {"NAME": "IGNORE_CELL"}

    def function(self, f):
        return ["FUNCTION", f]

    def function_name(self, fn):
        return str(fn[0])

    def argument_list(self, al):
        (s, *vals) = al
        
        return vals

    def mvf_assignment(self, kvp):
        (k, *vals) = kvp

        return self.evf_assignment([k, "json", json.dumps(vals)])  # key

    def KEY(self, k):
        return str(k)

    def false(self, _):
        return False

    def true(self, _):
        return True

    def null(self, _):
        return None

    def QSTRING(self, s):
        if len(str(s)) < 1:
            return ""
        return str(s)[1:-1]

    def value(self, v):
        return v[0]

    def number(self, n):
        (n,) = n
        try:
            return int(n.value)
        except BaseException:
            pass
        try:
            return float(n.value)
        except BaseException:
            pass

Ancestors

  • lark.visitors.Transformer
  • lark.visitors._Decoratable

Methods

def ENDBLOCK(self, eb)
Expand source code
def ENDBLOCK(self, eb):
    return ["ENDBLOCK"]
def IGNORE_CELL(self, _)
Expand source code
def IGNORE_CELL(self, _):
    return {"NAME": "IGNORE_CELL"}
def KEY(self, k)
Expand source code
def KEY(self, k):
    return str(k)
def QSTRING(self, s)
Expand source code
def QSTRING(self, s):
    if len(str(s)) < 1:
        return ""
    return str(s)[1:-1]
def argument_list(self, al)
Expand source code
def argument_list(self, al):
    (s, *vals) = al
    
    return vals
def block(self, b)
Expand source code
def block(self, b):

    statements = list(filter(lambda s: s != ["ENDBLOCK"], b))

    # every other token should be an endblock
    assert len(statements) == len(b) / 2

    # a block is made up only of statements
    for s in statements:
        assert s["TYPE"] == "STATEMENT"
        assert type(s["BODY"]) == list
        assert len(s["BODY"]) < 2

    nonempty_statements = list(filter(lambda s: len(s["BODY"]) > 0, statements))

    unwrapped_statements = list(map(lambda x: x["BODY"][0], nonempty_statements))

    return {
        "SYNTAX": "MARGO",
        "TYPE": "BLOCK",
        "VERSION": "0",
        "BODY": unwrapped_statements,
    }
def builtin(self, s)
Expand source code
def builtin(self, s):

    assert len(s) == 1

    return {"TYPE": "BUILTIN", "BODY": s[0]}
def directive(self, d)
Expand source code
def directive(self, d):
    assert len(d) == 1
    name = str(d[0])
    return {"TYPE": "DIRECTIVE", "NAME": name}
def evf_assignment(self, c)
Expand source code
def evf_assignment(self, c):
    assert len(c) == 3
    key = c[0]
    lang = c[1].lower().strip()
    body = c[2]
    parsed = False
    value = None

    if lang == "raw":
        value = body
        parsed = True

    if lang == "margo":
        value = body
        parsed = True

    if lang == "yaml":
        try:
            value = yaml.load(body, Loader=yaml.FullLoader)
            parsed = True
        except Exception as e:
            print(f"Error parsing YAML {body}: {e}")

    if lang == "json":
        try:
            value = json.loads(body)
            parsed = True
        except Exception as e:
            print(f"Error parsing JSON '{body}': {e}")
            pass

    return {
        "TYPE": "DECLARATION",
        "LANGUAGE": lang,
        "PARSED": parsed,
        "BODY": body,
        "NAME": key,
        "VALUE": value,
    }
def expression(self, e: lark.tree.Tree)
Expand source code
def expression(self, e: Tree):
    return {"TYPE": "EXPRESSION", "BODY": e}
def false(self, _)
Expand source code
def false(self, _):
    return False
def function(self, f)
Expand source code
def function(self, f):
    return ["FUNCTION", f]
def function_name(self, fn)
Expand source code
def function_name(self, fn):
    return str(fn[0])
def mvf_assignment(self, kvp)
Expand source code
def mvf_assignment(self, kvp):
    (k, *vals) = kvp

    return self.evf_assignment([k, "json", json.dumps(vals)])  # key
def null(self, _)
Expand source code
def null(self, _):
    return None
def number(self, n)
Expand source code
def number(self, n):
    (n,) = n
    try:
        return int(n.value)
    except BaseException:
        pass
    try:
        return float(n.value)
    except BaseException:
        pass
def statement(self, s: lark.tree.Tree)
Expand source code
def statement(self, s: Tree):

    # # Statements should contain two items, an expression and an endblock
    # assert len(s) == 2
    # assert s[1] == ["ENDBLOCK"]
    assert type(s) == list

    return {"TYPE": "STATEMENT", "BODY": s}
def string(self, s)
Expand source code
def string(self, s):
    return s[0]
def true(self, _)
Expand source code
def true(self, _):
    return True
def value(self, v)
Expand source code
def value(self, v):
    return v[0]