Module margo_parser.tokenizer.margo_transformer

Lark Transformer for Margo syntax

Expand source code
"""Lark Transformer for Margo syntax"""

from lark import Transformer, Tree
import json
import yaml


class MargoTransformer(Transformer):
    """Lark Transformer for Margo syntax"""

    def block(self, b):
        """
        
        *Transform a Margo block*

        A Margo block is a sequence of statements separated by endblocks, i.e.:

        ```margo
        {statement-1} ::
        {statement-2} ::
        {statement-3} ::
        ```
        """

        statements = list(filter(lambda s: s != ["ENDBLOCK"], b))

        # every other token should be an endblock, so there should always
        # be twice as many items in b as as there are statements
        assert len(statements) == len(b) / 2

        # a block is made up only of statements
        for s in statements:
            assert s["TYPE"] == "STATEMENT"
            assert type(s["BODY"]) == list
            assert len(s["BODY"]) < 2

        nonempty_statements = list(
            filter(lambda s: len(s["BODY"]) > 0, statements)
        )

        unwrapped_statements = list(
            map(lambda x: x["BODY"][0], nonempty_statements)
        )

        return {
            "SYNTAX": "MARGO",
            "TYPE": "BLOCK",
            "VERSION": "0",
            "BODY": unwrapped_statements,
        }

    def directive(self, d):
        """
        *Transform a Margo directive*
        
        A directive is a statement that makes no assignment, i.e.:

        ```margo
        ignore-cell ::
        ```

        In the above example, `ignore-cell` is the directive statement
        and `::` is the endblock.
        """

        assert len(d) == 1
        name = str(d[0])
        return {"TYPE": "DIRECTIVE", "NAME": name}

    def evf_assignment(self, c):
        """
        *Transform an external value format assignment*

        External value formats include:
        * JSON
        * YAML
        * raw (plain text)

        Example:

        ```margo
        requirements [yaml]:'
            - nbformat
            - requests
        ' ::
        ```

        In the above example:
        * `requirements` is the name
        * `[yaml]` specifies the format
        * The quoted string following the colon is the value
        assigned to the name.

        """

        assert len(c) == 3
        key = c[0]
        lang = c[1].lower().strip()
        body = c[2]
        parsed = False
        value = None

        if lang == "raw":
            value = body
            parsed = True

        if lang == "margo":
            value = body
            parsed = True

        if lang == "yaml":
            try:
                value = yaml.load(body, Loader=yaml.FullLoader)
                parsed = True
            except Exception as e:
                print(f"Error parsing YAML {body}: {e}")

        if lang == "json":
            try:
                value = json.loads(body)
                parsed = True
            except Exception as e:
                print(f"Error parsing JSON '{body}': {e}")
                pass

        return {
            "TYPE": "DECLARATION",
            "LANGUAGE": lang,
            "PARSED": parsed,
            "BODY": body,
            "NAME": key,
            "VALUE": value,
        }

    def statement(self, s: Tree):
        # # Statements should contain two items, an expression and an endblock
        # assert len(s) == 2
        # assert s[1] == ["ENDBLOCK"]
        assert type(s) == list

        return {"TYPE": "STATEMENT", "BODY": s}

    def expression(self, e: Tree):
        return {"TYPE": "EXPRESSION", "BODY": e}

    def ENDBLOCK(self, eb):
        return ["ENDBLOCK"]

    def string(self, s):
        return s[0]

    def builtin(self, s):
        assert len(s) == 1

        return {"TYPE": "BUILTIN", "BODY": s[0]}

    # def view_statement(self, v):
    #     (keys) = v
    #     return {"NAME": "view", "VIEW_LIST": v}

    def IGNORE_CELL(self, _):
        return {"NAME": "IGNORE_CELL"}

    def function(self, f):
        return ["FUNCTION", f]

    def function_name(self, fn):
        return str(fn[0])

    def argument_list(self, al):
        (s, *vals) = al

        return vals

    def mvf_assignment(self, kvp):
        (k, *vals) = kvp

        return self.evf_assignment([k, "json", json.dumps(vals)])  # key

    def KEY(self, k):
        return str(k)

    def false(self, _):
        return False

    def true(self, _):
        return True

    def null(self, _):
        return None

    def QSTRING(self, s):
        if len(str(s)) < 1:
            return ""
        return str(s)[1:-1]

    def value(self, v):
        return v[0]

    def number(self, n):
        (n,) = n
        try:
            return int(n.value)
        except BaseException:
            pass
        try:
            return float(n.value)
        except BaseException:
            pass

Classes

class MargoTransformer (visit_tokens=True)

Lark Transformer for Margo syntax

Expand source code
class MargoTransformer(Transformer):
    """Lark Transformer for Margo syntax"""

    def block(self, b):
        """
        
        *Transform a Margo block*

        A Margo block is a sequence of statements separated by endblocks, i.e.:

        ```margo
        {statement-1} ::
        {statement-2} ::
        {statement-3} ::
        ```
        """

        statements = list(filter(lambda s: s != ["ENDBLOCK"], b))

        # every other token should be an endblock, so there should always
        # be twice as many items in b as as there are statements
        assert len(statements) == len(b) / 2

        # a block is made up only of statements
        for s in statements:
            assert s["TYPE"] == "STATEMENT"
            assert type(s["BODY"]) == list
            assert len(s["BODY"]) < 2

        nonempty_statements = list(
            filter(lambda s: len(s["BODY"]) > 0, statements)
        )

        unwrapped_statements = list(
            map(lambda x: x["BODY"][0], nonempty_statements)
        )

        return {
            "SYNTAX": "MARGO",
            "TYPE": "BLOCK",
            "VERSION": "0",
            "BODY": unwrapped_statements,
        }

    def directive(self, d):
        """
        *Transform a Margo directive*
        
        A directive is a statement that makes no assignment, i.e.:

        ```margo
        ignore-cell ::
        ```

        In the above example, `ignore-cell` is the directive statement
        and `::` is the endblock.
        """

        assert len(d) == 1
        name = str(d[0])
        return {"TYPE": "DIRECTIVE", "NAME": name}

    def evf_assignment(self, c):
        """
        *Transform an external value format assignment*

        External value formats include:
        * JSON
        * YAML
        * raw (plain text)

        Example:

        ```margo
        requirements [yaml]:'
            - nbformat
            - requests
        ' ::
        ```

        In the above example:
        * `requirements` is the name
        * `[yaml]` specifies the format
        * The quoted string following the colon is the value
        assigned to the name.

        """

        assert len(c) == 3
        key = c[0]
        lang = c[1].lower().strip()
        body = c[2]
        parsed = False
        value = None

        if lang == "raw":
            value = body
            parsed = True

        if lang == "margo":
            value = body
            parsed = True

        if lang == "yaml":
            try:
                value = yaml.load(body, Loader=yaml.FullLoader)
                parsed = True
            except Exception as e:
                print(f"Error parsing YAML {body}: {e}")

        if lang == "json":
            try:
                value = json.loads(body)
                parsed = True
            except Exception as e:
                print(f"Error parsing JSON '{body}': {e}")
                pass

        return {
            "TYPE": "DECLARATION",
            "LANGUAGE": lang,
            "PARSED": parsed,
            "BODY": body,
            "NAME": key,
            "VALUE": value,
        }

    def statement(self, s: Tree):
        # # Statements should contain two items, an expression and an endblock
        # assert len(s) == 2
        # assert s[1] == ["ENDBLOCK"]
        assert type(s) == list

        return {"TYPE": "STATEMENT", "BODY": s}

    def expression(self, e: Tree):
        return {"TYPE": "EXPRESSION", "BODY": e}

    def ENDBLOCK(self, eb):
        return ["ENDBLOCK"]

    def string(self, s):
        return s[0]

    def builtin(self, s):
        assert len(s) == 1

        return {"TYPE": "BUILTIN", "BODY": s[0]}

    # def view_statement(self, v):
    #     (keys) = v
    #     return {"NAME": "view", "VIEW_LIST": v}

    def IGNORE_CELL(self, _):
        return {"NAME": "IGNORE_CELL"}

    def function(self, f):
        return ["FUNCTION", f]

    def function_name(self, fn):
        return str(fn[0])

    def argument_list(self, al):
        (s, *vals) = al

        return vals

    def mvf_assignment(self, kvp):
        (k, *vals) = kvp

        return self.evf_assignment([k, "json", json.dumps(vals)])  # key

    def KEY(self, k):
        return str(k)

    def false(self, _):
        return False

    def true(self, _):
        return True

    def null(self, _):
        return None

    def QSTRING(self, s):
        if len(str(s)) < 1:
            return ""
        return str(s)[1:-1]

    def value(self, v):
        return v[0]

    def number(self, n):
        (n,) = n
        try:
            return int(n.value)
        except BaseException:
            pass
        try:
            return float(n.value)
        except BaseException:
            pass

Ancestors

  • lark.visitors.Transformer
  • lark.visitors._Decoratable

Methods

def ENDBLOCK(self, eb)
Expand source code
def ENDBLOCK(self, eb):
    return ["ENDBLOCK"]
def IGNORE_CELL(self, _)
Expand source code
def IGNORE_CELL(self, _):
    return {"NAME": "IGNORE_CELL"}
def KEY(self, k)
Expand source code
def KEY(self, k):
    return str(k)
def QSTRING(self, s)
Expand source code
def QSTRING(self, s):
    if len(str(s)) < 1:
        return ""
    return str(s)[1:-1]
def argument_list(self, al)
Expand source code
def argument_list(self, al):
    (s, *vals) = al

    return vals
def block(self, b)

Transform a Margo block

A Margo block is a sequence of statements separated by endblocks, i.e.:

{statement-1} ::
{statement-2} ::
{statement-3} ::
Expand source code
def block(self, b):
    """
    
    *Transform a Margo block*

    A Margo block is a sequence of statements separated by endblocks, i.e.:

    ```margo
    {statement-1} ::
    {statement-2} ::
    {statement-3} ::
    ```
    """

    statements = list(filter(lambda s: s != ["ENDBLOCK"], b))

    # every other token should be an endblock, so there should always
    # be twice as many items in b as as there are statements
    assert len(statements) == len(b) / 2

    # a block is made up only of statements
    for s in statements:
        assert s["TYPE"] == "STATEMENT"
        assert type(s["BODY"]) == list
        assert len(s["BODY"]) < 2

    nonempty_statements = list(
        filter(lambda s: len(s["BODY"]) > 0, statements)
    )

    unwrapped_statements = list(
        map(lambda x: x["BODY"][0], nonempty_statements)
    )

    return {
        "SYNTAX": "MARGO",
        "TYPE": "BLOCK",
        "VERSION": "0",
        "BODY": unwrapped_statements,
    }
def builtin(self, s)
Expand source code
def builtin(self, s):
    assert len(s) == 1

    return {"TYPE": "BUILTIN", "BODY": s[0]}
def directive(self, d)

Transform a Margo directive

A directive is a statement that makes no assignment, i.e.:

ignore-cell ::

In the above example, ignore-cell is the directive statement and :: is the endblock.

Expand source code
def directive(self, d):
    """
    *Transform a Margo directive*
    
    A directive is a statement that makes no assignment, i.e.:

    ```margo
    ignore-cell ::
    ```

    In the above example, `ignore-cell` is the directive statement
    and `::` is the endblock.
    """

    assert len(d) == 1
    name = str(d[0])
    return {"TYPE": "DIRECTIVE", "NAME": name}
def evf_assignment(self, c)

Transform an external value format assignment

External value formats include: * JSON * YAML * raw (plain text)

Example:

requirements [yaml]:'
    - nbformat
    - requests
' ::

In the above example: * requirements is the name * [yaml] specifies the format * The quoted string following the colon is the value assigned to the name.

Expand source code
def evf_assignment(self, c):
    """
    *Transform an external value format assignment*

    External value formats include:
    * JSON
    * YAML
    * raw (plain text)

    Example:

    ```margo
    requirements [yaml]:'
        - nbformat
        - requests
    ' ::
    ```

    In the above example:
    * `requirements` is the name
    * `[yaml]` specifies the format
    * The quoted string following the colon is the value
    assigned to the name.

    """

    assert len(c) == 3
    key = c[0]
    lang = c[1].lower().strip()
    body = c[2]
    parsed = False
    value = None

    if lang == "raw":
        value = body
        parsed = True

    if lang == "margo":
        value = body
        parsed = True

    if lang == "yaml":
        try:
            value = yaml.load(body, Loader=yaml.FullLoader)
            parsed = True
        except Exception as e:
            print(f"Error parsing YAML {body}: {e}")

    if lang == "json":
        try:
            value = json.loads(body)
            parsed = True
        except Exception as e:
            print(f"Error parsing JSON '{body}': {e}")
            pass

    return {
        "TYPE": "DECLARATION",
        "LANGUAGE": lang,
        "PARSED": parsed,
        "BODY": body,
        "NAME": key,
        "VALUE": value,
    }
def expression(self, e: lark.tree.Tree)
Expand source code
def expression(self, e: Tree):
    return {"TYPE": "EXPRESSION", "BODY": e}
def false(self, _)
Expand source code
def false(self, _):
    return False
def function(self, f)
Expand source code
def function(self, f):
    return ["FUNCTION", f]
def function_name(self, fn)
Expand source code
def function_name(self, fn):
    return str(fn[0])
def mvf_assignment(self, kvp)
Expand source code
def mvf_assignment(self, kvp):
    (k, *vals) = kvp

    return self.evf_assignment([k, "json", json.dumps(vals)])  # key
def null(self, _)
Expand source code
def null(self, _):
    return None
def number(self, n)
Expand source code
def number(self, n):
    (n,) = n
    try:
        return int(n.value)
    except BaseException:
        pass
    try:
        return float(n.value)
    except BaseException:
        pass
def statement(self, s: lark.tree.Tree)
Expand source code
def statement(self, s: Tree):
    # # Statements should contain two items, an expression and an endblock
    # assert len(s) == 2
    # assert s[1] == ["ENDBLOCK"]
    assert type(s) == list

    return {"TYPE": "STATEMENT", "BODY": s}
def string(self, s)
Expand source code
def string(self, s):
    return s[0]
def true(self, _)
Expand source code
def true(self, _):
    return True
def value(self, v)
Expand source code
def value(self, v):
    return v[0]