Module margo_parser.tokenizer.tokenizer
Expand source code
from lark import Lark, Tree
from .margo_transformer import MargoTransformer
from ..exceptions import MargoParseException
import os
def grammar() -> Lark:
"""Get the Margo Lark grammar"""
grammar_file_path = os.path.join(os.path.split(__file__)[0], "margo.lark")
grammar_content = open(grammar_file_path).read()
return Lark(grammar_content, start="block", regex=True)
def get_tree(source: str) -> Tree:
"""Parse source and return a Lark Tree"""
return grammar().parse(source)
def transform(tree: Tree):
"""Transform a Lark Tree into a Margo dict"""
return MargoTransformer().transform(tree)
def tokenize(source: str):
"""Given a source file, return a Margo dict"""
try:
return transform(get_tree(source))
except Exception as e:
raise MargoParseException(e)
Functions
def get_tree(source: str) ‑> lark.tree.Tree
-
Parse source and return a Lark Tree
Expand source code
def get_tree(source: str) -> Tree: """Parse source and return a Lark Tree""" return grammar().parse(source)
def grammar() ‑> lark.lark.Lark
-
Get the Margo Lark grammar
Expand source code
def grammar() -> Lark: """Get the Margo Lark grammar""" grammar_file_path = os.path.join(os.path.split(__file__)[0], "margo.lark") grammar_content = open(grammar_file_path).read() return Lark(grammar_content, start="block", regex=True)
def tokenize(source: str)
-
Given a source file, return a Margo dict
Expand source code
def tokenize(source: str): """Given a source file, return a Margo dict""" try: return transform(get_tree(source)) except Exception as e: raise MargoParseException(e)
def transform(tree: lark.tree.Tree)
-
Transform a Lark Tree into a Margo dict
Expand source code
def transform(tree: Tree): """Transform a Lark Tree into a Margo dict""" return MargoTransformer().transform(tree)