Module margo_parser.tokenizer

Margo syntax tokenizer

Expand source code
"""Margo syntax tokenizer"""

from .tokenizer import tokenize

__all__ = ['tokenize']

Sub-modules

margo_parser.tokenizer.margo_transformer

Lark Transformer for Margo syntax

margo_parser.tokenizer.tokenizer

Functions

def tokenize(source: str)

Given a source file, return a Margo dict

Expand source code
def tokenize(source: str):
    """Given a source file, return a Margo dict"""
    try:
        return transform(get_tree(source))
    except Exception as e:
        raise MargoParseException(e)