Grammalecte  Diff

Differences From Artifact [eb2b3da492]:

To Artifact [b4ed6d4156]:


74
75
76
77
78
79
80



81
82
83
84
85
86
87
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90







+
+
+









def genTokenLines (sTokenLine, dDef):
    "tokenize a string and return a list of lines of tokens"
    lToken = sTokenLine.split()
    lTokenLines = None
    for sToken in lToken:
        # replace merger characters by spaces
        if "␣" in sToken:
            sToken = sToken.replace("␣", " ")
        # optional token?
        bNullPossible = sToken.startswith("?") and sToken.endswith("¿")
        if bNullPossible:
            sToken = sToken[1:-1]
        # token with definition?
        if sToken.startswith("({") and sToken.endswith("})") and sToken[1:-1] in dDef:
            sToken = "(" + dDef[sToken[1:-1]] + ")"