74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
def genTokenLines (sTokenLine, dDef):
"tokenize a string and return a list of lines of tokens"
lToken = sTokenLine.split()
lTokenLines = None
for sToken in lToken:
# optional token?
bNullPossible = sToken.startswith("?") and sToken.endswith("¿")
if bNullPossible:
sToken = sToken[1:-1]
# token with definition?
if sToken.startswith("({") and sToken.endswith("})") and sToken[1:-1] in dDef:
sToken = "(" + dDef[sToken[1:-1]] + ")"
|
>
>
>
|
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
def genTokenLines (sTokenLine, dDef):
"tokenize a string and return a list of lines of tokens"
lToken = sTokenLine.split()
lTokenLines = None
for sToken in lToken:
# replace merger characters by spaces
if "␣" in sToken:
sToken = sToken.replace("␣", " ")
# optional token?
bNullPossible = sToken.startswith("?") and sToken.endswith("¿")
if bNullPossible:
sToken = sToken[1:-1]
# token with definition?
if sToken.startswith("({") and sToken.endswith("})") and sToken[1:-1] in dDef:
sToken = "(" + dDef[sToken[1:-1]] + ")"
|