792
793
794
795
796
797
798
799
800
801
802
803
804
805
|
self.lTokens[nTokenRewriteStart]["bToRemove"] = True
else:
for i in range(nTokenRewriteStart, nTokenRewriteEnd+1):
self.lTokens[i]["bToRemove"] = True
elif sWhat == "␣":
# merge tokens
self.lTokens[nTokenRewriteStart]["nMergeUntil"] = nTokenRewriteEnd
elif sWhat == "_":
# neutralized token
if nTokenRewriteEnd - nTokenRewriteStart == 0:
self.lTokens[nTokenRewriteStart]["sNewValue"] = "_"
else:
for i in range(nTokenRewriteStart, nTokenRewriteEnd+1):
self.lTokens[i]["sNewValue"] = "_"
|
>
>
>
>
|
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
|
self.lTokens[nTokenRewriteStart]["bToRemove"] = True
else:
for i in range(nTokenRewriteStart, nTokenRewriteEnd+1):
self.lTokens[i]["bToRemove"] = True
elif sWhat == "␣":
# merge tokens
self.lTokens[nTokenRewriteStart]["nMergeUntil"] = nTokenRewriteEnd
elif sWhat.startswith("␣"):
sWhat = self._expand(sWhat, nTokenOffset, nLastToken)
self.lTokens[nTokenRewriteStart]["nMergeUntil"] = nTokenRewriteEnd
self.lTokens[nTokenRewriteStart]["sMergedValue"] = sWhat[1:]
elif sWhat == "_":
# neutralized token
if nTokenRewriteEnd - nTokenRewriteStart == 0:
self.lTokens[nTokenRewriteStart]["sNewValue"] = "_"
else:
for i in range(nTokenRewriteStart, nTokenRewriteEnd+1):
self.lTokens[i]["sNewValue"] = "_"
|
844
845
846
847
848
849
850
851
852
853
854
855
856
857
|
# token to merge
dTokenMerger["sValue"] += " " * (dToken["nStart"] - dTokenMerger["nEnd"]) + dToken["sValue"]
dTokenMerger["nEnd"] = dToken["nEnd"]
if bDebug:
echo(" MERGED TOKEN: " + dTokenMerger["sValue"])
dToken["bMerged"] = True
bKeepToken = False
if "nMergeUntil" in dToken:
# first token to be merge with
if iToken > nMergeUntil: # this token is not to be merged with a previous token
dTokenMerger = dToken
if dToken["nMergeUntil"] > nMergeUntil:
nMergeUntil = dToken["nMergeUntil"]
del dToken["nMergeUntil"]
|
>
>
>
>
|
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
|
# token to merge
dTokenMerger["sValue"] += " " * (dToken["nStart"] - dTokenMerger["nEnd"]) + dToken["sValue"]
dTokenMerger["nEnd"] = dToken["nEnd"]
if bDebug:
echo(" MERGED TOKEN: " + dTokenMerger["sValue"])
dToken["bMerged"] = True
bKeepToken = False
if iToken == nMergeUntil and "sMergedValue" in dTokenMerger:
dTokenMerger["sValue"] = dTokenMerger["sMergedValue"]
sSpaceFiller = " " * (dToken["nEnd"] - dTokenMerger["nStart"] - len(dTokenMerger["sMergedValue"]))
self.sSentence = self.sSentence[:dTokenMerger["nStart"]] + dTokenMerger["sMergedValue"] + sSpaceFiller + self.sSentence[dToken["nEnd"]:]
if "nMergeUntil" in dToken:
# first token to be merge with
if iToken > nMergeUntil: # this token is not to be merged with a previous token
dTokenMerger = dToken
if dToken["nMergeUntil"] > nMergeUntil:
nMergeUntil = dToken["nMergeUntil"]
del dToken["nMergeUntil"]
|