935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
|
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
|
+
+
+
-
-
-
+
+
+
|
else:
# several tokens
lTokenValue = sWhat.split("|")
if len(lTokenValue) != (nTokenRewriteEnd - nTokenRewriteStart + 1):
print("Error. Text processor: number of replacements != number of tokens.")
return
for i, sValue in zip(range(nTokenRewriteStart, nTokenRewriteEnd+1), lTokenValue):
if not sValue or sValue == "*":
self.lToken[i]["bToRemove"] = True
else:
if bUppercase:
sValue = sValue[0:1].upper() + sValue[1:]
self.lToken[i]["sNewValue"] = sValue
if bUppercase:
sValue = sValue[0:1].upper() + sValue[1:]
self.lToken[i]["sNewValue"] = sValue
def rewrite (self, bDebug=False):
"rewrite the sentence, modify tokens, purge the token list"
lNewToken = []
nMergeUntil = 0
dTokenMerger = None
for dToken in self.lToken:
|