771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
|
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
|
-
+
-
+
|
if bDebug:
print("ERROR:", sRuleId, dError[nErrorStart])
elif cActionType == "~":
# text processor
nEndToken = (nTokenOffset + eAct[1]) if eAct[1] else nLastToken
self._tagAndPrepareTokenForRewriting(sWhat, nTokenOffset + eAct[0], nEndToken, bDebug)
self._tagAndPrepareTokenForRewriting(sWhat, nTokenOffset + eAct[0], nEndToken, nTokenOffset, bDebug)
if bDebug:
print("RW:", sRuleId)
bChange = True
elif cActionType == "=":
# disambiguation
globals()[sWhat](self.lToken, nTokenOffset)
if bDebug:
print("DA:", sRuleId)
elif cActionType == ">":
# we do nothing, this test is just a condition to apply all following actions
if bDebug:
print(">>>", sRuleId)
pass
elif cActionType == "/":
# tags
# sentence tags
nTokenTag = nTokenOffset + eAct[0]
if sWhat not in self.dTags:
self.dTags[sWhat] = (nTokenTag, nTokenTag)
elif nTokenTag > self.dTags[sWhat][1]:
self.dTags[sWhat] = (self.dTags[sWhat][0], nTokenTag)
|
890
891
892
893
894
895
896
897
898
899
900
|
890
891
892
893
894
895
896
897
898
899
900
|
-
+
|
for m in re.finditer(r"\\([0-9]+)", sMsg):
sMsg = sMsg.replace(m.group(0), self.lToken[int(m.group(1))+nTokenOffset]["sValue"])
#print(">", sMsg)
return sMsg
def _tagAndPrepareTokenForRewriting (self, sWhat, nTokenRewriteStart, nTokenRewriteEnd, bUppercase=True, bDebug=False):
def _tagAndPrepareTokenForRewriting (self, sWhat, nTokenRewriteStart, nTokenRewriteEnd, nTokenOffset, bUppercase=True, bDebug=False):
"text processor: rewrite tokens between <nTokenRewriteStart> and <nTokenRewriteEnd> position"
if bDebug:
print("REWRITING:", nTokenRewriteStart, nTokenRewriteEnd)
if sWhat == "*":
# purge text
|
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
|
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
|
-
+
+
+
|
else:
for i in range(nTokenRewriteStart, nTokenRewriteEnd+1):
self.lToken[i]["sNewValue"] = "_"
else:
if sWhat.startswith("="):
sWhat = globals()[sWhat[1:]](self.lToken)
sWhat = globals()[sWhat[1:]](self.lToken, nTokenOffset)
bUppercase = bUppercase and self.lToken[nTokenRewriteStart]["sValue"][0:1].isupper()
if nTokenRewriteEnd - nTokenRewriteStart == 0:
# one token
sWhat = sWhat + " " * (len(self.lToken[nTokenRewriteStart]["sValue"])-len(sWhat))
if bUppercase:
sWhat = sWhat[0:1].upper() + sWhat[1:]
self.lToken[nTokenRewriteStart]["sNewValue"] = sWhat
else:
# several tokens
lTokenValue = sWhat.split("|")
if len(lTokenValue) != (nTokenRewriteEnd - nTokenRewriteStart + 1):
print("Error. Text processor: number of replacements != number of tokens.")
return
for i, sValue in zip(range(nTokenRewriteStart, nTokenRewriteEnd+1), lTokenValue):
|