807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
|
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
|
-
+
+
-
-
-
+
+
+
+
+
+
+
+
-
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
|
# disambiguation
if bDebug:
print(" DISAMBIGUATOR:\n ", dRule[sRuleId])
globals()[sWhat](self.lToken, nTokenOffset)
elif cActionType == ">":
# we do nothing, this test is just a condition to apply all following actions
if bDebug:
print(" COND_OK")
print(" COND_OK: ", sRuleId)
pass
elif cActionType == "/":
# Tag
if bDebug:
print(" SEMANTIC_TAG:\n ", dRule[sRuleId])
nTokenStart = nTokenOffset + eAct[0]
nTokenEnd = nTokenOffset + (eAct[1] if eAct[1] else eAct[0])
print(" TAG:\n ", dRule[sRuleId])
nTokenStart = nTokenOffset + eAct[0] if eAct[0] > 0 else nLastToken + eAct[0]
nTokenEnd = nTokenOffset + eAct[1] if eAct[1] > 0 else nLastToken + eAct[1]
for i in range(nTokenStart, nTokenEnd+1):
if "tags" in self.lToken[i]:
self.lToken[i]["tags"].update(sWhat.split("|"))
else:
self.lToken[i]["tags"] = set(sWhat.split("|"))
if sWhat not in self.dTags:
self.dTags[sWhat] = [nTokenStart, nTokenStart]
else:
self.dTags[sWhat][0] = min(nTokenStart, self.dTags[sWhat][0])
self.dTags[sWhat][1] = max(nTokenEnd, self.dTags[sWhat][1])
elif cActionType == "%":
# sentence tags
# immunity
if bDebug:
print(" SENTENCE_TAG:\n ", dRule[sRuleId])
nTokenTag = nTokenOffset + eAct[0]
if sWhat not in self.dTags:
self.dTags[sWhat] = (nTokenTag, nTokenTag)
elif nTokenTag > self.dTags[sWhat][1]:
self.dTags[sWhat] = (self.dTags[sWhat][0], nTokenTag)
|
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
|
925
926
927
928
929
930
931
932
933
934
935
936
937
938
|
-
-
-
-
-
-
-
|
self.lToken[nTokenRewriteStart]["bToRemove"] = True
else:
for i in range(nTokenRewriteStart, nTokenRewriteEnd+1):
self.lToken[i]["bToRemove"] = True
elif sWhat == "␣":
# merge tokens
self.lToken[nTokenRewriteStart]["nMergeUntil"] = nTokenRewriteEnd
elif sWhat == "!":
# immunity
if nTokenRewriteEnd - nTokenRewriteStart == 0:
self.lToken[nTokenRewriteStart]["bImmune"] = True
else:
for i in range(nTokenRewriteStart, nTokenRewriteEnd+1):
self.lToken[i]["bImmune"] = True
|
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
|
968
969
970
971
972
973
974
975
976
977
978
979
980
981
|
-
-
-
-
-
-
|
print("REWRITE")
lNewToken = []
nMergeUntil = 0
dTokenMerger = None
for iToken, dToken in enumerate(self.lToken):
bKeepToken = True
if dToken["sType"] != "INFO":
if "bImmune" in dToken:
nErrorStart = self.nOffsetWithinParagraph + dToken["nStart"]
if nErrorStart in self.dError:
if bDebug:
print("immunity -> error removed:", self.dError[nErrorStart])
del self.dError[nErrorStart]
|