Overview
| Comment: | [core] gc_engine: remove text merged in another token + code clarification |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | core | rg |
| Files: | files | file ages | folders |
| SHA3-256: |
acaa2bad837e3d8d3ed89a44cb5db072 |
| User & Date: | olr on 2018-06-27 18:51:46 |
| Other Links: | branch diff | manifest | tags |
Context
|
2018-06-27
| ||
| 18:52 | [fr] conversion: regex rules -> graph rules check-in: 84f047b454 user: olr tags: fr, rg | |
| 18:51 | [core] gc_engine: remove text merged in another token + code clarification check-in: acaa2bad83 user: olr tags: core, rg | |
| 18:30 | [core] gc engine: better debugging info check-in: 2c534c0712 user: olr tags: core, rg | |
Changes
Modified gc_core/py/lang_core/gc_engine.py from [62d0f14ee9] to [3d9d459362].
| ︙ | ︙ | |||
949 950 951 952 953 954 955 |
"rewrite the sentence, modify tokens, purge the token list"
if bDebug:
print("REWRITE")
lNewToken = []
nMergeUntil = 0
dTokenMerger = None
for dToken in self.lToken:
| | | > | < < > | < < | 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 |
"rewrite the sentence, modify tokens, purge the token list"
if bDebug:
print("REWRITE")
lNewToken = []
nMergeUntil = 0
dTokenMerger = None
for dToken in self.lToken:
bRemoveToken = False
if "bImmune" in dToken:
nErrorStart = self.nOffsetWithinParagraph + dToken["nStart"]
if nErrorStart in self.dError:
if bDebug:
print("immunity -> error removed:", self.dError[nErrorStart])
del self.dError[nErrorStart]
if nMergeUntil and dToken["i"] <= nMergeUntil:
dTokenMerger["sValue"] += " " * (dToken["nStart"] - dTokenMerger["nEnd"]) + dToken["sValue"]
dTokenMerger["nEnd"] = dToken["nEnd"]
if bDebug:
print(" MERGED TOKEN:", dTokenMerger["sValue"])
bRemoveToken = True
if "nMergeUntil" in dToken:
if dToken["i"] > nMergeUntil: # this token is not already merged with a previous token
dTokenMerger = dToken
if dToken["nMergeUntil"] > nMergeUntil:
nMergeUntil = dToken["nMergeUntil"]
del dToken["nMergeUntil"]
#
if bRemoveToken or "bToRemove" in dToken:
if bDebug:
print(" REMOVED:", dToken["sValue"])
self.sSentence = self.sSentence[:dToken["nStart"]] + " " * (dToken["nEnd"] - dToken["nStart"]) + self.sSentence[dToken["nEnd"]:]
else:
lNewToken.append(dToken)
if "sNewValue" in dToken:
# rewrite token and sentence
if bDebug:
print(dToken["sValue"], "->", dToken["sNewValue"])
dToken["sRealValue"] = dToken["sValue"]
dToken["sValue"] = dToken["sNewValue"]
|
| ︙ | ︙ |