Index: gc_core/js/lang_core/gc_engine.js ================================================================== --- gc_core/js/lang_core/gc_engine.js +++ gc_core/js/lang_core/gc_engine.js @@ -220,11 +220,11 @@ this.dTokenPos.set(dToken["nStart"], dToken); } } if (bFullInfo) { this.lTokens0 = Array.from(this.lTokens); - // the list of tokens is duplicated, to keep all tokens from being deleted when analysis + // the list of tokens is duplicated, to keep tokens from being deleted when analysis } this.parseText(this.sSentence, this.sSentence0, false, iStart, sCountry, dOpt, bShowRuleId, bDebug, bContext); if (bFullInfo) { for (let oToken of this.lTokens0) { if (oToken["sType"] == "WORD") { @@ -973,25 +973,23 @@ // rewrite the sentence, modify tokens, purge the token list if (bDebug) { console.log("REWRITE"); } let lNewToken = []; - let lNewTokens0 = []; let nMergeUntil = 0; let oMergingToken = null; for (let [iToken, oToken] of this.lTokens.entries()) { let bKeepToken = true; - let bKeepToken0 = true; if (oToken["sType"] != "INFO") { if (nMergeUntil && iToken <= nMergeUntil) { oMergingToken["sValue"] += " ".repeat(oToken["nStart"] - oMergingToken["nEnd"]) + oToken["sValue"]; oMergingToken["nEnd"] = oToken["nEnd"]; if (bDebug) { console.log(" MERGED TOKEN: " + oMergingToken["sValue"]); } + oToken["bMerged"] = true; bKeepToken = false; - bKeepToken0 = false; } if (oToken.hasOwnProperty("nMergeUntil")) { if (iToken > nMergeUntil) { // this token is not already merged with a previous token oMergingToken = oToken; } @@ -1031,23 +1029,16 @@ catch (e) { console.log(this.asString()); console.log(oToken); } } - if (this.lTokens0 !== null && bKeepToken0) { - lNewTokens0.push(oToken); - } } if (bDebug) { console.log(" TEXT REWRITED: " + this.sSentence); } this.lTokens.length = 0; this.lTokens = lNewToken; - if (this.lTokens0 !== null) { - this.lTokens0.length = 0; - this.lTokens0 = lNewTokens0; - } } }; if (typeof(exports) !== 'undefined') { Index: gc_core/py/lang_core/gc_engine.py ================================================================== --- gc_core/py/lang_core/gc_engine.py +++ gc_core/py/lang_core/gc_engine.py @@ -282,11 +282,11 @@ self.sSentence0 = self.sText0[iStart:iEnd] self.nOffsetWithinParagraph = iStart self.lTokens = list(_oTokenizer.genTokens(self.sSentence, True)) self.dTokenPos = { dToken["nStart"]: dToken for dToken in self.lTokens if dToken["sType"] != "INFO" } if bFullInfo: - self.lTokens0 = list(self.lTokens) # the list of tokens is duplicated, to keep all tokens from being deleted when analysis + self.lTokens0 = list(self.lTokens) # the list of tokens is duplicated, to keep tokens from being deleted when analysis self.parseText(self.sSentence, self.sSentence0, False, iStart, sCountry, dOpt, bShowRuleId, bDebug, bContext) if bFullInfo: for dToken in self.lTokens0: if dToken["sType"] == "WORD": dToken["bValidToken"] = _oSpellChecker.isValidToken(dToken["sValue"]) @@ -840,20 +840,19 @@ lNewTokens0 = [] nMergeUntil = 0 dTokenMerger = {} for iToken, dToken in enumerate(self.lTokens): bKeepToken = True - bKeepToken0 = True if dToken["sType"] != "INFO": if nMergeUntil and iToken <= nMergeUntil: # token to merge dTokenMerger["sValue"] += " " * (dToken["nStart"] - dTokenMerger["nEnd"]) + dToken["sValue"] dTokenMerger["nEnd"] = dToken["nEnd"] if bDebug: echo(" MERGED TOKEN: " + dTokenMerger["sValue"]) + dToken["bMerged"] = True bKeepToken = False - bKeepToken0 = False if "nMergeUntil" in dToken: # first token to be merge with if iToken > nMergeUntil: # this token is not to be merged with a previous token dTokenMerger = dToken if dToken["nMergeUntil"] > nMergeUntil: @@ -882,14 +881,9 @@ try: del self.dTokenPos[dToken["nStart"]] except KeyError: echo(self) echo(dToken) - if self.lTokens0 is not None and bKeepToken0: - lNewTokens0.append(dToken) if bDebug: echo(" TEXT REWRITED: " + self.sSentence) self.lTokens.clear() self.lTokens = lNewTokens - if self.lTokens0 is not None: - self.lTokens0.clear() - self.lTokens0 = lNewTokens0 Index: gc_lang/fr/webext/content_scripts/panel_gc.js ================================================================== --- gc_lang/fr/webext/content_scripts/panel_gc.js +++ gc_lang/fr/webext/content_scripts/panel_gc.js @@ -580,11 +580,11 @@ let xSentenceBlock = oGrammalecte.createNode("div", {className: "grammalecte_lxg_paragraph_sentence_block"}); xSentenceBlock.appendChild(oGrammalecte.createNode("div", {className: "grammalecte_lxg_list_num", textContent: this.nLxgCount})); xSentenceBlock.appendChild(oGrammalecte.createNode("p", {className: "grammalecte_lxg_paragraph_sentence", textContent: oSentence.sSentence})); let xTokenList = oGrammalecte.createNode("div", {className: "grammalecte_lxg_list_of_tokens"}); for (let oToken of oSentence.lTokens) { - if (oToken["sType"] != "INFO") { + if (oToken["sType"] != "INFO" && !oToken.hasOwnProperty("bMerged")) { xTokenList.appendChild(this._createTokenBlock2(oToken)); } } xSentenceBlock.appendChild(xTokenList); this.xLxgResultZone.appendChild(xSentenceBlock); Index: grammalecte-cli.py ================================================================== --- grammalecte-cli.py +++ grammalecte-cli.py @@ -344,11 +344,11 @@ lParagraphErrors, lSentences = oGrammarChecker.gce.parse(sParagraph, bDebug=xArgs.debug, bFullInfo=True) #echo(txt.getReadableErrors(lParagraphErrors, xArgs.width)) for dSentence in lSentences: echo("{nStart}:{nEnd} <{sSentence}>".format(**dSentence)) for dToken in dSentence["lTokens"]: - if dToken["sType"] == "INFO": + if dToken["sType"] == "INFO" or "bMerged" in dToken: continue echo(" {0[nStart]:>3}:{0[nEnd]:<3} {1} {0[sType]:<14} {2} {0[sValue]:<16} {3}".format(dToken, \ "×" if dToken.get("bToRemove", False) else " ", "!" if dToken["sType"] == "WORD" and not dToken.get("bValidToken", False) else " ", " ".join(dToken.get("aTags", "")) ) )