909
910
911
912
913
914
915
916
917
918
919
920
921
922
|
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
|
+
+
+
+
+
|
}
}
}
else if (sWhat === "␣") {
// merge tokens
this.lTokens[nTokenRewriteStart]["nMergeUntil"] = nTokenRewriteEnd;
}
else if (sWhat.startsWith("␣")) {
sWhat = this._expand(sWhat, nTokenOffset, nLastToken);
this.lTokens[nTokenRewriteStart]["nMergeUntil"] = nTokenRewriteEnd;
this.lTokens[nTokenRewriteStart]["sMergedValue"] = sWhat.slice(1);
}
else if (sWhat === "_") {
// neutralized token
if (nTokenRewriteEnd - nTokenRewriteStart == 0) {
this.lTokens[nTokenRewriteStart]["sNewValue"] = "_";
} else {
for (let i = nTokenRewriteStart; i <= nTokenRewriteEnd; i++) {
this.lTokens[i]["sNewValue"] = "_";
|
979
980
981
982
983
984
985
986
987
988
989
990
991
992
|
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
|
+
+
+
+
+
|
oMergingToken["sValue"] += " ".repeat(oToken["nStart"] - oMergingToken["nEnd"]) + oToken["sValue"];
oMergingToken["nEnd"] = oToken["nEnd"];
if (bDebug) {
console.log(" MERGED TOKEN: " + oMergingToken["sValue"]);
}
oToken["bMerged"] = true;
bKeepToken = false;
if (iToken == nMergeUntil && oMergingToken.hasOwnProperty("sMergedValue")) {
oMergingToken["sValue"] = oMergingToken["sMergedValue"];
let sSpaceFiller = " ".repeat(oToken["nEnd"] - oMergingToken["nStart"] - oMergingToken["sMergedValue"].length);
this.sSentence = this.sSentence.slice(0, oMergingToken["nStart"]) + oMergingToken["sMergedValue"] + sSpaceFiller + this.sSentence.slice(oToken["nEnd"]);
}
}
if (oToken.hasOwnProperty("nMergeUntil")) {
if (iToken > nMergeUntil) { // this token is not already merged with a previous token
oMergingToken = oToken;
}
if (oToken["nMergeUntil"] > nMergeUntil) {
nMergeUntil = oToken["nMergeUntil"];
|