Grammalecte  Diff

Differences From Artifact [541689c69f]:

To Artifact [1dc7e255ce]:


90
91
92
93
94
95
96
97
98
99
        if (bStartEndToken) {
            yield { "i": iToken+1, "sType": "INFO", "sValue": "<end>", "nStart": iEnd, "nEnd": iEnd, "lMorph": ["<end>"] };
        }
    }
}


if (typeof exports !== 'undefined') {
    exports.Tokenizer = Tokenizer;
}







|


90
91
92
93
94
95
96
97
98
99
        if (bStartEndToken) {
            yield { "i": iToken+1, "sType": "INFO", "sValue": "<end>", "nStart": iEnd, "nEnd": iEnd, "lMorph": ["<end>"] };
        }
    }
}


if (typeof(exports) !== 'undefined') {
    exports.Tokenizer = Tokenizer;
}