240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
|
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
|
-
+
|
function getListOfTokens (sText, oInfo={}) {
// lexicographer
try {
sText = sText.replace(//g, "").normalize("NFC");
for (let sParagraph of text.getParagraph(sText)) {
if (sParagraph.trim() !== "") {
postMessage(createResponse("getListOfTokens", { sParagraph: sParagraph, lTokens: lexgraph_fr.getListOfTokensReduc(sParagraph, true) }, oInfo, false));
postMessage(createResponse("getListOfTokens", { sParagraph: sParagraph, lTokens: lexgraph_fr.getListOfTokens(sParagraph, true) }, oInfo, false));
}
}
postMessage(createResponse("getListOfTokens", null, oInfo, true));
}
catch (e) {
console.error(e);
postMessage(createResponse("getListOfTokens", createErrorResult(e, "no tokens"), oInfo, true, true));
|