247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
|
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
|
+
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
|
// Lexicographer
function getListOfTokens (sText, dInfo={}) {
try {
for (let sParagraph of text.getParagraph(sText)) {
if (sParagraph.trim() !== "") {
let aElem = [];
let aRes = null;
for (let oToken of oTokenizer.genTokens(sParagraph)) {
aRes = oLxg.getInfoForToken(oToken);
if (aRes) {
aElem.push(aRes);
}
}
postMessage(createResponse("getListOfTokens", aElem, dInfo, false));
let aElem = [];
let aRes = null;
for (let oToken of oTokenizer.genTokens(sParagraph)) {
aRes = oLxg.getInfoForToken(oToken);
if (aRes) {
aElem.push(aRes);
}
}
postMessage(createResponse("getListOfTokens", aElem, dInfo, false));
}
}
postMessage(createResponse("getListOfTokens", null, dInfo, true));
}
catch (e) {
helpers.logerror(e);
postMessage(createResponse("getListOfTokens", createErrorResult(e, "no tokens"), dInfo, true, true));
}
|