Grammalecte  Diff

function getListOfTokens (sText, oInfo={}) {
    // lexicographer
    try {
        sText = sText.replace(/­/g, "").normalize("NFC");
        for (let sParagraph of text.getParagraph(sText)) {
            if (sParagraph.trim() !== "") {
                let lTokens = [ ...oTokenizer.genTokens(sParagraph) ];
                for (let oToken of lTokens) {
                    oSpellChecker.setLabelsOnToken(oToken);
                }
                postMessage(createResponse("getListOfTokens", { sParagraph: sParagraph, lTokens: lTokens }, oInfo, false));
            }
        }
        postMessage(createResponse("getListOfTokens", null, oInfo, true));
    }
    catch (e) {
        console.error(e);
        postMessage(createResponse("getListOfTokens", createErrorResult(e, "no tokens"), oInfo, true, true));