Grammalecte  Changes On Branch 4fe9cf871ef35493

// JavaScript
/*jslint esversion: 6*/
/*global require,exports*/

"use strict";


if (typeof(require) !== 'undefined') {
    var helpers = require("resource://grammalecte/helpers.js");
}


var text = {
    getParagraph: function* (sText, sSepParagraph = "\n") {
        // generator: returns paragraphs of text
        let iStart = 0;
        let iEnd = 0;
        sText = sText.replace("\r\n", "\n").replace("\r", "\n");
        while ((iEnd = sText.indexOf(sSepParagraph, iStart)) !== -1) {
            yield sText.slice(iStart, iEnd);
            iStart = iEnd + 1;
        }
        yield sText.slice(iStart);
    },

    wrap: function* (sText, nWidth=80) {

        }
        yield sText;
    },

    getReadableError: function (oErr) {
        // Returns an error oErr as a readable error
        try {
            let sResult = "\n* " + oErr['nStart'] + ":" + oErr['nEnd']
                        + "  # " + oErr['sLineId'] + "  # " + oErr['sRuleId'] + ":\n";
            sResult += "  " + oErr["sMessage"];
            if (oErr["aSuggestions"].length > 0) {
                sResult += "\n  > Suggestions : " + oErr["aSuggestions"].join(" | ");
            }
            if (oErr["URL"] !== "") {
                sResult += "\n  > URL: " + oErr["URL"];

                '    "dMasForm": ' +  json.dumps(dMasForm, ensure_ascii=False) + "\n}"
        open(sp+"/modules-js/mfsp_data.json", "w", encoding="utf-8", newline="\n").write(sCode)


def makePhonetTable (sp, bJS=False):
    print("> Correspondances phonétiques ", end="")
    print("(Python et JavaScript)"  if bJS  else "(Python seulement)")

    try:
        oDict = ibdawg.IBDAWG("French.bdic")
    except:
        traceback.print_exc()
        return

    with open(sp+"/data/phonet_simil.txt", 'r', encoding='utf-8') as hSrc:

    with open(sp+"/data/locutions.txt", 'r', encoding='utf-8') as hSrc:
        dLocutions = {}
        for sLine in hSrc.readlines():
            if not sLine.startswith("#") and sLine.strip():
                lElem = sLine.strip().split()
                dCur = dLocutions
                for sWord in lElem:
                    if sWord not in dCur and not sWord.startswith(":"):
                        dCur[sWord] = {}
                    if sWord not in dCur and sWord.startswith(":"):
                        dCur[sWord] = ''
                    dCur = dCur[sWord]

    sCode = "# generated data (do not edit)\n\n" + \
            "dLocutions = " + str(dLocutions) + "\n"
    open(sp+"/modules/locutions_data.py", "w", encoding="utf-8", newline="\n").write(sCode)
    if bJS:
        open(sp+"/modules-js/locutions_data.json", "w", encoding="utf-8", newline="\n").write(json.dumps(dLocutions, ensure_ascii=False))

// Grammalecte - Lexicographe
// License: MPL 2
/*jslint esversion: 6*/
/*global require,exports*/

"use strict";

${string}
${map}


if (typeof (require) !== 'undefined') {
    var helpers = require("resource://grammalecte/helpers.js");
}

const _dTAGS = new Map([
    [':G', "[mot grammatical]"],
    [':N', " nom,"],
    [':A', " adjectif,"],
    [':M1', " prénom,"],
    [':M2', " patronyme,"],
    [':MP', " nom propre,"],
    [':W', " adverbe,"],

    [':Oi', " pronom indéfini,"],
    [':On', " pronom indéfini négatif,"],
    [':Ot', " pronom interrogatif,"],
    [':Or', " pronom relatif,"],
    [':Ow', " pronom adverbial,"],
    [':Os', " pronom personnel sujet,"],
    [':Oo', " pronom personnel objet,"],
    [':C', " conjonction,"],

    [':Cc', " conjonction de coordination,"],
    [':Cs', " conjonction de subordination,"],


    [':Ĺ', " locution,"],
    [':Ĉ', " locution conjonctivale (él.),"],
    [':Ĉs', " locution conjonctivale de subordination (él.),"],
    [':Ŵ', " locution adverbiale (él.),"],
    [':Ñ', " locution nominale (él.),"],
    [':Â', " locution adjectivale (él.),"],
    [':Ṽ', " locution verbale (él.),"],
    [':Ŕ', " locution prépositive (él.),"],
    [':Ô', " locution pronominales (él.),"],
    [':Ĵ', " locution interjective (él.),"],

    [':Zp', " préfixe,"],
    [':Zs', " suffixe,"],

    [':V1', " verbe (1ᵉʳ gr.),"],
    [':V2', " verbe (2ᵉ gr.),"],
    [':V3', " verbe (3ᵉ gr.),"],
    [':V0e', " verbe,"],
    [':V0a', " verbe,"],

    [':O1', " 1ʳᵉ pers.,"],
    [':O2', " 2ᵉ pers.,"],
    [':O3', " 3ᵉ pers.,"],

    [':e', " épicène"],
    [':m', " masculin"],
    [':f', " féminin"],
    [':s', " singulier"],
    [':p', " pluriel"],
    [':i', " invariable"],

    ['/C', " {classique}"],
    ['/M', ""],
    ['/R', " {réforme}"],
    ['/A', ""],
    ['/X', ""]
]);

const _dPFX = new Map([
    ['d', "(de), déterminant épicène invariable"],
    ['l', "(le/la), déterminant masculin/féminin singulier"],
    ['j', "(je), pronom personnel sujet, 1ʳᵉ pers., épicène singulier"],
    ['m', "(me), pronom personnel objet, 1ʳᵉ pers., épicène singulier"],
    ['t', "(te), pronom personnel objet, 2ᵉ pers., épicène singulier"],
    ['s', "(se), pronom personnel objet, 3ᵉ pers., épicène singulier/pluriel"],
    ['n', "(ne), adverbe de négation"],
    ['c', "(ce), pronom démonstratif, masculin singulier/pluriel"],
    ['ç', "(ça), pronom démonstratif, masculin singulier"],
    ['qu', "(que), conjonction de subordination"],
    ['lorsqu', "(lorsque), conjonction de subordination"],
    ['quoiqu', "(quoique), conjonction de subordination"],
    ['jusqu', "(jusque), préposition"]
]);

const _dAD = new Map([
    ['je', " pronom personnel sujet, 1ʳᵉ pers. sing."],
    ['tu', " pronom personnel sujet, 2ᵉ pers. sing."],
    ['il', " pronom personnel sujet, 3ᵉ pers. masc. sing."],
    ['on', " pronom personnel sujet, 3ᵉ pers. sing. ou plur."],
    ['elle', " pronom personnel sujet, 3ᵉ pers. fém. sing."],
    ['nous', " pronom personnel sujet/objet, 1ʳᵉ pers. plur."],
    ['vous', " pronom personnel sujet/objet, 2ᵉ pers. plur."],
    ['ils', " pronom personnel sujet, 3ᵉ pers. masc. plur."],
    ['elles', " pronom personnel sujet, 3ᵉ pers. masc. plur."],

    ["là", " particule démonstrative"],
    ["ci", " particule démonstrative"],

    ['le', " COD, masc. sing."],
    ['la', " COD, fém. sing."],
    ['les', " COD, plur."],

    ['moi', " COI (à moi), sing."],
    ['toi', " COI (à toi), sing."],
    ['lui', " COI (à lui ou à elle), sing."],
    ['nous2', " COI (à nous), plur."],
    ['vous2', " COI (à vous), plur."],
    ['leur', " COI (à eux ou à elles), plur."],

    ['y', " pronom adverbial"],
    ["m'y", " (me) pronom personnel objet + (y) pronom adverbial"],
    ["t'y", " (te) pronom personnel objet + (y) pronom adverbial"],
    ["s'y", " (se) pronom personnel objet + (y) pronom adverbial"],

    ['en', " pronom adverbial"],
    ["m'en", " (me) pronom personnel objet + (en) pronom adverbial"],
    ["t'en", " (te) pronom personnel objet + (en) pronom adverbial"],
    ["s'en", " (se) pronom personnel objet + (en) pronom adverbial"]
]);

const _dSeparator = new Map([
    ['.', "point"],
    ['·', "point médian"],
    ['…', "points de suspension"],
    [':', "deux-points"],
    [';', "point-virgule"],
    [',', "virgule"],
    ['?', "point d’interrogation"],

    ['<', "inférieur à"],
    ['>', "supérieur à"],
]);


class Lexicographe {

    constructor(oDict, oTokenizer, oLocution) {
        this.oDict = oDict;
        this.oTokenizer = oTokenizer;
        this.oLocution = JSON.parse(oLocution);

        this._zElidedPrefix = new RegExp("^([dljmtsncç]|quoiqu|lorsqu|jusqu|puisqu|qu)['’](.+)", "i");
        this._zCompoundWord = new RegExp("([a-zA-Zà-ö0-9À-Öø-ÿØ-ßĀ-ʯ]+)-((?:les?|la)-(?:moi|toi|lui|[nv]ous|leur)|t-(?:il|elle|on)|y|en|[mts][’'](?:y|en)|les?|l[aà]|[mt]oi|leur|lui|je|tu|ils?|elles?|on|[nv]ous)$", "i");
        this._zTag = new RegExp("[:;/][a-zA-Zà-ö0-9À-Öø-ÿØ-ßĀ-ʯ*Ṽ][^:;/]*", "g");

    }

    getInfoForToken (oToken) {
        // Token: .sType, .sValue, .nStart, .nEnd
        // return a object {sType, sValue, aLabel}
        let m = null;
        try {
            switch (oToken.sType) {
                case 'SEPARATOR':
                    return {
                        sType: oToken.sType,
                        sValue: oToken.sValue,
                        aLabel: [_dSeparator.gl_get(oToken.sValue, "caractère indéterminé")]
                    };
                    break;
                case 'NUM':
                    return {
                        sType: oToken.sType,
                        sValue: oToken.sValue,
                        aLabel: ["nombre"]
                    };
                    break;
                case 'LINK':
                    return {
                        sType: oToken.sType,
                        sValue: oToken.sValue.slice(0, 40) + "…",
                        aLabel: ["hyperlien"]
                    };
                    break;
                case 'ELPFX':
                    let sTemp = oToken.sValue.replace("’", "").replace("'", "").replace("`", "").toLowerCase();
                    return {
                        sType: oToken.sType,
                        sValue: oToken.sValue,
                        aLabel: [_dPFX.gl_get(sTemp, "préfixe élidé inconnu")]
                    };
                    break;
                case 'FOLDER':
                    return {
                        sType: oToken.sType,
                        sValue: oToken.sValue.slice(0, 40) + "…",
                        aLabel: ["dossier"]
                    };
                    break;
                case 'WORD':
                    if (oToken.sValue.gl_count("-") > 4) {
                        return {
                            sType: "COMPLEX",
                            sValue: oToken.sValue,
                            aLabel: ["élément complexe indéterminé"]
                        };
                    } else if (this.oDict.isValidToken(oToken.sValue)) {
                        let lMorph = this.oDict.getMorph(oToken.sValue);
                        let aElem = [];
                        for (let s of lMorph) {
                            if (s.includes(":")) aElem.push(this._formatTags(s));
                        }
                        return {
                            sType: oToken.sType,
                            sValue: oToken.sValue,
                            aLabel: aElem
                        };
                    } else if (m = this._zCompoundWord.exec(oToken.sValue)) {
                        // mots composés
                        let lMorph = this.oDict.getMorph(m[1]);
                        let aElem = [];
                        for (let s of lMorph) {
                            if (s.includes(":")) aElem.push(this._formatTags(s));
                        }
                        aElem.push("-" + m[2] + ": " + this._formatSuffix(m[2].toLowerCase()));
                        return {
                            sType: oToken.sType,
                            sValue: oToken.sValue,
                            aLabel: aElem
                        };
                    } else {
                        return {
                            sType: "UNKNOWN",
                            sValue: oToken.sValue,
                            aLabel: ["inconnu du dictionnaire"]
                        };
                    }
                    break;
            }

        } catch (e) {
            helpers.logerror(e);
        }
        return null;
    }

    _formatTags (sTags) {
        let sRes = "";

        if (!s.includes("-")) {
            return _dAD.get(s.replace("’", "'"));
        }
        if (s.endsWith("ous")) {
            s += '2';
        }
        let nPos = s.indexOf("-");
        return _dAD.get(s.slice(0, nPos)) + " +" + _dAD.get(s.slice(nPos + 1));
    }

    getListOfTokens (sText, bInfo = true) {
        let aElem = [];
        if (sText !== "") {
            let aRes = null;
            for (let oToken of this.oTokenizer.genTokens(sText)) {
                if (bInfo) {
                    aRes = this.getInfoForToken(oToken);
                    if (aRes) {
                        aElem.push(aRes);
                    }
                } else if (oToken.sType !== "SPACE") {
                    aElem.push(oToken);
                }
            }
        }
        return aElem;
    }

    _unifyStr (sWord){
        return sWord.replace('’', 'e').toLowerCase();
    }

    getListOfTokensReduc (sText, bInfo = true) {
        let aTokenList = this.getListOfTokens(sText.replace("'", "’").trim(), false);
        let iKey = 0;
        let aElem = [];
        let aRes = null;
        let isType = {'WORD':1,'ELPFX':1};
        do {
            let oToken = aTokenList[iKey]
            let aLocution = this.oLocution[this._unifyStr(oToken.sValue)];
            let bStop = false;
            let bOk = false;
            let iKeyTree = iKey + 1;
            let sTokenTmpKey = '';
            let aTokenTempList = [];
            aTokenTempList.push(oToken);
            while (!bStop && typeof aLocution !== "undefined") {
                if (iKeyTree > aTokenList.length){
                    break;
                }
                let sTokenNextType = '';
                let oTokenNext = aTokenList[iKeyTree];

                if ( typeof oTokenNext !== "undefined" ) {
                    aLocution = aLocution[this._unifyStr(oTokenNext.sValue)];
                    sTokenNextType = oTokenNext.sType;
                } else {
                    aLocution = "undefined";
                }

                if ( sTokenNextType in isType && typeof aLocution !== "undefined") {
                    sTokenTmpKey = Object.keys(aLocution)[0];
                    iKeyTree++;
                    aTokenTempList.push(oTokenNext);
                } else if ( !(sTokenNextType in isType) || typeof aLocution == "undefined") {
                    bStop = true;
                    if ( sTokenTmpKey.substring(0, 1) == ':' ) {
                        bOk = true;
                    }
                }
            };

            if ( bOk ){
                let sWord = '';
                for (let oTokenWord of aTokenTempList) {
                    sWord += oTokenWord.sValue+' ';
                }
                iKey = iKey + aTokenTempList.length-1;
                let oTokenLocution = {
                    'nEnd':aTokenTempList[aTokenTempList.length-1].nEnd,
                    'niKeyTree':aTokenTempList[0].niKeyTree,
                    'sType':"LOC",
                    'sValue':sWord.replace('’ ','’').trim()
                };
                if (bInfo) {
                    let aFormatedTag = [];
                    for (let sTagMulti of sTokenTmpKey.split('|') ){
                        aFormatedTag.push( this._formatTags(sTagMulti).replace(/(\(él.\))/g,'').trim() );
                    }
                    aElem.push({
                        sType: oTokenLocution.sType,
                        sValue: oTokenLocution.sValue,
                        aLabel: aFormatedTag
                    });
                } else {
                    aElem.push(oTokenLocution);
                }
            } else {
                if (bInfo) {
                    aRes = this.getInfoForToken(oToken);
                    if (aRes) {
                        aElem.push(aRes);
                    }
                } else {
                    aElem.push(oToken);
                }
            }
            iKey++;
        } while (iKey < aTokenList.length);
        return aElem;
    }
}


if (typeof (exports) !== 'undefined') {
    exports.Lexicographe = Lexicographe;
}

}
.grammalecte_lxg_morph_list {
    padding: 2px 0 10px 20px;
}
.grammalecte_lxg_morph_elem {
    color: hsl(0, 0%, 0%);
}
.grammalecte_lxg_token_LOC {
    background-color: hsla(150, 50%, 30%, 1);
}
.grammalecte_lxg_token_WORD {
    background-color: hsla(150, 50%, 50%, 1);
}
.grammalecte_lxg_token_ELPFX {
    background-color: hsla(150, 30%, 50%, 1);
}
.grammalecte_lxg_token_UNKNOWN {

let bInitDone = false;

let oDict = null;
let oTokenizer = null;
let oLxg = null;
let oTest = null;
let oLocution = null;


/*
    Technical note:
    This worker don’t work as a PromiseWorker (which returns a promise),  so when we send request
    to this worker, we can’t wait the return of the answer just after the request made.
    The answer is received by the background in another function (onmessage).

            conj.init(helpers.loadFile(sExtensionPath + "/grammalecte/fr/conj_data.json"));
            phonet.init(helpers.loadFile(sExtensionPath + "/grammalecte/fr/phonet_data.json"));
            mfsp.init(helpers.loadFile(sExtensionPath + "/grammalecte/fr/mfsp_data.json"));
            //console.log("[Worker] Modules have been initialized…");
            gc_engine.load(sContext, sExtensionPath+"grammalecte/_dictionaries");
            oDict = gc_engine.getDictionary();
            oTest = new TestGrammarChecking(gc_engine, sExtensionPath+"/grammalecte/fr/tests_data.json");
            oTokenizer = new Tokenizer("fr");

            oLocution =  helpers.loadFile(sExtensionPath + "/grammalecte/fr/locutions_data.json");

            oLxg = new Lexicographe(oDict, oTokenizer, oLocution);
            if (dOptions !== null) {
                gc_engine.setOptions(dOptions);
            }

            //tests();
            bInitDone = true;
        } else {
            console.log("[Worker] Already initialized…")
        }
        // we always retrieve options from the gc_engine, for setOptions filters obsolete options
        postMessage(createResponse("init", gc_engine.getOptions(), dInfo, true));

// Lexicographer

function getListOfTokens (sText, dInfo={}) {
    try {
        for (let sParagraph of text.getParagraph(sText)) {
            if (sParagraph.trim() !== "") {








                postMessage(createResponse("getListOfTokens", oLxg.getListOfTokensReduc(sParagraph, true), dInfo, false));
            }
        }
        postMessage(createResponse("getListOfTokens", null, dInfo, true));
    }
    catch (e) {
        helpers.logerror(e);
        postMessage(createResponse("getListOfTokens", createErrorResult(e, "no tokens"), dInfo, true, true));
    }
}