Grammalecte  Changes On Branch c70980cc3cc5fab0

"""
Grammalecte: compile rules
"""

import re
import os
import traceback
import json
import colorsys
import time

import compile_rules_js_convert as jsconv
import compile_rules_graph as crg


dDEF = {}
lFUNCTIONS = []

    dColor = {}
    dOptLabel = {}
    dOptPriority = {}
    for sLine in lOptionLines:
        sLine = sLine.strip()
        if sLine.startswith("OPTGROUP/"):
            m = re.match("OPTGROUP/([a-z0-9]+):(.+)$", sLine)
            lStructOpt.append( [m.group(1), list(map(str.split, m.group(2).split(",")))] )
        elif sLine.startswith("OPTSOFTWARE:"):
            lOpt = [ [s, {}]  for s in sLine[12:].strip().split() ]  # don’t use tuples (s, {}), because unknown to JS
        elif sLine.startswith("OPT/"):
            m = re.match("OPT/([a-z0-9]+):(.+)$", sLine)
            for i, sOpt in enumerate(m.group(2).split()):
                lOpt[i][1][m.group(1)] = eval(sOpt)
        elif sLine.startswith("OPTCOLORTHEME:"):

def printBookmark (nLevel, sComment, nLine):
    "print bookmark within the rules file"
    print("  {:>6}:  {}".format(nLine, "  " * nLevel + sComment))


def make (spLang, sLang, bUseCache=False):
    "compile rules, returns a dictionary of values"
    # for clarity purpose, don’t create any file here

    if bUseCache and os.path.isfile("_build/data_cache.json"):
        print("> don’t rebuild rules, use cache...")
        sJSON = open("_build/data_cache.json", "r", encoding="utf-8").read()
        dCacheVars = json.loads(sJSON)
        print("  build made at: " + time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(dCacheVars.get("fBuildTime", 0))))
        return dCacheVars

    fBuildTime = time.time()

    print("> read rules file...")
    try:
        lRules = open(spLang + "/rules.grx", 'r', encoding="utf-8").readlines()
    except:
        print("Error. Rules file in project [" + sLang + "] not found.")
        exit()

        sJSCallables += "        return " + jsconv.py2js(sReturn) + ";\n"
        sJSCallables += "    },\n"

    displayStats(lParagraphRules, lSentenceRules)

    print("Unnamed rules: " + str(nRULEWITHOUTNAME))

    dVars = {
        "fBuildTime": fBuildTime,
        "callables": sPyCallables,
        "callablesJS": sJSCallables,
        "gctests": sGCTests,
        "gctestsJS": sGCTestsJS,
        "paragraph_rules": mergeRulesByOption(lParagraphRules),
        "sentence_rules": mergeRulesByOption(lSentenceRules),
        "paragraph_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lParagraphRulesJS)),
        "sentence_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lSentenceRulesJS))
    }
    dVars.update(dOptions)

    # compile graph rules
    dVars2 = crg.make(lGraphRule, dDEF, sLang, dOptPriority)
    dVars.update(dVars2)

    with open("_build/data_cache.json", "w", encoding="utf-8") as hDst:
        hDst.write(json.dumps(dVars, ensure_ascii=False))
    return dVars

        sAction = createFunction("da", sActionId, sAction)
        return [sOption, sCondition, cAction, sAction]
    else:
        print(" # Unknown action.", sActionId)
        return None


def make (lRule, dDef, sLang, dOptPriority):
    "compile rules, returns a dictionary of values"
    # for clarity purpose, don’t create any file here

    # removing comments, zeroing empty lines, creating definitions, storing tests, merging rule lines
    print("  parsing rules...")
    lTokenLine = []
    sActions = ""

        print("\nFunctions:")
        print(sPyCallables)

    # Result
    return {
        "graph_callables": sPyCallables,
        "graph_callablesJS": sJSCallables,
        "rules_graphs": str(dAllGraph),
        "rules_graphsJS": str(dAllGraph).replace("True", "true").replace("False", "false"),
        "rules_actions": str(dACTIONS),
        "rules_actionsJS": str(dACTIONS).replace("True", "true").replace("False", "false")
    }

// JavaScript

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, console */

"use strict";

//console.log('\x1B[2J\x1B[0f'); //Clear the console (cmd win)

var oGrammalecte = require('./fr/gc_engine.js');

oGrammalecte.load();

var sPhrase = 'Le silences s’amasses Étoile brillantesss';
console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Test de vérification de:', sPhrase);
console.log( oGrammalecte.parse(sPhrase) );

var sWord = 'toutt';
console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Test de suggestion:', sWord);
var oSpellCheckGramma = oGrammalecte.getSpellChecker();
console.log( Array.from( oSpellCheckGramma.suggest(sWord) ) );

// Grammar checker engine

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";

${string}
${regex}
${map}


if(typeof(process) !== 'undefined') {
    var gc_options = require("./gc_options.js");
    var gc_rules = require("./gc_rules.js");
    var gc_rules_graph = require("./gc_rules_graph.js");
    var cregex = require("./cregex.js");
    var text = require("../text.js");
} else if (typeof(require) !== 'undefined') {
    var gc_options = require("resource://grammalecte/${lang}/gc_options.js");
    var gc_rules = require("resource://grammalecte/${lang}/gc_rules.js");
    var gc_rules_graph = require("resource://grammalecte/${lang}/gc_rules_graph.js");
    var cregex = require("resource://grammalecte/${lang}/cregex.js");
    var text = require("resource://grammalecte/text.js");
}

    version: "${version}",
    author: "${author}",

    //// Initialization

    load: function (sContext="JavaScript", sColorType="aRGB", sPath="") {
        try {
            if(typeof(process) !== 'undefined') {
                var spellchecker = require("../graphspell/spellchecker.js");
                _oSpellChecker = new spellchecker.SpellChecker("${lang}", "", "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}");
            } else if (typeof(require) !== 'undefined') {
                var spellchecker = require("resource://grammalecte/graphspell/spellchecker.js");
                _oSpellChecker = new spellchecker.SpellChecker("${lang}", "", "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}");
            } else {
                _oSpellChecker = new SpellChecker("${lang}", sPath, "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}");
            }
            _sAppContext = sContext;
            _dOptions = gc_options.getOptions(sContext).gl_shallowCopy();     // duplication necessary, to be able to reset to default

        this.dTokenPos = new Map();
        this.dTags = new Map();
        this.dError = new Map();
        this.dErrorPriority = new Map();  // Key = position; value = priority
    }

    asString () {
        let s = "===== TEXT =====\n";
        s += "sentence: " + this.sSentence0 + "\n";
        s += "now:      " + this.sSentence  + "\n";
        for (let dToken of this.lToken) {
            s += `#${dToken["i"]}\t${dToken["nStart"]}:${dToken["nEnd"]}\t${dToken["sValue"]}\t${dToken["sType"]}`;
            if (dToken.hasOwnProperty("lMorph")) {
                s += "\t" + dToken["lMorph"].toString();
            }

// Options for Grammalecte

/* jshint esversion:6 */
/* jslint esversion:6 */
/* global exports */

${map}


var gc_options = {
    getOptions: function (sContext="JavaScript") {
        if (this.dOpt.hasOwnProperty(sContext)) {

            for (let [sOpt, sColor] of Object.entries(dOptColor)) {
                dColor[sOpt] = dColorType[sColor];
            }
            return dColor;
        }
        catch (e) {
            console.error(e);
            return {};
        }
    },

    lStructOpt: ${lStructOpt},

    dOpt: {
        "JavaScript": new Map (${dOptJavaScript}),
        "Firefox": new Map (${dOptFirefox}),
        "Thunderbird": new Map (${dOptThunderbird}),
    },

    dColorType: ${dColorType},

    dOptColor: ${dOptColor},

    dOptLabel: ${dOptLabel}
};


if (typeof(exports) !== 'undefined') {
    exports.getOptions = gc_options.getOptions;
    exports.getOptionsColors = gc_options.getOptionsColors;
    exports.lStructOpt = gc_options.lStructOpt;
    exports.dOpt = gc_options.dOpt;
    exports.dColorType = gc_options.dColorType;
    exports.dOptColor = gc_options.dOptColor;
    exports.dOptLabel = gc_options.dOptLabel;
}

// Grammar checker rules

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/*global exports*/

"use strict";

${string}
${regex}

var gc_rules = {
    lParagraphRules: ${paragraph_rules_JS},

    lSentenceRules: ${sentence_rules_JS}
};


if (typeof(exports) !== 'undefined') {
    exports.lParagraphRules = gc_rules.lParagraphRules;
    exports.lSentenceRules = gc_rules.lSentenceRules;
}

// Grammar checker graph rules

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global exports */

"use strict";

${string}


var gc_rules_graph = {
    dAllGraph: ${rules_graphsJS},

    dRule: ${rules_actionsJS}
};


if (typeof(exports) !== 'undefined') {
    exports.dAllGraph = gc_rules_graph.dAllGraph;
    exports.dRule = gc_rules_graph.dRule;
}

// JavaScript

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";


if(typeof(process) !== 'undefined') {
    var helpers = require("./graphspell/helpers.js");
} else if (typeof(require) !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}


class TestGrammarChecking {

    constructor (gce, spfTests="") {
        this.gce = gce;
        this.spfTests = spfTests;
        this._aRuleTested = new Set();
    }

    * testParse (bDebug=false) {
        const t0 = Date.now();
        let sURL;
        if(typeof(process) !== 'undefined') {
            sURL = (this.spfTests !== "") ? this.spfTests : "./"+this.gce.lang+"/tests_data.json";
        } else {
            sURL = (this.spfTests !== "") ? this.spfTests : "resource://grammalecte/"+this.gce.lang+"/tests_data.json";
        }
        const aData = JSON.parse(helpers.loadFile(sURL)).aData;
        let nInvalid = 0;
        let nTotal = 0;
        let sErrorText;
        let sSugg;
        let sExpectedErrors;
        let sTextToCheck;

// JavaScript

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";


var text = {
    getParagraph: function* (sText, sSepParagraph = "\n") {
        // generator: returns paragraphs of text

// Grammalecte - Conjugueur
// License: GPL 3

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, self, browser, chrome, __dirname */

"use strict";

${map}


if(typeof(process) !== 'undefined') {
    var helpers = require("../graphspell/helpers.js");
} else if (typeof(require) !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}

var conj = {
    _lVtyp: [],
    _lTags: [],
    _dPatternConj: {},

        }
        return (this.dConj.get(":Q").get(":Q4")) ? this.dConj.get(":Q").get(":Q4") : this.dConj.get(":Q").get(":Q1");
    }
}


// Initialization
if(!conj.bInit && typeof(process) !== 'undefined') {
    // Work with nodejs
    conj.init(helpers.loadFile(__dirname+"/conj_data.json"));
} else if (!conj.bInit && typeof(browser) !== 'undefined') {
    // WebExtension Standard (but not in Worker)
    conj.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/conj_data.json")));
} else if (!conj.bInit && typeof(chrome) !== 'undefined') {
    // WebExtension Chrome (but not in Worker)
    conj.init(helpers.loadFile(chrome.extension.getURL("grammalecte/fr/conj_data.json")));
} else if (!conj.bInit && typeof(require) !== 'undefined') {
    // Add-on SDK and Thunderbird
    conj.init(helpers.loadFile("resource://grammalecte/fr/conj_data.json"));
} else if (!conj.bInit && typeof(self) !== 'undefined' && typeof(self.port) !== 'undefined' && typeof(self.port.on) !== 'undefined') {
    // used within Firefox content script (conjugation panel).
    // can’t load JSON from here, so we do it in ui.js and send it here.
    self.port.on("provideConjData", function (sJSONData) {
        conj.init(sJSONData);
    });
} else if (conj.bInit){
    console.log("Module conj déjà initialisé");
} else {
    //console.log("Module conj non initialisé");
}

//  JavaScript


//    Conjugation generator
//    beta stage, unfinished, the root for a new way to generate flexions…


/* jshint esversion:6, -W097 */
/* jslint esversion:6 */

"use strict";


var conj_generator = {

    conjugate: function (sVerb, sVerbTag="i_____a", bVarPpas=true) {

            [2,     "isses",        ":Sp:Sq:2s/*",      false],
            [2,     "isse",         ":Sp:3s/*",         false],
            [2,     "ît",           ":Sq:3s/*",         false],
            [2,     "is",           ":E:2s/*",          false],
            [2,     "issons",       ":E:1p/*",          false],
            [2,     "issez",        ":E:2p/*",          false]
        ],

        // premier groupe (bien plus irrégulier que prétendu)
        "V1": {
            // a
            // verbes en -er, -ger, -yer, -cer
            "er": [
                [2,      "er",        ":Y/*",               false],
                [2,      "ant",       ":P/*",               false],

// Grammalecte - Compiled regular expressions

/* jshint esversion:6 */
/* jslint esversion:6 */


var cregex = {
    ///// Lemme
    _zLemma: new RegExp(">([a-zà-öø-ÿ0-9Ā-ʯ][a-zà-öø-ÿ0-9Ā-ʯ-]+)"),

    ///// Masculin / féminin / singulier / pluriel

// GRAMMAR CHECKING ENGINE PLUGIN: Parsing functions for French language

/* jshint esversion:6 */
/* jslint esversion:6 */

function g_morphVC (dToken, sPattern, sNegPattern="") {
    let nEnd = dToken["sValue"].lastIndexOf("-");
    if (dToken["sValue"].includes("-t-")) {
        nEnd = nEnd - 2;
    }
    return g_morph(dToken, sPattern, sNegPattern, 0, nEnd, false);

    if (s.length > 1 && s.length < 16 && s.slice(0, 1).gl_isLowerCase() && (!s.slice(1).gl_isLowerCase() || /[0-9]/.test(s))) {
        return true;
    }
    return false;
}


// Exceptions

const aREGULARPLURAL = new Set(["abricot", "amarante", "aubergine", "acajou", "anthracite", "brique", "caca", "café",
                                "carotte", "cerise", "chataigne", "corail", "citron", "crème", "grave", "groseille",
                                "jonquille", "marron", "olive", "pervenche", "prune", "sable"]);
const aSHOULDBEVERB = new Set(["aller", "manger"]);

// GRAMMAR CHECKING ENGINE PLUGIN


// Check date validity
// WARNING: when creating a Date, month must be between 0 and 11

/* jshint esversion:6 */
/* jslint esversion:6 */


const _lDay = ["dimanche", "lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi"];
const _dMonth = new Map ([
    ["janvier", 1], ["février", 2], ["mars", 3], ["avril", 4], ["mai", 5], ["juin", 6], ["juillet", 7],
    ["août", 8], ["aout", 8], ["septembre", 9], ["octobre", 10], ["novembre", 11], ["décembre", 12]
]);

// GRAMMAR CHECKING ENGINE PLUGIN: Suggestion mechanisms

/* jshint esversion:6 */
/* jslint esversion:6 */
/* global require */

if(typeof(process) !== 'undefined') {
    var conj = require("./conj.js");
    var mfsp = require("./mfsp.js");
    var phonet = require("./phonet.js");
} else if (typeof(require) !== 'undefined') {
    var conj = require("resource://grammalecte/fr/conj.js");
    var mfsp = require("resource://grammalecte/fr/mfsp.js");
    var phonet = require("resource://grammalecte/fr/phonet.js");
}


//// verbs

// Grammalecte - Lexicographe
// License: MPL 2

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";

${string}
${map}

// Grammalecte

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, browser,__dirname */

"use strict";


if(typeof(process) !== 'undefined') {
    var helpers = require("../graphspell/helpers.js");
} else if (typeof(require) !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}


var mfsp = {
    // list of affix codes
    _lTagMiscPlur: [],

            return "## erreur, code : " + sSfx + " ##";
        }
    }
};


// Initialization
if(!mfsp.bInit && typeof(process) !== 'undefined') {
    //Nodejs
    mfsp.init(helpers.loadFile(__dirname+"/mfsp_data.json"));
} else if (!mfsp.bInit && typeof(browser) !== 'undefined') {
    // WebExtension
    mfsp.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/mfsp_data.json")));
} else if (!mfsp.bInit && typeof(require) !== 'undefined') {
    // Add-on SDK and Thunderbird
    mfsp.init(helpers.loadFile("resource://grammalecte/fr/mfsp_data.json"));
} else if (mfsp.bInit){
    console.log("Module mfsp déjà initialisé");

// Grammalecte - Suggestion phonétique

/* jshint esversion:6 */
/* jslint esversion:6 */
/* global __dirname */

if(typeof(process) !== 'undefined') {
    var helpers = require("../graphspell/helpers.js");
} else if (typeof(require)  !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}


var phonet = {
    _dWord: new Map(),
    _lSet: [],

        }
        return aSelect;
    }
};


// Initialization
if (!phonet.bInit && typeof(process) !== 'undefined') {
    //Nodejs
    phonet.init(helpers.loadFile(__dirname+"/phonet_data.json"));
} else if (!phonet.bInit && typeof(browser) !== 'undefined') {
    // WebExtension
    phonet.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/phonet_data.json")));
} else if (!phonet.bInit && typeof(require) !== 'undefined') {
    // Add-on SDK and Thunderbird
    phonet.init(helpers.loadFile("resource://grammalecte/fr/phonet_data.json"));
} else if (phonet.bInit){
    console.log("Module phonet déjà initialisé");

// Grammalecte - text formatter

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global exports, console */

"use strict";

//!${map}


// Latin letters: http://unicode-table.com/fr/
// 0-9
// A-Z
// a-z
// À-Ö     00C0-00D6   (upper case)

    "erase_non_breaking_hyphens": [ [/­/g, ""] ],
    //// typographic signs
    "ts_apostrophe":              [ [/\b([ldnjmtscç])['´‘′`](?=[a-zA-Zà-ö0-9À-Öø-ÿØ-ßĀ-ʯ])/ig, "$1’"],
                                    [/\b(qu|jusqu|lorsqu|puisqu|quoiqu|quelqu|presqu|entr|aujourd|prud)['´‘′`]/ig, "$1’"] ],
    "ts_ellipsis":                [ [/\.\.\./g, "…"],
                                    [/…\.\./g, "……"],
                                    [/…\.(?!\.)/g, "…"] ],
    "ts_n_dash_middle":           [ [/ [-—] /g, " – "],
                                    [/ [-—],/g, " –,"] ],
    "ts_m_dash_middle":           [ [/ [-–] /g, " — "],
                                    [/ [-–],/g, " —,"] ],
    "ts_n_dash_start":            [ [/^[-—][  ]/gm, "– "],
                                    [/^– /gm, "– "],
                                    [/^[-–—](?=[a-zA-Zà-ö0-9À-Öø-ÿØ-ßĀ-ʯ.…])/gm, "– "] ],
    "ts_m_dash_start":            [ [/^[-–][  ]/gm, "— "],

    ["etc", true],
    ["missing_hyphens", true],
    ["ma_word", true],
    ["ma_1letter_lowercase", false],
    ["ma_1letter_uppercase", false]
]);




class TextFormatter {

    constructor (bDebug=false) {
        this.sLang = "fr";
        this.bDebug = bDebug;
        //don't change this in external ;)
        this._optionsUsed = dTFDefaultOptions.gl_shallowCopy();
    }

    formatText (sText, dOpt=null) {
        if (dOpt !== null) {
            this._optionsUsed.gl_updateOnlyExistingKeys(dOpt);
        }
        for (let [sOptName, bVal] of this._optionsUsed) {
            //console.log(oReplTable);
            if (bVal && oReplTable[sOptName]) {
                for (let [zRgx, sRep] of oReplTable[sOptName]) {
                    sText = sText.replace(zRgx, sRep);
                }
            }
        }
        return sText;
    }

    formatTextCount (sText, dOpt=null) {
        let nCount = 0;
        if (dOpt !== null) {
            this._optionsUsed.gl_updateOnlyExistingKeys(dOpt);
        }
        for (let [sOptName, bVal] of this._optionsUsed) {
            if (bVal && oReplTable[sOptName]) {
                for (let [zRgx, sRep] of oReplTable[sOptName]) {
                    nCount += (sText.match(zRgx) || []).length;
                    sText = sText.replace(zRgx, sRep);
                }
            }
        }
        return [sText, nCount];
    }

    formatTextRule (sText, sRuleName) {
        if (oReplTable[sRuleName]) {
            for (let [zRgx, sRep] of oReplTable[sRuleName]) {
                sText = sText.replace(zRgx, sRep);
            }
        } else if (this.bDebug){
            console.log("# Error. TF: there is no option “" + sRuleName+ "”.");
        }
        return sText;
    }

    formatTextRuleCount (sText, sRuleName) {
        let nCount = 0;
        if (oReplTable[sRuleName]) {
            for (let [zRgx, sRep] of oReplTable[sRuleName]) {
                nCount += (sText.match(zRgx) || []).length;
                sText = sText.replace(zRgx, sRep);
            }
        } else if (this.bDebug){
            console.log("# Error. TF: there is no option “" + sRuleName+ "”.");
        }
        return [sText, nCount];
    }

    getDefaultOptions () {
        //we return a copy to make sure they are no modification in external
        return dTFDefaultOptions.gl_shallowCopy();
    }

    getUsedOptions () {
        //we return a copy to make sure they are no modification in external
        return this._optionsUsed.gl_shallowCopy();
    }

    setUsedOptions (dOpt=null) {
        if (dOpt !== null) {
            this._optionsUsed.gl_updateOnlyExistingKeys(dOpt);
        } else if (this.bDebug){
            console.log("# Error. TF: no option to change.");
        }
    }

    getReplTable(){
        return oReplTable;
    }
}


if (typeof exports !== 'undefined') {
    exports.TextFormatter = TextFormatter;
    exports.oReplTable = oReplTable;
}

// list of similar chars
// useful for suggestion mechanism

/* jshint esversion:6 */
/* jslint esversion:6 */

${map}


var char_player = {

    _xTransCharsForSpelling: new Map([
        ['ſ', 's'],  ['ffi', 'ffi'],  ['ffl', 'ffl'],  ['ff', 'ff'],  ['ſt', 'ft'],  ['fi', 'fi'],  ['fl', 'fl'],  ['st', 'st']

    _xTransCharsForSimplification: new Map([
        ['à', 'a'],  ['é', 'e'],  ['î', 'i'],  ['ô', 'o'],  ['û', 'u'],  ['ÿ', 'i'],  ['y', 'i'],
        ['â', 'a'],  ['è', 'e'],  ['ï', 'i'],  ['ö', 'o'],  ['ù', 'u'],  ['ŷ', 'i'],
        ['ä', 'a'],  ['ê', 'e'],  ['í', 'i'],  ['ó', 'o'],  ['ü', 'u'],  ['ý', 'i'],
        ['á', 'a'],  ['ë', 'e'],  ['ì', 'i'],  ['ò', 'o'],  ['ú', 'u'],  ['ỳ', 'i'],
        ['ā', 'a'],  ['ē', 'e'],  ['ī', 'i'],  ['ō', 'o'],  ['ū', 'u'],  ['ȳ', 'i'],
        ['ç', 'c'],  ['ñ', 'n'],  ['k', 'q'],  ['w', 'v'],
        ['œ', 'oe'], ['æ', 'ae'],
        ['ſ', 's'],  ['ffi', 'ffi'],  ['ffl', 'ffl'],  ['ff', 'ff'],  ['ſt', 'ft'],  ['fi', 'fi'],  ['fl', 'fl'],  ['st', 'st']
    ]),

    simplifyWord: function (sWord) {
        // word simplication before calculating distance between words
        sWord = sWord.toLowerCase();
        sWord = [...sWord].map(c => this._xTransCharsForSimplification.gl_get(c, c)).join('');

        ["Ë", "EeÉéÈèÊêËëĒēŒœ"],

        ["f", "fF"],
        ["F", "Ff"],

        ["g", "gGjJĵĴ"],
        ["G", "GgJjĴĵ"],

        ["h", "hH"],
        ["H", "Hh"],

        ["i", "iIîÎïÏyYíÍìÌīĪÿŸ"],
        ["I", "IiÎîÏïYyÍíÌìĪīŸÿ"],
        ["î", "iIîÎïÏyYíÍìÌīĪÿŸ"],
        ["Î", "IiÎîÏïYyÍíÌìĪīŸÿ"],

    },

    // Other functions
    filterSugg: function (aSugg) {
        return aSugg.filter((sSugg) => { return !sSugg.endsWith("è") && !sSugg.endsWith("È"); });
    }

};


if (typeof(exports) !== 'undefined') {
    exports._xTransCharsForSpelling = char_player._xTransCharsForSpelling;
    exports.spellingNormalization = char_player.spellingNormalization;
    exports._xTransCharsForSimplification = char_player._xTransCharsForSimplification;
    exports.simplifyWord = char_player.simplifyWord;

// JavaScript

// FSA DICTIONARY BUILDER
//
// by Olivier R.
// License: MPL 2
//
// This tool encodes lexicon into an indexable binary dictionary
// Input files MUST be encoded in UTF-8.

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, helpers */

"use strict";

if(typeof(process) !== 'undefined') {
    var str_transform = require("./str_transform.js");
} else if (typeof(require) !== 'undefined') {
    var str_transform = require("resource://grammalecte/graphspell/str_transform.js");
}


${map}


class DAWG {
    /*  DIRECT ACYCLIC WORD GRAPH
        This code is inspired from Steve Hanov’s DAWG, 2011. (http://stevehanov.ca/blog/index.php?id=115)

        let lWord = [];
        for (let [sFlex, iAff, iTag] of lEntry) {
            let lTemp = [];
            for (let c of sFlex) {
                lTemp.push(dChar.get(c));
            }
            lTemp.push(iAff+nChar);
            lTemp.push(iTag+nChar+nAff);
            lWord.push(lTemp);
        }
        lEntry.length = 0; // clear the array

        // Dictionary of arc values occurrency, to sort arcs of each node
        let lKeyVal = [];
        for (let c of dChar.keys()) { lKeyVal.push([dChar.get(c), dCharOccur.get(c)]); }

        this.nNextId += 1;
        return this.nNextId-1;
    },

    reset: function () {
        this.nNextId = 0;
    }
};


class DawgNode {

    constructor () {
        this.i = oNodeCounter.getId();
        this.final = false;

    convValueToHexString (nVal, nByte) {
        // nVal: value to convert, nByte: number of bytes
        let sHexVal = nVal.toString(16); // conversion to hexadecimal string
        //console.log(`value: ${nVal} in ${nByte} bytes`);
        if (sHexVal.length < (nByte*2)) {
            return "0".repeat((nByte*2) - sHexVal.length) + sHexVal;
        } else if (sHexVal.length == (nByte*2)) {
            return sHexVal;
        } else {
            throw "Conversion to byte string: value bigger than allowed.";
        }
    }
}

// HELPERS

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, XMLHttpRequest */

"use strict";


var helpers = {

    inspect: function (o) {
        let sMsg = "__inspect__: " + typeof o;
        for (let sParam in o) {
            sMsg += "\n" + sParam + ": " + o.sParam;
        }
        sMsg += "\n" + JSON.stringify(o) + "\n__end__";
        console.log(sMsg);
    },

    loadFile: function (spf) {
        // load ressources in workers (suggested by Mozilla extensions reviewers)
        // for more options have a look here: https://gist.github.com/Noitidart/ec1e6b9a593ec7e3efed
        // if not in workers, use sdk/data.load() instead
        try {
            if(typeof(process) !== 'undefined') {
                //console.log('loadFile(disque): ' + spf);
                let fs = require("fs");
                return fs.readFileSync(spf, "utf8");
            } else {
                console.log("loadFile: " + spf);
                let xRequest;
                xRequest = new XMLHttpRequest();
                xRequest.open('GET', spf, false); // 3rd arg is false for synchronous, sync is acceptable in workers
                xRequest.overrideMimeType('text/json');
                xRequest.send();
                return xRequest.responseText;
            }
        }
        catch (e) {
            console.error(e);
            return null;
        }
    },

// IBDAWG

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, __dirname */

"use strict";

if(typeof(process) !== 'undefined') {
    var str_transform = require("./str_transform.js");
    var helpers = require("./helpers.js");
    var char_player = require("./char_player.js");
} else if (typeof(require) !== 'undefined') {
    var str_transform = require("resource://grammalecte/graphspell/str_transform.js");
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
    var char_player = require("resource://grammalecte/graphspell/char_player.js");
}


// Don’t remove <string>. Necessary in TB.
${string}
${map}
${set}

    // INDEXABLE BINARY DIRECT ACYCLIC WORD GRAPH

    constructor (param1, sPath="") {
        // param1 can be a filename or a object with all the necessary data.
        try {
            let oData = null;
            if (typeof(param1) == "string") {
                let sURL;
                if(typeof(process) !== 'undefined') {
                    sURL = (sPath !== "") ? sPath + "/" + param1 : __dirname + "/_dictionaries/"+param1;
                } else {
                    sURL = (sPath !== "") ? sPath + "/" + param1 : "resource://grammalecte/graphspell/_dictionaries/"+param1;
                }
                oData = JSON.parse(helpers.loadFile(sURL));
            } else {
                oData = param1;
            }
            Object.assign(this, oData);
        }
        catch (e) {

            "l2grams": this.l2grams
        };
        return oJSON;
    }

    isValidToken (sToken) {
        // checks if sToken is valid (if there is hyphens in sToken, sToken is split, each part is checked)
        sToken = char_player.spellingNormalization(sToken);
        if (this.isValid(sToken)) {
            return true;
        }
        if (sToken.includes("-")) {
            if (sToken.gl_count("-") > 4) {
                return true;
            }

            }
        }
        return Boolean(this._convBytesToInteger(this.byDic.slice(iAddr, iAddr+this.nBytesArc)) & this._finalNodeMask);
    }

    getMorph (sWord) {
        // retrieves morphologies list, different casing allowed
        sWord = char_player.spellingNormalization(sWord);
        let l = this.morph(sWord);
        if (sWord[0].gl_isUpperCase()) {
            l.push(...this.morph(sWord.toLowerCase()));
            if (sWord.gl_isUpperCase() && sWord.length > 1) {
                l.push(...this.morph(sWord.gl_toCapitalize()));
            }
        }
        return l;
    }

    suggest (sWord, nSuggLimit=10) {
        // returns a array of suggestions for <sWord>
	//console.time("Suggestions for " + sWord + ");
        sWord = char_player.spellingNormalization(sWord);
        let sPfx = "";
        let sSfx = "";
        [sPfx, sWord, sSfx] = char_player.cut(sWord);
        let nMaxSwitch = Math.max(Math.floor(sWord.length / 3), 1);
        let nMaxDel = Math.floor(sWord.length / 5);
        let nMaxHardRepl = Math.max(Math.floor((sWord.length - 5) / 4), 1);
        let nMaxJump = Math.max(Math.floor(sWord.length / 4), 1);
        let oSuggResult = new SuggResult(sWord);
        this._suggest(oSuggResult, sWord, nMaxSwitch, nMaxDel, nMaxHardRepl, nMaxJump);
        let aSugg = oSuggResult.getSuggestions(nSuggLimit);
        if (sSfx || sPfx) {
            // we add what we removed
            return aSugg.map( (sSugg) => { return sPfx + sSugg + sSfx; } );
        }

	//console.timeEnd("Suggestions for " + sWord + ");
        return aSugg;
    }

    _suggest (oSuggResult, sRemain, nMaxSwitch=0, nMaxDel=0, nMaxHardRepl=0, nMaxJump=0, nDist=0, nDeep=0, iAddr=0, sNewWord="", bAvoidLoop=false) {
        // returns a set of suggestions
        // recursive function
        if (sRemain == "") {

                }
                iAddr = iEndArcAddr + this.nBytesNodeAddress;
            }
        }
    }

    * _getArcs1 (iAddr) {
        // generator: return all arcs at <iAddr> as tuples of (nVal, iAddr)
        while (true) {
            let iEndArcAddr = iAddr+this.nBytesArc;
            let nRawArc = this._convBytesToInteger(this.byDic.slice(iAddr, iEndArcAddr));
            yield [nRawArc & this._arcMask, this._convBytesToInteger(this.byDic.slice(iEndArcAddr, iEndArcAddr+this.nBytesNodeAddress))];
            if (nRawArc & this._lastArcMask) {
                break;
            }

// Spellchecker
// Wrapper for the IBDAWG class.
// Useful to check several dictionaries at once.

// To avoid iterating over a pile of dictionaries, it is assumed that 3 are enough:
// - the main dictionary, bundled with the package
// - the extended dictionary
// - the community dictionary, added by an organization
// - the personal dictionary, created by the user for its own convenience

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, IBDAWG, Tokenizer */

"use strict";

if(typeof(process) !== 'undefined') {
    var ibdawg = require("./ibdawg.js");
    var tokenizer = require("./tokenizer.js");
} else if (typeof(require) !== 'undefined') {
    var ibdawg = require("resource://grammalecte/graphspell/ibdawg.js");
    var tokenizer = require("resource://grammalecte/graphspell/tokenizer.js");
}


${map}


const dDefaultDictionaries = new Map([
    ["fr", "fr-allvars.json"],
    ["en", "en.json"]

            }
        }
        catch (e) {
            let sfDictionary = (typeof(dictionary) == "string") ? dictionary : dictionary.sLangName + "/" + dictionary.sFileName;
            if (bNecessary) {
                throw "Error: <" + sfDictionary + "> not loaded. " + e.message;
            }
            console.log("Error: <" + sfDictionary + "> not loaded.");
            console.log(e.message);
            return null;
        }
    }

    loadTokenizer () {
        if (typeof(tokenizer) !== 'undefined') {

        // checks if sWord is valid (different casing tested if the first letter is a capital)
        if (this.oMainDic.isValid(sWord)) {
            return true;
        }
        if (this.bExtendedDic && this.oExtendedDic.isValid(sWord)) {
            return true;
        }
        if (this.bCommunityDic && this.oCommunityDic.isValid(sWord)) {
            return true;
        }
        if (this.bPersonalDic && this.oPersonalDic.isValid(sWord)) {
            return true;
        }
        return false;
    }

    lookup (sWord) {
        // checks if sWord is in dictionary as is (strict verification)
        if (this.oMainDic.lookup(sWord)) {
            return true;
        }
        if (this.bExtendedDic && this.oExtendedDic.lookup(sWord)) {
            return true;
        }
        if (this.bCommunityDic && this.oCommunityDic.lookup(sWord)) {
            return true;
        }
        if (this.bPersonalDic && this.oPersonalDic.lookup(sWord)) {
            return true;
        }
        return false;
    }

        if (this.bPersonalDic) {
            yield this.oPersonalDic.suggest(sWord, nSuggLimit);
        }
    }

    * select (sFlexPattern="", sTagsPattern="") {
        // generator: returns all entries which flexion fits <sFlexPattern> and morphology fits <sTagsPattern>
        yield* this.oMainDic.select(sFlexPattern, sTagsPattern);
        if (this.bExtendedDic) {
            yield* this.oExtendedDic.select(sFlexPattern, sTagsPattern);
        }
        if (this.bCommunityDic) {
            yield* this.oCommunityDic.select(sFlexPattern, sTagsPattern);
        }
        if (this.bPersonalDic) {

// STRING TRANSFORMATION

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global exports, console */

"use strict";


// Note: 48 is the ASCII code for "0"

var str_transform = {

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, console */

"use strict";

/*
Reset = "\x1b[0m"
Bright = "\x1b[1m"
Dim = "\x1b[2m"
Underscore = "\x1b[4m"
Blink = "\x1b[5m"
Reverse = "\x1b[7m"
Hidden = "\x1b[8m"

FgBlack = "\x1b[30m"
FgRed = "\x1b[31m"
FgGreen = "\x1b[32m"
FgYellow = "\x1b[33m"
FgBlue = "\x1b[34m"
FgMagenta = "\x1b[35m"
FgCyan = "\x1b[36m"
FgWhite = "\x1b[37m"

BgBlack = "\x1b[40m"
BgRed = "\x1b[41m"
BgGreen = "\x1b[42m"
BgYellow = "\x1b[43m"
BgBlue = "\x1b[44m"
BgMagenta = "\x1b[45m"
BgCyan = "\x1b[46m"
BgWhite = "\x1b[47m"
*/

//console.log('\x1B[2J\x1B[0f'); //Clear the console (cmd win)

var spellCheck = require("../spellchecker.js");
var checker = new spellCheck.SpellChecker('fr', '../_dictionaries');

function perf(sWord){
    console.log('\x1b[1m\x1b[31m%s\x1b[0m', '--------------------------------');

    console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Vérification de:', sWord);
    console.time('Valid:'+sWord);
    console.log(sWord, checker.isValid(sWord) );
    console.timeEnd('Valid:'+sWord);

    console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Suggestion de:', sWord);
    console.time('Suggestion:'+sWord);
    console.log( JSON.stringify( Array.from(checker.suggest(sWord)) ) );
    console.timeEnd('Suggestion:'+sWord);
}

perf('binjour');
perf('saluté');
perf('graphspell');
perf('dicollecte');

// JavaScript
// Very simple tokenizer

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/*global require, exports, console*/

"use strict";


const aTkzPatterns = {
    // All regexps must start with ^.
    "default":

        while (sText) {
            let iCut = 1;
            for (let [zRegex, sType] of this.aRules) {
                if (sType !== "SPACE"  ||  bWithSpaces) {
                    try {
                        if ((m = zRegex.exec(sText)) !== null) {
                            iToken += 1;
                            yield { "i": iToken, "sType": sType, "sValue": m[0], "nStart": iNext, "nEnd": iNext + m[0].length };
                            iCut = m[0].length;
                            break;
                        }
                    }
                    catch (e) {
                        console.error(e);
                    }

        if not os.path.isdir("grammalecte/graphspell/_dictionaries/"+sf):
            hZip.write("grammalecte/graphspell/_dictionaries/"+sf, sAddPath+"grammalecte/graphspell/_dictionaries/"+sf)
    for sf in os.listdir(spLangPack):
        if not os.path.isdir(spLangPack+"/"+sf):
            hZip.write(spLangPack+"/"+sf, sAddPath+spLangPack+"/"+sf)


def create (sLang, xConfig, bInstallOXT, bJavaScript, bUseCache):
    "make Grammalecte for project <sLang>"
    oNow = datetime.datetime.now()
    print("============== MAKE GRAMMALECTE [{0}] at {1.hour:>2} h {1.minute:>2} min {1.second:>2} s ==============".format(sLang, oNow))

    #### READ CONFIGURATION
    print("> read configuration...")
    spLang = "gc_lang/" + sLang

    dVars = xConfig._sections['args']
    dVars['locales'] = dVars["locales"].replace("_", "-")
    dVars['loc'] = str(dict([ [s, [s[0:2], s[3:5], ""]] for s in dVars["locales"].split(" ") ]))

    ## COMPILE RULES
    dResult = compile_rules.make(spLang, dVars['lang'], bUseCache)
    dVars.update(dResult)

    ## READ GRAMMAR CHECKER PLUGINS
    print("PYTHON:")
    print("+ Plugins: ", end="")
    sCodePlugins = ""
    for sf in os.listdir(spLang+"/modules"):

def main ():
    "build Grammalecte with requested options"
    print("Python: " + sys.version)
    xParser = argparse.ArgumentParser()
    xParser.add_argument("lang", type=str, nargs='+', help="lang project to generate (name of folder in /lang)")
    xParser.add_argument("-uc", "--use_cache", help="use data cache instead of rebuilding rules", action="store_true")
    xParser.add_argument("-b", "--build_data", help="launch build_data.py (part 1 and 2)", action="store_true")
    xParser.add_argument("-bb", "--build_data_before", help="launch build_data.py (only part 1: before dictionary building)", action="store_true")
    xParser.add_argument("-ba", "--build_data_after", help="launch build_data.py (only part 2: before dictionary building)", action="store_true")
    xParser.add_argument("-d", "--dict", help="generate FSA dictionary", action="store_true")
    xParser.add_argument("-t", "--tests", help="run unit tests", action="store_true")
    xParser.add_argument("-p", "--perf", help="run performance tests", action="store_true")
    xParser.add_argument("-pm", "--perf_memo", help="run performance tests and store results in perf_memo.txt", action="store_true")

            if databuild and xArgs.build_data_after:
                databuild.after('gc_lang/'+sLang, dVars, xArgs.javascript)

            # copy dictionaries from Graphspell
            copyGraphspellDictionaries(dVars, xArgs.javascript, xArgs.add_extended_dictionary, xArgs.add_community_dictionary, xArgs.add_personal_dictionary)

            # make
            sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, xArgs.use_cache)

            # tests
            if xArgs.tests or xArgs.perf or xArgs.perf_memo:
                print("> Running tests")
                try:
                    tests = importlib.import_module("grammalecte."+sLang+".tests")
                    print(tests.__file__)