Grammalecte  Changes On Branch 401f29a39dc81bc1

"""
Grammalecte: compile rules
"""

import re
import os
import traceback
import json
import colorsys
import time

import compile_rules_js_convert as jsconv
import compile_rules_graph as crg


dDEF = {}
lFUNCTIONS = []

    dColor = {}
    dOptLabel = {}
    dOptPriority = {}
    for sLine in lOptionLines:
        sLine = sLine.strip()
        if sLine.startswith("OPTGROUP/"):
            m = re.match("OPTGROUP/([a-z0-9]+):(.+)$", sLine)
            lStructOpt.append( (m.group(1), list(map(str.split, m.group(2).split(",")))) )
            lStructOpt.append( [m.group(1), list(map(str.split, m.group(2).split(",")))] )
        elif sLine.startswith("OPTSOFTWARE:"):
            lOpt = [ [s, {}]  for s in sLine[12:].strip().split() ]  # don’t use tuples (s, {}), because unknown to JS
        elif sLine.startswith("OPT/"):
            m = re.match("OPT/([a-z0-9]+):(.+)$", sLine)
            for i, sOpt in enumerate(m.group(2).split()):
                lOpt[i][1][m.group(1)] = eval(sOpt)
        elif sLine.startswith("OPTCOLORTHEME:"):

def printBookmark (nLevel, sComment, nLine):
    "print bookmark within the rules file"
    print("  {:>6}:  {}".format(nLine, "  " * nLevel + sComment))


def make (spLang, sLang, bJavaScript):
def make (spLang, sLang, bUseCache=False):
    "compile rules, returns a dictionary of values"
    # for clarity purpose, don’t create any file here

    if bUseCache and os.path.isfile("_build/data_cache.json"):
        print("> don’t rebuild rules, use cache...")
        sJSON = open("_build/data_cache.json", "r", encoding="utf-8").read()
        dCacheVars = json.loads(sJSON)
        print("  build made at: " + time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(dCacheVars.get("fBuildTime", 0))))
        return dCacheVars

    fBuildTime = time.time()

    print("> read rules file...")
    try:
        lRules = open(spLang + "/rules.grx", 'r', encoding="utf-8").readlines()
    except:
        print("Error. Rules file in project [" + sLang + "] not found.")
        exit()

        sJSCallables += "        return " + jsconv.py2js(sReturn) + ";\n"
        sJSCallables += "    },\n"

    displayStats(lParagraphRules, lSentenceRules)

    print("Unnamed rules: " + str(nRULEWITHOUTNAME))

    dVars = {
        "fBuildTime": fBuildTime,
    dVars = {   "callables": sPyCallables,
                "callablesJS": sJSCallables,
                "gctests": sGCTests,
                "gctestsJS": sGCTestsJS,
                "paragraph_rules": mergeRulesByOption(lParagraphRules),
                "sentence_rules": mergeRulesByOption(lSentenceRules),
                "paragraph_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lParagraphRulesJS)),
                "sentence_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lSentenceRulesJS)) }
        "callables": sPyCallables,
        "callablesJS": sJSCallables,
        "gctests": sGCTests,
        "gctestsJS": sGCTestsJS,
        "paragraph_rules": mergeRulesByOption(lParagraphRules),
        "sentence_rules": mergeRulesByOption(lSentenceRules),
        "paragraph_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lParagraphRulesJS)),
        "sentence_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lSentenceRulesJS))
    }
    dVars.update(dOptions)

    # compile graph rules
    dVars2 = crg.make(lGraphRule, dDEF, sLang, dOptPriority, bJavaScript)
    dVars2 = crg.make(lGraphRule, dDEF, sLang, dOptPriority)
    dVars.update(dVars2)

    with open("_build/data_cache.json", "w", encoding="utf-8") as hDst:
        hDst.write(json.dumps(dVars, ensure_ascii=False))
    return dVars

        sAction = createFunction("da", sActionId, sAction)
        return [sOption, sCondition, cAction, sAction]
    else:
        print(" # Unknown action.", sActionId)
        return None


def make (lRule, dDef, sLang, dOptPriority, bJavaScript):
def make (lRule, dDef, sLang, dOptPriority):
    "compile rules, returns a dictionary of values"
    # for clarity purpose, don’t create any file here

    # removing comments, zeroing empty lines, creating definitions, storing tests, merging rule lines
    print("  parsing rules...")
    lTokenLine = []
    sActions = ""

        print("\nFunctions:")
        print(sPyCallables)

    # Result
    return {
        "graph_callables": sPyCallables,
        "graph_callablesJS": sJSCallables,
        "rules_graphs": dAllGraph,
        "rules_graphs": str(dAllGraph),
        "rules_graphsJS": str(dAllGraph).replace("True", "true").replace("False", "false"),
        "rules_actions": dACTIONS,
        "rules_actions": str(dACTIONS),
        "rules_actionsJS": str(dACTIONS).replace("True", "true").replace("False", "false")
    }

// Grammar checker engine

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global console,require,exports*/
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";

${string}
${regex}
${map}


if(typeof(process) !== 'undefined') {
    var gc_options = require("./gc_options.js");
    var gc_rules = require("./gc_rules.js");
    var gc_rules_graph = require("./gc_rules_graph.js");
    var cregex = require("./cregex.js");
    var text = require("../text.js");
if (typeof(require) !== 'undefined') {
} else if (typeof(require) !== 'undefined') {
    var gc_options = require("resource://grammalecte/${lang}/gc_options.js");
    var gc_rules = require("resource://grammalecte/${lang}/gc_rules.js");
    var gc_rules_graph = require("resource://grammalecte/${lang}/gc_rules_graph.js");
    var cregex = require("resource://grammalecte/${lang}/cregex.js");
    var text = require("resource://grammalecte/text.js");
}

    version: "${version}",
    author: "${author}",

    //// Initialization

    load: function (sContext="JavaScript", sColorType="aRGB", sPath="") {
        try {
            if(typeof(process) !== 'undefined') {
                var spellchecker = require("../graphspell/spellchecker.js");
                _oSpellChecker = new spellchecker.SpellChecker("${lang}", "", "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}");
            if (typeof(require) !== 'undefined') {
            } else if (typeof(require) !== 'undefined') {
                var spellchecker = require("resource://grammalecte/graphspell/spellchecker.js");
                _oSpellChecker = new spellchecker.SpellChecker("${lang}", "", "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}");
            } else {
                _oSpellChecker = new SpellChecker("${lang}", sPath, "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}");
            }
            _sAppContext = sContext;
            _dOptions = gc_options.getOptions(sContext).gl_shallowCopy();     // duplication necessary, to be able to reset to default

        this.dTokenPos = new Map();
        this.dTags = new Map();
        this.dError = new Map();
        this.dErrorPriority = new Map();  // Key = position; value = priority
    }

    asString () {
        let s = "===== TEXT =====\n"
        let s = "===== TEXT =====\n";
        s += "sentence: " + this.sSentence0 + "\n";
        s += "now:      " + this.sSentence  + "\n";
        for (let dToken of this.lToken) {
            s += `#${dToken["i"]}\t${dToken["nStart"]}:${dToken["nEnd"]}\t${dToken["sValue"]}\t${dToken["sType"]}`;
            if (dToken.hasOwnProperty("lMorph")) {
                s += "\t" + dToken["lMorph"].toString();
            }

// Options for Grammalecte

/* jshint esversion:6 */
/*jslint esversion: 6*/
/*global exports*/
/* jslint esversion:6 */
/* global exports */

${map}


var gc_options = {
    getOptions: function (sContext="JavaScript") {
        if (this.dOpt.hasOwnProperty(sContext)) {

            for (let [sOpt, sColor] of Object.entries(dOptColor)) {
                dColor[sOpt] = dColorType[sColor];
            }
            return dColor;
        }
        catch (e) {
            console.error(e);
            return {}
            return {};
        }
    },

    lStructOpt: ${lStructOpt},

    dOpt: {
        "JavaScript": new Map (${dOptJavaScript}),
        "Firefox": new Map (${dOptFirefox}),
        "Thunderbird": new Map (${dOptThunderbird}),
    },

    dColorType: ${dColorType},

    dOptColor: ${dOptColor},

    dOptLabel: ${dOptLabel}
}
};


if (typeof(exports) !== 'undefined') {
	exports.getOptions = gc_options.getOptions;
    exports.getOptions = gc_options.getOptions;
    exports.getOptionsColors = gc_options.getOptionsColors;
	exports.lStructOpt = gc_options.lStructOpt;
    exports.lStructOpt = gc_options.lStructOpt;
    exports.dOpt = gc_options.dOpt;
    exports.dColorType = gc_options.dColorType;
    exports.dOptColor = gc_options.dOptColor;
	exports.dOptLabel = gc_options.dOptLabel;
    exports.dOptLabel = gc_options.dOptLabel;
}

// Grammar checker rules

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/* jslint esversion:6 */
/*global exports*/

"use strict";

${string}
${regex}

var gc_rules = {
    lParagraphRules: ${paragraph_rules_JS},

    lSentenceRules: ${sentence_rules_JS}
}
};


if (typeof(exports) !== 'undefined') {
    exports.lParagraphRules = gc_rules.lParagraphRules;
    exports.lSentenceRules = gc_rules.lSentenceRules;
}

// Grammar checker graph rules

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global exports*/
/* jslint esversion:6 */
/* global exports */

"use strict";

${string}


var gc_rules_graph = {
    dAllGraph: ${rules_graphsJS},

    dRule: ${rules_actionsJS}
}
};


if (typeof(exports) !== 'undefined') {
    exports.dAllGraph = gc_rules_graph.dAllGraph;
    exports.dRule = gc_rules_graph.dRule;
}

// JavaScript

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global console,require,exports*/
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";


if(typeof(process) !== 'undefined') {
    var helpers = require("./graphspell/helpers.js");
if (typeof(require) !== 'undefined') {
} else if (typeof(require) !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}


class TestGrammarChecking {

    constructor (gce, spfTests="") {
        this.gce = gce;
        this.spfTests = spfTests;
        this._aRuleTested = new Set();
    }

    * testParse (bDebug=false) {
        const t0 = Date.now();
        let sURL;
        if(typeof(process) !== 'undefined') {
            sURL = (this.spfTests !== "") ? this.spfTests : "./"+this.gce.lang+"/tests_data.json";
        } else {
        let sURL = (this.spfTests !== "") ? this.spfTests : "resource://grammalecte/"+this.gce.lang+"/tests_data.json";
            sURL = (this.spfTests !== "") ? this.spfTests : "resource://grammalecte/"+this.gce.lang+"/tests_data.json";
        }
        const aData = JSON.parse(helpers.loadFile(sURL)).aData;
        let nInvalid = 0;
        let nTotal = 0;
        let sErrorText;
        let sSugg;
        let sExpectedErrors;
        let sTextToCheck;

// JavaScript

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global require,exports*/
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";


var text = {
    getParagraph: function* (sText, sSepParagraph = "\n") {
        // generator: returns paragraphs of text

import helpers


def build (sLang, dVars, spLangPack):
    "complementary build launched from make.py"
    createWebExtension(sLang, dVars)
    createThunderbirdExtension(sLang, dVars, spLangPack)
    createNodeJSPackage(sLang)


def createWebExtension (sLang, dVars):
    "create Web-extension"
    print("Building WebExtension")
    helpers.createCleanFolder("_build/webext/"+sLang)
    dir_util.copy_tree("gc_lang/"+sLang+"/webext/", "_build/webext/"+sLang)

            hZip.write("grammalecte-js/graphspell/"+sf, sAddPath+"grammalecte-js/graphspell/"+sf)
    for sf in os.listdir("grammalecte-js/graphspell/_dictionaries"):
        if not os.path.isdir("grammalecte-js/graphspell/_dictionaries/"+sf):
            hZip.write("grammalecte-js/graphspell/_dictionaries/"+sf, sAddPath+"grammalecte-js/graphspell/_dictionaries/"+sf)
    for sf in os.listdir(spLangPack):
        if not os.path.isdir(spLangPack+"/"+sf):
            hZip.write(spLangPack+"/"+sf, sAddPath+spLangPack+"/"+sf)


def createNodeJSPackage (sLang):
    helpers.createCleanFolder("_build/nodejs/"+sLang)
    dir_util.copy_tree("gc_lang/"+sLang+"/nodejs/", "_build/nodejs/"+sLang)
    dir_util.copy_tree("grammalecte-js", "_build/nodejs/"+sLang+"/core/grammalecte")

// Grammalecte - Conjugueur
// License: GPL 3

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global console,require,exports,self,browser*/
/* jslint esversion:6 */
/* global require, exports, console, self, browser, chrome, __dirname */

"use strict";

${map}


if(typeof(process) !== 'undefined') {
    var helpers = require("../graphspell/helpers.js");
if (typeof(require) !== 'undefined') {
} else if (typeof(require) !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}

var conj = {
    _lVtyp: [],
    _lTags: [],
    _dPatternConj: {},

        }
        return (this.dConj.get(":Q").get(":Q4")) ? this.dConj.get(":Q").get(":Q4") : this.dConj.get(":Q").get(":Q1");
    }
}


// Initialization
if(!conj.bInit && typeof(process) !== 'undefined') {
    // Work with nodejs
    conj.init(helpers.loadFile(__dirname+"/conj_data.json"));
if (!conj.bInit && typeof(browser) !== 'undefined') {
} else if (!conj.bInit && typeof(browser) !== 'undefined') {
    // WebExtension Standard (but not in Worker)
    conj.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/conj_data.json")));
} else if (!conj.bInit && typeof(chrome) !== 'undefined') {
    // WebExtension Chrome (but not in Worker)
    conj.init(helpers.loadFile(chrome.extension.getURL("grammalecte/fr/conj_data.json")));
} else if (!conj.bInit && typeof(require) !== 'undefined') {
    // Add-on SDK and Thunderbird
    conj.init(helpers.loadFile("resource://grammalecte/fr/conj_data.json"));
} else if (!conj.bInit && typeof(self) !== 'undefined' && typeof(self.port) !== 'undefined' && typeof(self.port.on) !== "undefined") {
} else if (!conj.bInit && typeof(self) !== 'undefined' && typeof(self.port) !== 'undefined' && typeof(self.port.on) !== 'undefined') {
    // used within Firefox content script (conjugation panel).
    // can’t load JSON from here, so we do it in ui.js and send it here.
    self.port.on("provideConjData", function (sJSONData) {
        conj.init(sJSONData);
    });    
    });
} else if (conj.bInit){
    console.log("Module conj déjà initialisé");
} else {
    //console.log("Module conj non initialisé");
}

//  JavaScript

/*
    Conjugation generator
    beta stage, unfinished, the root for a new way to generate flexions…
*/
//    Conjugation generator
//    beta stage, unfinished, the root for a new way to generate flexions…


/* jshint esversion:6, -W097 */
/* jslint esversion:6 */

"use strict";


var conj_generator = {

    conjugate: function (sVerb, sVerbTag="i_____a", bVarPpas=true) {

            [2,     "isses",        ":Sp:Sq:2s/*",      false],
            [2,     "isse",         ":Sp:3s/*",         false],
            [2,     "ît",           ":Sq:3s/*",         false],
            [2,     "is",           ":E:2s/*",          false],
            [2,     "issons",       ":E:1p/*",          false],
            [2,     "issez",        ":E:2p/*",          false]
        ],
        

        // premier groupe (bien plus irrégulier que prétendu)
        "V1": {
            // a
            // verbes en -er, -ger, -yer, -cer
            "er": [
                [2,      "er",        ":Y/*",               false],
                [2,      "ant",       ":P/*",               false],

//// Grammalecte - Compiled regular expressions
/*jslint esversion: 6*/
// Grammalecte - Compiled regular expressions

/* jshint esversion:6 */
/* jslint esversion:6 */


var cregex = {
    ///// Lemme
    _zLemma: new RegExp(">([a-zà-öø-ÿ0-9Ā-ʯ][a-zà-öø-ÿ0-9Ā-ʯ-]+)"),

    ///// Masculin / féminin / singulier / pluriel

//// GRAMMAR CHECKING ENGINE PLUGIN: Parsing functions for French language
/*jslint esversion: 6*/
// GRAMMAR CHECKING ENGINE PLUGIN: Parsing functions for French language

/* jshint esversion:6 */
/* jslint esversion:6 */

function g_morphVC (dToken, sPattern, sNegPattern="") {
    let nEnd = dToken["sValue"].lastIndexOf("-");
    if (dToken["sValue"].includes("-t-")) {
        nEnd = nEnd - 2;
    }
    return g_morph(dToken, sPattern, sNegPattern, 0, nEnd, false);

    if (s.length > 1 && s.length < 16 && s.slice(0, 1).gl_isLowerCase() && (!s.slice(1).gl_isLowerCase() || /[0-9]/.test(s))) {
        return true;
    }
    return false;
}


//// Exceptions
// Exceptions

const aREGULARPLURAL = new Set(["abricot", "amarante", "aubergine", "acajou", "anthracite", "brique", "caca", "café",
                                "carotte", "cerise", "chataigne", "corail", "citron", "crème", "grave", "groseille",
                                "jonquille", "marron", "olive", "pervenche", "prune", "sable"]);
const aSHOULDBEVERB = new Set(["aller", "manger"]);

//// GRAMMAR CHECKING ENGINE PLUGIN
// GRAMMAR CHECKING ENGINE PLUGIN
/*jslint esversion: 6*/

// Check date validity
// WARNING: when creating a Date, month must be between 0 and 11

// WARNING: when creating a Date, month must be between 0 and 11
/* jshint esversion:6 */
/* jslint esversion:6 */


const _lDay = ["dimanche", "lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi"];
const _dMonth = new Map ([
    ["janvier", 1], ["février", 2], ["mars", 3], ["avril", 4], ["mai", 5], ["juin", 6], ["juillet", 7],
    ["août", 8], ["aout", 8], ["septembre", 9], ["octobre", 10], ["novembre", 11], ["décembre", 12]
]);

//// GRAMMAR CHECKING ENGINE PLUGIN: Suggestion mechanisms
/*jslint esversion: 6*/
/*global require*/
// GRAMMAR CHECKING ENGINE PLUGIN: Suggestion mechanisms

/* jshint esversion:6 */
/* jslint esversion:6 */
/* global require */

if(typeof(process) !== 'undefined') {
    var conj = require("./conj.js");
    var mfsp = require("./mfsp.js");
    var phonet = require("./phonet.js");
if (typeof(require) !== 'undefined') {
} else if (typeof(require) !== 'undefined') {
    var conj = require("resource://grammalecte/fr/conj.js");
    var mfsp = require("resource://grammalecte/fr/mfsp.js");
    var phonet = require("resource://grammalecte/fr/phonet.js");
}


//// verbs

// Grammalecte - Lexicographe
// License: MPL 2

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global require,exports*/
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";

${string}
${map}

// Grammalecte

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global console,require,exports,browser*/
/* jslint esversion:6 */
/* global require, exports, console, browser,__dirname */

"use strict";


if(typeof(process) !== 'undefined') {
    var helpers = require("../graphspell/helpers.js");
if (typeof(require) !== 'undefined') {
} else if (typeof(require) !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}


var mfsp = {
    // list of affix codes
    _lTagMiscPlur: [],

            return "## erreur, code : " + sSfx + " ##";
        }
    }
};


// Initialization
if(!mfsp.bInit && typeof(process) !== 'undefined') {
    //Nodejs
    mfsp.init(helpers.loadFile(__dirname+"/mfsp_data.json"));
if (!mfsp.bInit && typeof(browser) !== 'undefined') {
} else if (!mfsp.bInit && typeof(browser) !== 'undefined') {
    // WebExtension
    mfsp.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/mfsp_data.json")));
} else if (!mfsp.bInit && typeof(require) !== 'undefined') {
    // Add-on SDK and Thunderbird
    mfsp.init(helpers.loadFile("resource://grammalecte/fr/mfsp_data.json"));
} else if (mfsp.bInit){
    console.log("Module mfsp déjà initialisé");

// Grammalecte - Suggestion phonétique

/* jshint esversion:6 */
/*jslint esversion: 6*/
/* jslint esversion:6 */
/* global __dirname */

if (typeof(require) !== 'undefined') {
if(typeof(process) !== 'undefined') {
    var helpers = require("../graphspell/helpers.js");
} else if (typeof(require)  !== 'undefined') {
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
}


var phonet = {
    _dWord: new Map(),
    _lSet: [],

        }
        return aSelect;
    }
};


// Initialization
if (!phonet.bInit && typeof(process) !== 'undefined') {
    //Nodejs
    phonet.init(helpers.loadFile(__dirname+"/phonet_data.json"));
if (!phonet.bInit && typeof(browser) !== 'undefined') {
} else if (!phonet.bInit && typeof(browser) !== 'undefined') {
    // WebExtension
    phonet.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/phonet_data.json")));
} else if (!phonet.bInit && typeof(require) !== 'undefined') {
    // Add-on SDK and Thunderbird
    phonet.init(helpers.loadFile("resource://grammalecte/fr/phonet_data.json"));
} else if (phonet.bInit){
    console.log("Module phonet déjà initialisé");

// Grammalecte - text formatter

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global exports*/
/* jslint esversion:6 */
/* global exports, console */

"use strict";

${map}
//!${map}


// Latin letters: http://unicode-table.com/fr/
// 0-9
// A-Z
// a-z
// À-Ö     00C0-00D6   (upper case)

    "erase_non_breaking_hyphens": [ [/­/g, ""] ],
    //// typographic signs
    "ts_apostrophe":              [ [/\b([ldnjmtscç])['´‘′`](?=[a-zA-Zà-ö0-9À-Öø-ÿØ-ßĀ-ʯ])/ig, "$1’"],
                                    [/\b(qu|jusqu|lorsqu|puisqu|quoiqu|quelqu|presqu|entr|aujourd|prud)['´‘′`]/ig, "$1’"] ],
    "ts_ellipsis":                [ [/\.\.\./g, "…"],
                                    [/…\.\./g, "……"],
                                    [/…\.(?!\.)/g, "…"] ],
    "ts_n_dash_middle":           [ [/ [-—] /g, " – "], 
    "ts_n_dash_middle":           [ [/ [-—] /g, " – "],
                                    [/ [-—],/g, " –,"] ],
    "ts_m_dash_middle":           [ [/ [-–] /g, " — "],
                                    [/ [-–],/g, " —,"] ],
    "ts_n_dash_start":            [ [/^[-—][  ]/gm, "– "],
                                    [/^– /gm, "– "],
                                    [/^[-–—](?=[a-zA-Zà-ö0-9À-Öø-ÿØ-ßĀ-ʯ.…])/gm, "– "] ],
    "ts_m_dash_start":            [ [/^[-–][  ]/gm, "— "],

    ["etc", true],
    ["missing_hyphens", true],
    ["ma_word", true],
    ["ma_1letter_lowercase", false],
    ["ma_1letter_uppercase", false]
]);

const dTFOptions = dTFDefaultOptions.gl_shallowCopy();


class TextFormatter {

    constructor () {
    constructor (bDebug=false) {
        this.sLang = "fr";
        this.bDebug = bDebug;
        //don't change this in external ;)
        this.dOptions = dTFDefaultOptions.gl_shallowCopy();
    }

    formatText (sText, dOpt=null) {
        if (dOpt !== null) {
            dTFOptions.gl_updateOnlyExistingKeys(dOpt);
            this.dOptions.gl_updateOnlyExistingKeys(dOpt);
        }
        for (let [sOptName, bVal] of dTFOptions) {
            if (bVal && oReplTable.has(sOptName)) {
        for (let [sOptName, bVal] of this.dOptions) {
            //console.log(oReplTable);
            if (bVal && oReplTable[sOptName]) {
                for (let [zRgx, sRep] of oReplTable[sOptName]) {
                    sText = sText.replace(zRgx, sRep);
                }
            }
        }
        return sText;
    }

    formatTextCount (sText, dOpt=null) {
        let nCount = 0;
        if (dOpt !== null) {
            this.dOptions.gl_updateOnlyExistingKeys(dOpt);
        }
        for (let [sOptName, bVal] of this.dOptions) {
            if (bVal && oReplTable[sOptName]) {
                for (let [zRgx, sRep] of oReplTable[sOptName]) {
                    nCount += (sText.match(zRgx) || []).length;
                    sText = sText.replace(zRgx, sRep);
                }
            }
        }
        return [sText, nCount];
    }

    formatTextRule (sText, sRuleName) {
        if (oReplTable[sRuleName]) {
            for (let [zRgx, sRep] of oReplTable[sRuleName]) {
                sText = sText.replace(zRgx, sRep);
            }
        } else if (this.bDebug){
            console.log("# Error. TF: there is no option “" + sRuleName+ "”.");
        }
        return sText;
    }

    formatTextRuleCount (sText, sRuleName) {
        let nCount = 0;
        if (oReplTable[sRuleName]) {
            for (let [zRgx, sRep] of oReplTable[sRuleName]) {
                nCount += (sText.match(zRgx) || []).length;
                sText = sText.replace(zRgx, sRep);
            }
        } else if (this.bDebug){
            console.log("# Error. TF: there is no option “" + sRuleName+ "”.");
        }
        return [sText, nCount];
    }

    removeHyphenAtEndOfParagraphs (sText) {
        sText = sText.replace(/-[  ]*\n/gm, "");
        return sText;
    }

    removeHyphenAtEndOfParagraphsCount (sText) {
        let nCount = (sText.match(/-[  ]*\n/gm) || []).length;
        sText = sText.replace(/-[  ]*\n/gm, "");
        return [sText, nCount];
    }

    mergeContiguousParagraphs (sText) {
        sText = sText.replace(/^[  ]+$/gm, ""); // clear empty paragraphs
        let s = "";
        for (let sParagraph of this.getParagraph(sText)) {
            if (sParagraph === "") {
                s += "\n";
            } else {
                s += sParagraph + " ";
            }
        }
        s = s.replace(/  +/gm, " ").replace(/ $/gm, "");
        return s;
    }

    mergeContiguousParagraphsCount (sText) {
        let nCount = 0;
        sText = sText.replace(/^[  ]+$/gm, ""); // clear empty paragraphs
        let s = "";
        for (let sParagraph of this.getParagraph(sText)) {
            if (sParagraph === "") {
                s += "\n";
            } else {
                s += sParagraph + " ";
                nCount += 1;
            }
        }
        s = s.replace(/  +/gm, " ").replace(/ $/gm, "");
        return [s, nCount];
    }

    * getParagraph (sText, sSep="\n") {
        // generator: returns paragraphs of text
        let iStart = 0;
        let iEnd = 0;
        while ((iEnd = sText.indexOf(sSep, iStart)) !== -1) {
            yield sText.slice(iStart, iEnd);
            iStart = iEnd + 1;
        }
        yield sText.slice(iStart);
    }

    getDefaultOptions () {
        //we return a copy to make sure they are no modification in external
        return dTFDefaultOptions;
        return dTFDefaultOptions.gl_shallowCopy();
    }

    getOptions () {
        //we return a copy to make sure they are no modification in external
        return this.dOptions.gl_shallowCopy();
    }

    setOptions (dOpt=null) {
        if (dOpt !== null) {
            this.dOptions.gl_updateOnlyExistingKeys(dOpt);
        } else if (this.bDebug){
            console.log("# Error. TF: no option to change.");
        }
    }
}


if (typeof(exports) !== 'undefined') {
    exports.TextFormatter = TextFormatter;
    exports.oReplTable = oReplTable;
}

@node gramma-cli.js %*

#! /usr/bin/env node
// -*- js -*-

// Gramma-Cli
// Grammalect client pour node

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, console */

/*
Doc :
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Destructuring_assignment
https://stackoverflow.com/questions/41058569/what-is-the-difference-between-const-and-const-in-javascript
*/

const argCmd = require("../lib/minimist.js")(process.argv.slice(2));
const { performance } = require("perf_hooks");

//Initialisation des messages
const msgStart = "\x1b[31mBienvenue sur Grammalecte pour NodeJS!!!\x1b[0m\n";
const msgPrompt = "\x1b[36mGrammaJS\x1b[33m>\x1b[0m ";
const msgSuite = "\x1b[33m…\x1b[0m ";
const msgEnd = "\x1b[31m\x1b[5m\x1b[5mBye bye!\x1b[0m";

var repPreference = {
    json: false,
    perf: false
};

var sBufferConsole = "";
var sCmdToExec = "";
var sText = "";

var cmdAction = {
    help: {
        short: "",
        arg: "",
        description: "Affiche les informations que vous lisez ;)",
        execute: ""
    },
    perf: {
        short: "",
        arg: "on/off",
        description: "Permet d’afficher le temps d’exécution des commandes.",
        execute: ""
    },
    json: {
        short: "",
        arg: "on/off",
        description: "Réponse en format format json.",
        execute: ""
    },
    exit: {
        short: "",
        arg: "",
        description: "Client interactif: Permet de le quitter.",
        execute: ""
    },
    text: {
        short: "",
        arg: "texte",
        description: "Client / Server: Définir un texte pour plusieurs actions.",
        execute: ""
    },
    format: {
        short: "",
        arg: "texte",
        description: "Permet de mettre en forme le texte.",
        execute: "formatText"
    },
    check: {
        short: "",
        arg: "texte",
        description: "Vérifie la grammaire et l’orthographe d'un texte.",
        execute: "verifParagraph"
    },
    lexique: {
        short: "",
        arg: "texte",
        description: "Affiche le lexique du texte.",
        execute: "lexique"
    },
    spell: {
        short: "",
        arg: "mot",
        description: "Vérifie l’existence d’un mot.",
        execute: "spell"
    },
    suggest: {
        short: "",
        arg: "mot",
        description: "Suggestion des graphies proches d’un mot.",
        execute: "suggest"
    },
    morph: {
        short: "",
        arg: "mot",
        description: "Affiche les informations pour un mot.",
        execute: "morph"
    },
    lemma: {
        short: "",
        arg: "mot",
        description: "Donne le lemme d’un mot.",
        execute: "lemma"
    },
    gceoption: {
        short: "",
        arg: "+/-name",
        description: "Définit les options à utiliser par le correcteur grammatical.",
        execute: ""
    },
    tfoption: {
        short: "",
        arg: "+/-name",
        description: "Définit les options à utiliser par le formateur de texte.",
        execute: ""
    }
};

var cmdOne = ["json", "perf", "help", "exit"];
var cmdMulti = ["text", "format", "check", "lexique", "spell", "suggest", "morph", "lemma"];

var cmdAll = [...cmdOne, ...cmdMulti];

function getArgVal(aArg, lArgOk) {
    for (let eArgOk of lArgOk) {
        if (typeof aArg[eArgOk] !== "undefined") {
            return aArg[eArgOk];
        }
    }
    return false;
}

function getArg(aArg, lArgOk) {
    for (let eArgOk of lArgOk) {
        if (typeof aArg[eArgOk] !== "undefined") {
            return true;
        }
    }
    return false;
}

function toBool(aStr) {
    return aStr === "true" || aStr === "on";
}

function isBool(aStr) {
    if (typeof aStr === "boolean" || typeof aStr === "undefined") {
        return true;
    }
    aStr = aStr.toLowerCase();
    return aStr === "true" || aStr === "on" || aStr === "false" || aStr === "off" || aStr === "";
}

function toTitle(aStr) {
    return aStr.charAt(0).toUpperCase() + aStr.slice(1);
}

function repToText(oRep) {
    //console.log(oRep);
    let repText = "";
    for (const action of ["json", "perf", "gceoption", "tfoption"]) {
        if (action in oRep) {
            repText += toTitle(action) + " " + oRep[action];
        }
    }

    for (const action of ["morph", "lemma"]) {
        if (action in oRep) {
            for (const toAff of oRep[action]) {
                if (toAff.text == "NoText") {
                    repText += "\n" + toTitle(action) + ": Pas de texte à vérifier.";
                } else {
                    if (toAff.reponse.length == 0) {
                        repText += "\nAuncun " + toTitle(action) + " existant pour: «" + toAff.text + "»";
                    } else {
                        let ascii = "├";
                        let numRep = 0;
                        repText += "\n" + toTitle(action) + " possible de: «" + toAff.text + "»";
                        for (let reponse of toAff.reponse) {
                            numRep++;
                            if (numRep == toAff.reponse.length) {
                                ascii = "└";
                            }
                            repText += "\n " + ascii + " " + reponse;
                        }
                    }
                    repText += affPerf(toAff.time);
                }
            }
        }
    }

    if ("spell" in oRep) {
        for (const toAff of oRep.spell) {
            if (toAff.text == "NoText") {
                repText += "\nSpell: Pas de texte à vérifier.";
            } else {
                repText += "\nLe mot «" + toAff.text + "» " + (toAff.reponse ? "existe" : "innexistant");
                repText += affPerf(toAff.time);
            }
        }
    }

    if ("suggest" in oRep) {
        for (const toAff of oRep.suggest) {
            if (toAff.text == "NoText") {
                repText += "\nSuggest : Pas de texte à vérifier.";
            } else {
                //let numgroup = 0;
                if (toAff.reponse.length == 0) {
                    repText += "\nAucune suggestion possible pour: «" + toAff.text + "»";
                } else {
                    repText += "\nSuggestion possible de: «" + toAff.text + "»";
                    let ascii = "├";
                    let numRep = 0;
                    for (let reponse of toAff.reponse) {
                        numRep++;
                        if (numRep == toAff.reponse.length) {
                            ascii = "└";
                        }
                        repText += "\n " + ascii + " " + reponse;
                    }
                }
                repText += affPerf(toAff.time);
            }
        }
    }

    if ("format" in oRep) {
        for (const toAff of oRep.format) {
            if (toAff.text == "NoText") {
                repText += "\nPas de texte à formatter.";
            } else {
                repText += "\nMise en forme:\n" + toAff.reponse;
                repText += affPerf(toAff.time);
            }
        }
    }

    if ("lexique" in oRep) {
        for (const toAff of oRep.lexique) {
            if (toAff.text == "NoText") {
                repText += "\nLexique: Pas de texte à vérifier.";
            } else {
                repText += "\nLexique:";

                let ascii1, ascii1a, numRep1, ascii2, numRep2, replength;

                ascii1 = "├";
                ascii1a = "│";
                numRep1 = 0;

                replength = toAff.reponse.length;
                for (let reponse of toAff.reponse) {
                    numRep1++;
                    if (numRep1 == replength) {
                        ascii1 = "└";
                        ascii1a = " ";
                    }
                    repText += "\n  " + ascii1 + " " + reponse.sValue;
                    let ascii = "├";
                    let numRep = 0;
                    for (let label of reponse.aLabel) {
                        numRep++;
                        if (numRep == reponse.aLabel.length) {
                            ascii = "└";
                        }
                        repText += "\n  " + ascii1a + " " + ascii + " " + label.trim();
                    }
                }
                repText += affPerf(toAff.time);
            }
        }
    }

    if ("check" in oRep) {
        for (const toAff of oRep.check) {
            if (toAff.text == "NoText") {
                repText += "\nCheck: Pas de texte à vérifier.";
            } else {
                let ascii1, ascii1a, numRep1, ascii2, numRep2, replength;

                ascii1 = "├";
                ascii1a = "│";
                numRep1 = 0;
                replength = Object.keys(toAff.reponse.lGrammarErrors).length;
                if (replength == 0) {
                    repText += "\nPas de faute de grammaire";
                } else {
                    repText += "\nFaute(s) de grammaire";
                    for (let gramma of toAff.reponse.lGrammarErrors) {
                        numRep1++;
                        if (numRep1 == replength) {
                            ascii1 = "└";
                            ascii1a = " ";
                        }
                        repText += "\n " + ascii1 + " " + gramma.nStart + "->" + gramma.nEnd + " " + gramma.sMessage;
                        ascii2 = "├";
                        numRep2 = 0;
                        for (let suggestion of gramma.aSuggestions) {
                            numRep2++;
                            if (numRep2 == gramma.aSuggestions.length) {
                                ascii2 = "└";
                            }
                            repText += "\n " + ascii1a + "  " + ascii2 + ' "' + suggestion + '"';
                        }
                    }
                }

                ascii1 = "├";
                ascii1a = "│";
                numRep1 = 0;
                replength = Object.keys(toAff.reponse.lSpellingErrors).length;
                if (replength == 0) {
                    repText += "\nPas de faute d'orthographe";
                } else {
                    repText += "\nFaute(s) d'orthographe";
                    for (let ortho of toAff.reponse.lSpellingErrors) {
                        numRep1++;
                        if (numRep1 == replength) {
                            ascii1 = "└";
                            ascii1a = " ";
                        }
                        repText += "\n " + ascii1 + " " + ortho.nStart + "->" + ortho.nEnd + " " + ortho.sValue;
                        ascii2 = "├";
                        numRep2 = 0;
                        for (let suggestion of ortho.aSuggestions) {
                            numRep2++;
                            if (numRep2 == ortho.aSuggestions.length) {
                                ascii2 = "└";
                            }
                            repText += "\n " + ascii1a + "  " + ascii2 + ' "' + suggestion + '"';
                        }
                    }
                }
                repText += affPerf(toAff.time);
            }
        }
    }

    if ("help" in oRep) {
        let colorNum = 31;
        for (const action of oRep.help) {
            //Uniquement pour le fun on met de la couleur ;)
            if (action.indexOf("===") > -1) {
                console.log("\x1b[" + colorNum + "m" + action + "\x1b[0m");
                colorNum = colorNum + 2;
            } else {
                console.log(action);
            }
        }
    }

    return repText.trim("\n");
}

function affPerf(aTime) {
    if (aTime == "NA") {
        return "";
    }
    return "\nExécuté en: " + aTime + " ms";
}

function actionGramma(repPreference, action, aAction) {
    let tStart, tEnd;
    let tmpRep = {
        text: "",
        reponse: "",
        time: "NA"
    };

    if (!isBool(aAction) && aAction !== "") {
        tmpRep.text = aAction;
        sText = aAction;
    } else if (!isBool(sText)) {
        //Utilisation du dernier texte connu
        tmpRep.text = sText;
    } else {
        tmpRep.text = "NoText";
    }

    if (repPreference.perf) {
        tStart = performance.now();
    }

    tmpRep.reponse = oGrammarChecker[cmdAction[action].execute](tmpRep.text);

    if (repPreference.perf) {
        tEnd = performance.now();
        tmpRep["time"] = (Math.round((tEnd - tStart) * 1000) / 1000).toString();
    }

    return tmpRep;
}

function actionToExec(aArg) {
    let repAction = {};

    if (!isBool(aArg.text)) {
        sText = aArg.text;
    }

    for (const action of ["json", "perf"]) {
        if (getArg(aArg, [action])) {
            repPreference[action] = getArgVal(aArg, [action]);
            repAction[action] = repPreference[action] ? "ON" : "OFF";
        }
    }

    for (const action of ["gceoption", "tfoption"]) {
        if (getArg(aArg, [action])) {
            let sFonction = action == "gceoption" ? "GceOption" : "TfOption";
            let sOpt = sText.split(" ");
            if (sOpt[0] == "reset") {
                oGrammarChecker["reset" + sFonction + "s"]();
                repAction[action] = "reset";
            } else {
                for (const optAction of sOpt) {
                    let bOptVal = optAction[0] == "+" ? true : false;
                    let sOptName = optAction.slice(1, optAction.length);
                    oGrammarChecker["set" + sFonction](sOptName, bOptVal);
                    repAction[action] = sText;
                }
            }
        }
    }

    for (const action in aArg) {
        if (cmdAction[action] && cmdAction[action].execute !== "") {
            //console.log(aArg, aArg[action], !isBool(aArg[action]), !isBool(repAction.text));
            if (!repAction[action]) {
                repAction[action] = [];
            }

            if (typeof aArg[action] === "object") {
                for (const valAction of aArg[action]) {
                    tmpRep = actionGramma(repPreference, action, valAction);
                    repAction[action].push(tmpRep);
                }
            } else {
                tmpRep = actionGramma(repPreference, action, aArg[action]);
                repAction[action].push(tmpRep);
            }
        }
    }

    if (getArg(aArg, ["help"])) {
        repAction["help"] = [];

        repAction["help"].push("================================== Aide: ==================================");
        repAction["help"].push("");
        repAction["help"].push("Il y a trois modes de fonctionnement: client / client intératif / serveur.");

        repAction["help"].push(" * le client intéractif: «gramma-cli -i».");
        repAction["help"].push(' * pour le client exemple: «gramma-cli --command "mot/texte"».');
        repAction["help"].push(" * le serveur se lance avec la commande «gramma-cli --server --port 8085».");

        repAction["help"].push("");
        repAction["help"].push("========================= Les commandes/arguments: ========================");
        repAction["help"].push("");
        for (const action in cmdAction) {
            repAction["help"].push(action.padEnd(10, " ") + ": " + cmdAction[action].arg.padEnd(8, " ") + ": " + cmdAction[action].description);
        }
        repAction["help"].push("");
        repAction["help"].push("================================== Note: ==================================");
        repAction["help"].push("");
        repAction["help"].push("En mode client: les arguments sont de la forme «--argument» !");
        repAction["help"].push("En mode client intéractif: pour les commandes concernant un texte, vous");
        repAction["help"].push("  pouvez taper la commande puis Entrée (pour saisir le texte) pour ");
        repAction["help"].push('  terminer la saisie du texte et exécuter la commande taper /"commande"');
    }

    if (repPreference.json) {
        return JSON.stringify(repAction);
    } else {
        return repToText(repAction);
    }
}

function argToExec(aCommand, aText, rl, resetCmd = true) {
    let execAct = {};
    aCommand = aCommand.toLowerCase();

    if (!isBool(aText)) {
        execAct["text"] = aText;
        execAct[aCommand] = true;
    } else {
        execAct[aCommand] = toBool(aText);
    }

    console.log(actionToExec(execAct));
    //sBufferConsole = "";
    if (resetCmd) {
        sCmdToExec = "";
    }

    if (typeof rl !== "undefined") {
        rl.setPrompt(msgPrompt);
    }
}

function completer(line) {
    var hits = cmdAll.filter(function(c) {
        if (c.indexOf(line) == 0) {
            return c;
        }
    });
    return [hits && hits.length ? hits : cmdAll, line];
}

if (process.argv.length <= 2) {
    console.log(actionToExec({ help: true }));
} else {
    //var GrammarChecker = require("./api.js");
    //console.log(module.paths);
    var GrammarChecker = require("grammalecte");
    var oGrammarChecker = new GrammarChecker.GrammarChecker(["Grammalecte", "Graphspell", "TextFormatter", "Lexicographer", "Tokenizer"], "fr");

    if (argCmd.server) {
        var http = require("http");
        var url = require("url");
        var querystring = require("querystring");

        var collectRequestData = function(aRequest, aResponse, callback) {
            let sBody = "";
            aRequest.on("data", chunk => {
                sBody += chunk.toString();
            });
            aRequest.on("end", () => {
                let oParams = querystring.parse(sBody);
                //console.log(oParams /*, page*/);
                callback(querystring.parse(sBody), aResponse);
            });
        };

        var reponseRequest = function(aParms, aResponse) {
            aResponse.setHeader("access-control-allow-origin", "*");
            aResponse.writeHead(200, { "Content-Type": "application/json" });
            aParms["json"] = true; //Forcage de la réponse en json
            aResponse.write(actionToExec(aParms));
            aResponse.end();
        };

        var server = http.createServer(function(aRequest, aResponse) {
            var sPage = url.parse(aRequest.url).pathname;
            if (sPage !== "/") {
                //favicon.ico
                aResponse.writeHead(404, { "Content-Type": "text/plain" });
                aResponse.write("Error 404");
                aResponse.end();
            } else {
                if (aRequest.method === "POST") {
                    collectRequestData(aRequest, aResponse, reponseRequest);
                } else {
                    let oParams = querystring.parse(url.parse(aRequest.url).query);
                    reponseRequest(oParams, aResponse);
                }
            }
        });
        server.listen(argCmd.port || 2212);
        console.log("Server started on http://127.0.0.1:" + (argCmd.port || 2212) + "/");
    } else if (getArg(argCmd, ["i", "interactive"])) {
        process.stdin.setEncoding("utf8");

        const readline = require("readline");
        const rl = readline.createInterface({
            crlfDelay: Infinity,
            input: process.stdin,
            output: process.stdout,
            completer: completer,
            prompt: msgPrompt
        });

        //console.log( process.stdin.isTTY );
        console.log(msgStart);
        rl.prompt();
        rl.on("line", sBuffer => {
            //process.stdout.write
            if (sBuffer == "exit") {
                console.log(msgEnd);
                process.exit(0);
            }

            let lg = sBuffer.toLowerCase().trim();
            let bSpace = lg.indexOf(" ") > -1;
            if (!bSpace) {
                if (cmdOne.indexOf(lg) > -1) {
                    argToExec(lg, sBuffer, rl, true);
                } else if (cmdAll.indexOf(lg) > -1) {
                    sBufferConsole = "";
                    sCmdToExec = lg;
                    //Prompt simple pour distinguer que c"est une suite d"une commande
                    rl.setPrompt(msgSuite);
                } else if (lg.slice(1) == sCmdToExec) {
                    argToExec(sCmdToExec, sBufferConsole, rl, true);
                } else if (cmdAll.indexOf(lg.slice(0, lg.length - 1)) > -1) {
                    argToExec(lg.slice(0, lg.length - 1), sBufferConsole, rl, true);
                } else if (lg == "") {
                    sBufferConsole += "\n";
                }
            } else if (sCmdToExec == "") {
                let regRep = /(.*?) (.*)/gm.exec(sBuffer);
                //console.log(regRep.length,sBuffer);
                if (regRep && regRep.length == 3) {
                    argToExec(regRep[1], regRep[2]);
                }
            } else {
                sBufferConsole += sBuffer + "\n";
            }

            rl.prompt();
        }).on("close", () => {
            console.log(msgEnd);
            process.exit(0);
        });
    } else {
        if (
            typeof argCmd.text !== "object" &&
            typeof argCmd.json !== "object" &&
            typeof argCmd.perf !== "object" &&
            typeof argCmd.gceoption !== "object" &&
            typeof argCmd.tfoption !== "object"
        ) {
            console.log(actionToExec(argCmd));
        } else {
            console.log("Votre demmande est confuse.");
        }
    }
}

json false
perf true
spell salut
suggest salut
morph salut
lemma salut
gceoption -typo
check
salut comment,il vas???bienss...et tu!  "salut commentss il vas???"
/check
gceoption +typo
check/
#lexique/
tfoption +nnbsp_before_punctuation -ts_ellipsis
format/
tfoption reset
format/

module.exports = function (args, opts) {
    if (!opts) opts = {};
    
    var flags = { bools : {}, strings : {}, unknownFn: null };

    if (typeof opts['unknown'] === 'function') {
        flags.unknownFn = opts['unknown'];
    }

    if (typeof opts['boolean'] === 'boolean' && opts['boolean']) {
      flags.allBools = true;
    } else {
      [].concat(opts['boolean']).filter(Boolean).forEach(function (key) {
          flags.bools[key] = true;
      });
    }
    
    var aliases = {};
    Object.keys(opts.alias || {}).forEach(function (key) {
        aliases[key] = [].concat(opts.alias[key]);
        aliases[key].forEach(function (x) {
            aliases[x] = [key].concat(aliases[key].filter(function (y) {
                return x !== y;
            }));
        });
    });

    [].concat(opts.string).filter(Boolean).forEach(function (key) {
        flags.strings[key] = true;
        if (aliases[key]) {
            flags.strings[aliases[key]] = true;
        }
     });

    var defaults = opts['default'] || {};
    
    var argv = { _ : [] };
    Object.keys(flags.bools).forEach(function (key) {
        setArg(key, defaults[key] === undefined ? false : defaults[key]);
    });
    
    var notFlags = [];

    if (args.indexOf('--') !== -1) {
        notFlags = args.slice(args.indexOf('--')+1);
        args = args.slice(0, args.indexOf('--'));
    }

    function argDefined(key, arg) {
        return (flags.allBools && /^--[^=]+$/.test(arg)) ||
            flags.strings[key] || flags.bools[key] || aliases[key];
    }

    function setArg (key, val, arg) {
        if (arg && flags.unknownFn && !argDefined(key, arg)) {
            if (flags.unknownFn(arg) === false) return;
        }

        var value = !flags.strings[key] && isNumber(val)
            ? Number(val) : val
        ;
        setKey(argv, key.split('.'), value);
        
        (aliases[key] || []).forEach(function (x) {
            setKey(argv, x.split('.'), value);
        });
    }

    function setKey (obj, keys, value) {
        var o = obj;
        keys.slice(0,-1).forEach(function (key) {
            if (o[key] === undefined) o[key] = {};
            o = o[key];
        });

        var key = keys[keys.length - 1];
        if (o[key] === undefined || flags.bools[key] || typeof o[key] === 'boolean') {
            o[key] = value;
        }
        else if (Array.isArray(o[key])) {
            o[key].push(value);
        }
        else {
            o[key] = [ o[key], value ];
        }
    }
    
    function aliasIsBoolean(key) {
      return aliases[key].some(function (x) {
          return flags.bools[x];
      });
    }

    for (var i = 0; i < args.length; i++) {
        var arg = args[i];
        
        if (/^--.+=/.test(arg)) {
            // Using [\s\S] instead of . because js doesn't support the
            // 'dotall' regex modifier. See:
            // http://stackoverflow.com/a/1068308/13216
            var m = arg.match(/^--([^=]+)=([\s\S]*)$/);
            var key = m[1];
            var value = m[2];
            if (flags.bools[key]) {
                value = value !== 'false';
            }
            setArg(key, value, arg);
        }
        else if (/^--no-.+/.test(arg)) {
            var key = arg.match(/^--no-(.+)/)[1];
            setArg(key, false, arg);
        }
        else if (/^--.+/.test(arg)) {
            var key = arg.match(/^--(.+)/)[1];
            var next = args[i + 1];
            if (next !== undefined && !/^-/.test(next)
            && !flags.bools[key]
            && !flags.allBools
            && (aliases[key] ? !aliasIsBoolean(key) : true)) {
                setArg(key, next, arg);
                i++;
            }
            else if (/^(true|false)$/.test(next)) {
                setArg(key, next === 'true', arg);
                i++;
            }
            else {
                setArg(key, flags.strings[key] ? '' : true, arg);
            }
        }
        else if (/^-[^-]+/.test(arg)) {
            var letters = arg.slice(1,-1).split('');
            
            var broken = false;
            for (var j = 0; j < letters.length; j++) {
                var next = arg.slice(j+2);
                
                if (next === '-') {
                    setArg(letters[j], next, arg)
                    continue;
                }
                
                if (/[A-Za-z]/.test(letters[j]) && /=/.test(next)) {
                    setArg(letters[j], next.split('=')[1], arg);
                    broken = true;
                    break;
                }
                
                if (/[A-Za-z]/.test(letters[j])
                && /-?\d+(\.\d*)?(e-?\d+)?$/.test(next)) {
                    setArg(letters[j], next, arg);
                    broken = true;
                    break;
                }
                
                if (letters[j+1] && letters[j+1].match(/\W/)) {
                    setArg(letters[j], arg.slice(j+2), arg);
                    broken = true;
                    break;
                }
                else {
                    setArg(letters[j], flags.strings[letters[j]] ? '' : true, arg);
                }
            }
            
            var key = arg.slice(-1)[0];
            if (!broken && key !== '-') {
                if (args[i+1] && !/^(-|--)[^-]/.test(args[i+1])
                && !flags.bools[key]
                && (aliases[key] ? !aliasIsBoolean(key) : true)) {
                    setArg(key, args[i+1], arg);
                    i++;
                }
                else if (args[i+1] && /true|false/.test(args[i+1])) {
                    setArg(key, args[i+1] === 'true', arg);
                    i++;
                }
                else {
                    setArg(key, flags.strings[key] ? '' : true, arg);
                }
            }
        }
        else {
            if (!flags.unknownFn || flags.unknownFn(arg) !== false) {
                argv._.push(
                    flags.strings['_'] || !isNumber(arg) ? arg : Number(arg)
                );
            }
            if (opts.stopEarly) {
                argv._.push.apply(argv._, args.slice(i + 1));
                break;
            }
        }
    }
    
    Object.keys(defaults).forEach(function (key) {
        if (!hasKey(argv, key.split('.'))) {
            setKey(argv, key.split('.'), defaults[key]);
            
            (aliases[key] || []).forEach(function (x) {
                setKey(argv, x.split('.'), defaults[key]);
            });
        }
    });
    
    if (opts['--']) {
        argv['--'] = new Array();
        notFlags.forEach(function(key) {
            argv['--'].push(key);
        });
    }
    else {
        notFlags.forEach(function(key) {
            argv._.push(key);
        });
    }

    return argv;
};

function hasKey (obj, keys) {
    var o = obj;
    keys.slice(0,-1).forEach(function (key) {
        o = (o[key] || {});
    });

    var key = keys[keys.length - 1];
    return key in o;
}

function isNumber (x) {
    if (typeof x === 'number') return true;
    if (/^0x[0-9a-f]+$/i.test(x)) return true;
    return /^[-+]?(?:\d+(?:\.\d*)?|\.\d+)(e[-+]?\d+)?$/.test(x);
}

{
  "name": "grammalecte-cli",
  "version": "1.0.0",
  "description": "Grammalecte command line interface",
  "keywords": [
    "cli",
    "french",
    "grammar",
    "proofreader"
  ],
  "author": "Sébastien GRAVIER, Olivier R.",
  "license": "GPL-3.0-or-later",
  "homepage": "https://www.dicollecte.org/",
  "bin": {
    "gramma-cli": "bin/gramma-cli.js"
  },
  "engines": {
    "node": ">=9.0.0"
  },
  "scripts": {
    "test": "node bin/gramma-cli.js"
  },
  "dependencies": {
    "grammalecte": "~1.0.0"
  },
  "files": [
    "bin",
    "lib",
    "data",
    "readme.md"
  ]
}

# Client/Serveur de Grammalecte pour NodeJS

## Informations

Il y a trois modes de fonctionnement : client / client interactif / serveur.

* Client interactif: `gramma-cli -i`.
* Client: `gramma-cli --command \"mot/texte\"`.
* Serveur: lancé avec la commande `gramma-cli --server --port NumPort`.

## Installation

> npm install grammalecte-cli -g

## Commandes

| Commande  | Argument | Description                                                   |
| --------- | -------- | ------------------------------------------------------------- |
| help      |          | Affiche les informations que vous lisez ;)                    |
| perf      | on/off   | Permet d’afficher le temps d’exécution des commandes.         |
| json      | on/off   | Réponse en format json.                                       |
| exit      |          | Client interactif : permet de le quitter.                     |
| text      | texte    | Client / Server: Définir un texte pour plusieurs actions.     |
| format    | texte    | Permet de mettre en forme le texte.                           |
| check     | texte    | Vérifie la grammaire et l’orthographe d'un texte.             |
| lexique   | texte    | Affiche le lexique du texte.                                  |
| spell     | mot      | Vérifie l’existence d'un mot.                                 |
| suggest   | mot      | Suggestion des orthographes possible d’un mot.                |
| morph     | mot      | Affiche les informations pour un mot.                         |
| lemma     | mot      | Donne le lemme d’un mot.                                      |
| gceoption | +/-name  | Définit les options à utiliser par le correcteur grammatical. |
| tfoption  | +/-name  | Définit les options à utiliser par le formateur de texte.     |

## Client interactif

Le mode interactif est un mode question/réponse. Pour le lancer vous devez saisir `gramma-cli -i`.

Exemple pour les vérifications portant sur un mot:

```
CMD> gramma-cli -i
Bienvenu sur Grammalecte pour NodeJS!!!
GrammaJS> suggest salit
Suggestion possible de: salit
 ├ salit
 ├ salît
 ├ salie
 ├ salis
 ├ salir
 ├ salin
 ├ sali
 ├ salait
 ├ salut
 └ salât
GrammaJS> exit
```

Exemple pour les vérifications portant sur un texte:

```
CMD> gramma-cli -i
Bienvenue sur Grammalecte pour NodeJS!!!
GrammaJS> format
> salut,les copains!!!
> vous allez bien ?
> /format
Mise en forme:
salut, les copains!!!
vous allez bien ?
GrammaJS> exit
```

## Client

Exemple simple:

```
CMD> gramma-cli --spell saluti
Le mot saluti innexistant

CMD>
```

Exemple faisant plusieurs actions:

```
CMD> gramma-cli --lemma --morph --suggest --text salut
Morph possible de: salut
 └ >salut/:N:m:s/*
Lemme possible de: salut
 └ salut
Suggestion possible de: salut
 ├ salut
 ├ salit
 ├ salue
 ├ salua
 ├ saluai
 ├ saluts
 ├ salué
 ├ saluât
 ├ salât
 └ salît

CMD>
```

Note :

Il ne peut pas y avoir plusieurs fois les arguments text, json, perf, gceoption et tfoption.

Il est cependant possible de vérifier par exemple plusieurs morph en faisant: `gramma-cli --morph mot1 --morph mot2`

## Serveur

Le serveur supporte les requêtes POST et GET...

Par défaut le port d’écoute est le 2212, pour le changer utilisez l’argument `--port` lors du lancement.

## Les fichiers

* bin/gramma-cli.bat  : Fait juste un appel `node gramma-cli.js argument(s)`
* bin/gramma-cli.js   : Le code principal pour la console
* data/script.gramma  : Exemple de script pour faire des vérifications automatiques
  * (sous widows) `type script.gramma | gramma-cli -i`
  * (sous linux)  `cat script.gramma  | gramma-cli -i`
* lib/minimist.js     : Une librairie pour simplifier la gestion des arguments
* package.json        : Fichier d’information pour npm
* readme.md           : Le fichier que vous lisez (ou pas) actuellement ;)

## Utilisation d'une librairie (incluse)

* [Minimist](https://github.com/substack/minimist) => Simplify parser argument

/*
    ! Grammalecte, grammar checker !
    API pour faciliter l'utilisation de Grammalecte.
*/

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console */

"use strict";

class GrammarChecker {

    constructor(aInit, sLangCode = "fr", sContext = "Javascript") {
        this.sLangCode = sLangCode;
        this.sContext = sContext;

        //Importation des fichiers nécessaire
        this.sPathRoot = __dirname + "/grammalecte";
        this._helpers = require(this.sPathRoot + "/graphspell/helpers.js");

        this.isInit = {
            Grammalecte: false,
            Graphspell: false,
            Tokenizer: false,
            TextFormatter: false,
            Lexicographer: false
        };

        if (aInit){
            this.load(aInit);
        }
    }

    //Auto-chargement avec dépendence
    load(aInit = ["Grammalecte", "Graphspell", "TextFormatter", "Lexicographer", "Tokenizer"]){
        //aInit permet de charger que certain composant
        // => évite de charger toutes données si par exemple on a besoin que du lexigraphe
        // => sorte de gestionnaire de dépendence (peut être amélioré)
        this.isInit = {};
        if ( aInit.indexOf("Grammalecte") !== false ){
            //console.log('init Grammalecte');
            this._oGce = require(this.sPathRoot + "/fr/gc_engine.js");
            this._oGce.load(this.sContext);
            this.isInit.Grammalecte = true;
            this.oSpellChecker = this._oGce.getSpellChecker();
            this.isInit.Graphspell = true;
            this.oTokenizer = this.oSpellChecker.getTokenizer();
            this.isInit.Tokenizer = true;
        }

        if ( !this.isInit.Graphspell && (aInit.indexOf("Graphspell") !== false || aInit.indexOf("Lexicographer") !== false)){
            //console.log('init Graphspell');
            this._SpellChecker = require(this.sPathRoot + "/graphspell/spellchecker.js");
            this.oSpellChecker = new this._SpellChecker.SpellChecker(this.sLangCode, this.sPathRoot + "/graphspell/_dictionaries");
            this.isInit.Graphspell = true;
            this.oTokenizer = this.oSpellChecker.getTokenizer();
            this.isInit.Tokenizer = true;
        }

        if ( !this.isInit.Tokenizer && aInit.indexOf("Tokenizer") !== false ){
            //console.log('init Tokenizer');
            this._Tokenizer = require(this.sPathRoot + "/graphspell/tokenizer.js");
            this.oTokenizer = new this._Tokenizer.Tokenizer(this.sLangCode);
            this.isInit.Tokenizer = true;
        }

        if ( aInit.indexOf("TextFormatter") !== false ){
            //console.log('init TextFormatter');
            this._oText = require(this.sPathRoot + "/fr/textformatter.js");
            this.oTextFormatter = new this._oText.TextFormatter();
            this.isInit.TextFormatter = true;
        }

        if ( aInit.indexOf("Lexicographer") !== false ){
            //console.log('init Lexicographer');
            this._oLex = require(this.sPathRoot + "/fr/lexicographe.js");
            this.oLexicographer = new this._oLex.Lexicographe(
                this.oSpellChecker,
                this.oTokenizer,
                this._helpers.loadFile(this.sPathRoot + "/fr/locutions_data.json")
            );
            this.isInit.Lexicographer = true;
        }
    }

    //Fonctions concernant: Grammalecte
    getGrammalecte(){
        if (!this.isInit.Grammalecte) {
            this.load(["Grammalecte"]);
        }
        return this._oGce;
    }

    gramma(sText){
        if (!this.isInit.Grammalecte) {
            this.load(["Grammalecte"]);
        }
        return Array.from(this._oGce.parse(sText, this.sLangCode));
    }

    getGceOptions () {
        if (!this.isInit.Grammalecte) {
            this.load(["Grammalecte"]);
        }
        return this._helpers.mapToObject(this._oGce.getOptions());
    }

    getGceDefaultOptions () {
        if (!this.isInit.Grammalecte) {
            this.load(["Grammalecte"]);
        }
        return this._helpers.mapToObject(this._oGce.getDefaultOptions());
    }

    setGceOptions (dOptions) {
        if (!this.isInit.Grammalecte) {
            this.load(["Grammalecte"]);
        }
        if (!(dOptions instanceof Map)) {
            dOptions = this._helpers.objectToMap(dOptions);
        }
        this._oGce.setOptions(dOptions);
        return this._helpers.mapToObject(this._oGce.getOptions());
    }

    setGceOption (sOptName, bValue) {
        if (!this.isInit.Grammalecte) {
            this.load(["Grammalecte"]);
        }
        if (sOptName) {
            this._oGce.setOption(sOptName, bValue);
            return this._helpers.mapToObject(this._oGce.getOptions());
        }
        return false;
    }

    resetGceOptions () {
        if (!this.isInit.Grammalecte) {
            this.load(["Grammalecte"]);
        }
        this._oGce.resetOptions();
        return this._helpers.mapToObject(this._oGce.getOptions());
    }

    //Fonctions concernant: Graphspell
    getGraphspell(){
        if (!this.isInit.Graphspell) {
            this.load(["Graphspell"]);
        }
        return this.oSpellChecker;
    }

    spellParagraph(sText, bSuggest = true){
        if (!this.isInit.Graphspell) {
            this.load(["Graphspell"]);
        }
        if (bSuggest){
            let lError = this.oSpellChecker.parseParagraph(sText);
            for (let token of lError) {
                token.aSuggestions = this.suggest(token.sValue);
            }
            return lError;
        } else {
            return this.oSpellChecker.parseParagraph(sText);
        }
    }

    spell(sWord){
        if (!this.isInit.Graphspell) {
            this.load(["Graphspell"]);
        }
        return this.oSpellChecker.isValid(sWord);
    }

    suggest(sWord, nbLimit = 10, bMerge = true){
        if (!this.isInit.Graphspell) {
            this.load(["Graphspell"]);
        }
        let lSuggest = this.oSpellChecker.suggest(sWord, nbLimit);
        if (bMerge){
            let lSuggestRep = [];
            for (let lSuggestTmp of lSuggest) {
                for (let word of lSuggestTmp) {
                    lSuggestRep.push(word);
                }
            }
            return lSuggestRep;
        } else {
            return Array.from(lSuggest);
        }

    }

    lemma(sWord){
        if (!this.isInit.Graphspell) {
            this.load(["Graphspell"]);
        }
        return this.oSpellChecker.getLemma(sWord);
    }

    morph(sWord){
        if (!this.isInit.Graphspell) {
            this.load(["Graphspell"]);
        }
        return this.oSpellChecker.getMorph(sWord);
    }

    //Fonctions concernant: Lexicographer
    getLexicographer(){
        if (!this.isInit.Lexicographer) {
            this.load(["Lexicographer"]);
        }
        return this.oLexicographer;
    }

    lexique(sText){
        if (!this.isInit.Lexicographer) {
            this.load(["Lexicographer"]);
        }
        return this.oLexicographer.getListOfTokensReduc(sText);
    }

    //Fonctions concernant: TextFormatter
    getTextFormatter(){
        if (!this.isInit.TextFormatter) {
            this.load(["TextFormatter"]);
        }
        return this.oTextFormatter;
    }

    formatText(sText){
        if (!this.isInit.TextFormatter) {
            this.load(["TextFormatter"]);
        }
        return this.oTextFormatter.formatText(sText);
    }

    setTfOptions(dOptions) {
        if (!this.isInit.TextFormatter) {
            this.load(["TextFormatter"]);
        }
        this.oTextFormatter.setOptions(dOptions);
        return this._helpers.mapToObject(this.oTextFormatter.getOptions());
    }

    setTfOption(sOptName, bValue) {
        if (!this.isInit.TextFormatter) {
            this.load(["TextFormatter"]);
        }
        if (sOptName) {
            let optionsTF = this.oTextFormatter.getOptions();
            optionsTF.set(sOptName, bValue);
            return this._helpers.mapToObject(this.oTextFormatter.getOptions());
        }
        return false;
    }

    resetTfOptions() {
        if (!this.isInit.TextFormatter) {
            this.load(["TextFormatter"]);
        }
        let optionsTF = this.oTextFormatter.getDefaultOptions();
        this.oTextFormatter.setOptions(optionsTF);
        return this._helpers.mapToObject(this.oTextFormatter.getOptions());
    }

    //fonctions concernant plussieurs parties
    verifParagraph(sText, bSuggest = true){
        if (!this.isInit.Grammalecte || !this.isInit.Graphspell) {
            this.load(["Grammalecte"]);
        }
        return {
            lGrammarErrors: Array.from(this._oGce.parse(sText, this.sLangCode)),
            lSpellingErrors: this.spellParagraph(sText, bSuggest)
        };
    }

}

if (typeof exports !== "undefined") {
    exports.GrammarChecker = GrammarChecker;
}

{
  "name": "grammalecte",
  "version": "1.0.0",
  "description": "Grammalecte is a grammar proofreader",
  "keywords": [
    "french",
    "grammar",
    "proofreader"
  ],
  "author": "Olivier R.",
  "license": "GPL-3.0-or-later",
  "homepage": "https://www.dicollecte.org/",
  "main": "api.js",
  "engines": {
    "node": ">=9.0.0"
  },
  "scripts": {
    "test": "echo \"Error: no test specified\""
  }
}

# Grammalecte pour NodeJS

## Informations

Grammalecte est un correcteur grammatical open source dédié à la langue française.
Site Internet : https://www.dicollecte.org/

## Installation

Dans votre répertoire de développement :

> npm install grammalecte

## Les fichiers

* grammalecte/*       : Tout le contenu de Grammalecte pour JavaScript
* api.js              : Une API pour simplifier l’utilisation de Grammalecte
* package.json        : Fichier d’information pour npm
* readme.md           : Le fichier que vous lisez (ou pas) actuellement ;)

# Note pour le dévellepement pour NodeJS

## Commande pour définir l’utilisation d’un paquetage local

```
cd core
npm link
cd ..
cd cli
npm link grammalecte
npm install --global
cd ..
```

## Commande désinstaller le paquetage local et son utilisation

```
npm rm grammalecte --global
cd cli
npm unlink grammalecte
npm rm grammalecte-cli --global
cd ..
```

// JavaScript
// Text formatter

"use strict";


class GrammalecteTextFormatter extends GrammalectePanel {

    constructor (...args) {
        super(...args);
        this.xTFNode = this._createTextFormatter();
        this.xPanelContent.appendChild(this.xTFNode);
        this.xTextArea = null;

        this.TextFormatter = new TextFormatter();
        this.formatText = this.TextFormatter.formatTextRuleCount;
        this.removeHyphenAtEndOfParagraphs = this.TextFormatter.removeHyphenAtEndOfParagraphsCount;
        this.mergeContiguousParagraphs = this.TextFormatter.mergeContiguousParagraphsCount;
        this.getParagraph = this.TextFormatter.getParagraph;
    }

    _createTextFormatter () {
        let xTFNode = document.createElement("div");
        try {
            // Options
            let xOptions = oGrammalecte.createNode("div", {id: "grammalecte_tf_options"});

    _createOrdinalOptions () {
        let xLine = oGrammalecte.createNode("div", {className: "grammalecte_tf_blockopt grammalecte_tf_underline"});
        xLine.appendChild(this._createOption("o_ordinals_no_exponant", true, "Ordinaux (15e, XXIe…)"));
        xLine.appendChild(this._createOption("o_ordinals_exponant", true, "e → ᵉ"));
        xLine.appendChild(oGrammalecte.createNode("div", {id: "res_"+"o_ordinals_no_exponant", className: "grammalecte_tf_result", textContent: "·"}));
        return xLine;
    }
    

    /*
        Actions
    */
    start (xNode) {
        if (xNode !== null && xNode.tagName == "TEXTAREA") {
            this.xTextArea = xNode;
            if (bChrome) {

        try {
            const t0 = Date.now();
            //window.setCursor("wait"); // change pointer
            this.resetProgressBar();
            let sText = this.xTextArea.value.normalize("NFC");
            document.getElementById('grammalecte_tf_progressbar').max = 7;
            let n1 = 0, n2 = 0, n3 = 0, n4 = 0, n5 = 0, n6 = 0, n7 = 0;
            

            // Restructuration
            if (this.isSelected("o_group_struct")) {
                if (this.isSelected("o_remove_hyphens_at_end_of_paragraphs")) {
                    [sText, n1] = this.removeHyphenAtEndOfParagraphs(sText);
                    document.getElementById('res_o_remove_hyphens_at_end_of_paragraphs').textContent = n1;
                }
                if (this.isSelected("o_merge_contiguous_paragraphs")) {

            this.xTextArea.value = sText;
        }
        catch (e) {
            showError(e);
        }
    }

    formatText (sText, sOptName) {
        let nCount = 0;
        try {
            if (!oReplTable.hasOwnProperty(sOptName)) {
                console.log("# Error. TF: there is no option “" + sOptName+ "”.");
                return [sText, nCount];
            }
            for (let [zRgx, sRep] of oReplTable[sOptName]) {
                nCount += (sText.match(zRgx) || []).length;
                sText = sText.replace(zRgx, sRep);
            }
        }
        catch (e) {
            showError(e);
        }
        return [sText, nCount];
    }

    removeHyphenAtEndOfParagraphs (sText) {
        let nCount = (sText.match(/-[  ]*\n/gm) || []).length;
        sText = sText.replace(/-[  ]*\n/gm, "");
        return [sText, nCount];
    }

    mergeContiguousParagraphs (sText) {
        let nCount = 0;
        sText = sText.replace(/^[  ]+$/gm, ""); // clear empty paragraphs
        let s = "";
        for (let sParagraph of this.getParagraph(sText)) {
            if (sParagraph === "") {
                s += "\n";
            } else {
                s += sParagraph + " ";
                nCount += 1;
            }
        }
        s = s.replace(/  +/gm, " ").replace(/ $/gm, "");
        return [s, nCount];
    }

    * getParagraph (sText) {
        // generator: returns paragraphs of text
        let iStart = 0;
        let iEnd = 0;
        while ((iEnd = sText.indexOf("\n", iStart)) !== -1) {
            yield sText.slice(iStart, iEnd);
            iStart = iEnd + 1;
        }
        yield sText.slice(iStart);
    }

    getTimeRes (n) {
        // returns duration in seconds as string
        if (n < 10) {
            return n.toFixed(3).toString() + " s";
        }
        if (n < 100) {
            return n.toFixed(2).toString() + " s";

// list of similar chars
// useful for suggestion mechanism

/* jshint esversion:6 */
/* jslint esversion:6 */

${map}


var char_player = {

    _xTransCharsForSpelling: new Map([
        ['ſ', 's'],  ['ffi', 'ffi'],  ['ffl', 'ffl'],  ['ff', 'ff'],  ['ſt', 'ft'],  ['fi', 'fi'],  ['fl', 'fl'],  ['st', 'st']

    _xTransCharsForSimplification: new Map([
        ['à', 'a'],  ['é', 'e'],  ['î', 'i'],  ['ô', 'o'],  ['û', 'u'],  ['ÿ', 'i'],  ['y', 'i'],
        ['â', 'a'],  ['è', 'e'],  ['ï', 'i'],  ['ö', 'o'],  ['ù', 'u'],  ['ŷ', 'i'],
        ['ä', 'a'],  ['ê', 'e'],  ['í', 'i'],  ['ó', 'o'],  ['ü', 'u'],  ['ý', 'i'],
        ['á', 'a'],  ['ë', 'e'],  ['ì', 'i'],  ['ò', 'o'],  ['ú', 'u'],  ['ỳ', 'i'],
        ['ā', 'a'],  ['ē', 'e'],  ['ī', 'i'],  ['ō', 'o'],  ['ū', 'u'],  ['ȳ', 'i'],
        ['ç', 'c'],  ['ñ', 'n'],  ['k', 'q'],  ['w', 'v'],
        ['œ', 'oe'], ['æ', 'ae'], 
        ['œ', 'oe'], ['æ', 'ae'],
        ['ſ', 's'],  ['ffi', 'ffi'],  ['ffl', 'ffl'],  ['ff', 'ff'],  ['ſt', 'ft'],  ['fi', 'fi'],  ['fl', 'fl'],  ['st', 'st']
    ]),

    simplifyWord: function (sWord) {
        // word simplication before calculating distance between words
        sWord = sWord.toLowerCase();
        sWord = [...sWord].map(c => this._xTransCharsForSimplification.gl_get(c, c)).join('');

        ["Ë", "EeÉéÈèÊêËëĒēŒœ"],

        ["f", "fF"],
        ["F", "Ff"],

        ["g", "gGjJĵĴ"],
        ["G", "GgJjĴĵ"],
        

        ["h", "hH"],
        ["H", "Hh"],

        ["i", "iIîÎïÏyYíÍìÌīĪÿŸ"],
        ["I", "IiÎîÏïYyÍíÌìĪīŸÿ"],
        ["î", "iIîÎïÏyYíÍìÌīĪÿŸ"],
        ["Î", "IiÎîÏïYyÍíÌìĪīŸÿ"],

    },

    // Other functions
    filterSugg: function (aSugg) {
        return aSugg.filter((sSugg) => { return !sSugg.endsWith("è") && !sSugg.endsWith("È"); });
    }

}
};


if (typeof(exports) !== 'undefined') {
    exports._xTransCharsForSpelling = char_player._xTransCharsForSpelling;
    exports.spellingNormalization = char_player.spellingNormalization;
    exports._xTransCharsForSimplification = char_player._xTransCharsForSimplification;
    exports.simplifyWord = char_player.simplifyWord;

// JavaScript

// FSA DICTIONARY BUILDER
//
// by Olivier R.
// License: MPL 2
//
// This tool encodes lexicon into an indexable binary dictionary
// Input files MUST be encoded in UTF-8.

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, helpers */

"use strict";


if (typeof(require) !== 'undefined') {
if(typeof(process) !== 'undefined') {
    var str_transform = require("./str_transform.js");
} else if (typeof(require) !== 'undefined') {
    var str_transform = require("resource://grammalecte/graphspell/str_transform.js");
}


${map}


class DAWG {
    /*  DIRECT ACYCLIC WORD GRAPH
        This code is inspired from Steve Hanov’s DAWG, 2011. (http://stevehanov.ca/blog/index.php?id=115)

        let lWord = [];
        for (let [sFlex, iAff, iTag] of lEntry) {
            let lTemp = [];
            for (let c of sFlex) {
                lTemp.push(dChar.get(c));
            }
            lTemp.push(iAff+nChar);
            lTemp.push(iTag+nChar+nAff)
            lTemp.push(iTag+nChar+nAff);
            lWord.push(lTemp);
        }
        lEntry.length = 0; // clear the array

        // Dictionary of arc values occurrency, to sort arcs of each node
        let lKeyVal = [];
        for (let c of dChar.keys()) { lKeyVal.push([dChar.get(c), dCharOccur.get(c)]); }

        this.nNextId += 1;
        return this.nNextId-1;
    },

    reset: function () {
        this.nNextId = 0;
    }
}
};


class DawgNode {

    constructor () {
        this.i = oNodeCounter.getId();
        this.final = false;

    convValueToHexString (nVal, nByte) {
        // nVal: value to convert, nByte: number of bytes
        let sHexVal = nVal.toString(16); // conversion to hexadecimal string
        //console.log(`value: ${nVal} in ${nByte} bytes`);
        if (sHexVal.length < (nByte*2)) {
            return "0".repeat((nByte*2) - sHexVal.length) + sHexVal;
        } else if (sHexVal.length == (nByte*2)) {
            return sHexVal
            return sHexVal;
        } else {
            throw "Conversion to byte string: value bigger than allowed.";
        }
    }
}

// HELPERS

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global console,require,exports,XMLHttpRequest*/
/* jslint esversion:6 */
/* global require, exports, console, XMLHttpRequest */

"use strict";


var helpers = {

    inspect: function (o) {
        let sMsg = "__inspect__: " + typeof o;
        for (let sParam in o) {
            sMsg += "\n" + sParam + ": " + o.sParam;
        }
        sMsg += "\n" + JSON.stringify(o) + "\n__end__";
        console.log(sMsg);
    },

    loadFile: function (spf) {
        // load ressources in workers (suggested by Mozilla extensions reviewers)
        // for more options have a look here: https://gist.github.com/Noitidart/ec1e6b9a593ec7e3efed
        // if not in workers, use sdk/data.load() instead
        try {
            if(typeof(process) !== 'undefined') {
                //console.log('loadFile(disque): ' + spf);
                let fs = require("fs");
                return fs.readFileSync(spf, "utf8");
            } else {
            console.log("loadFile: " + spf);
            let xRequest;
            xRequest = new XMLHttpRequest();
            xRequest.open('GET', spf, false); // 3rd arg is false for synchronous, sync is acceptable in workers
            xRequest.overrideMimeType('text/json');
            xRequest.send();
            return xRequest.responseText;
                console.log("loadFile: " + spf);
                let xRequest;
                xRequest = new XMLHttpRequest();
                xRequest.open('GET', spf, false); // 3rd arg is false for synchronous, sync is acceptable in workers
                xRequest.overrideMimeType('text/json');
                xRequest.send();
                return xRequest.responseText;
            }
        }
        catch (e) {
            console.error(e);
            return null;
        }
    },

//// IBDAWG
/*jslint esversion: 6*/
/*global console,require,exports*/
// IBDAWG

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, __dirname */

"use strict";


if (typeof(require) !== 'undefined') {
if(typeof(process) !== 'undefined') {
    var str_transform = require("./str_transform.js");
    var helpers = require("./helpers.js");
    var char_player = require("./char_player.js");
} else if (typeof(require) !== 'undefined') {
    var str_transform = require("resource://grammalecte/graphspell/str_transform.js");
    var helpers = require("resource://grammalecte/graphspell/helpers.js");
    var char_player = require("resource://grammalecte/graphspell/char_player.js");
}


// Don’t remove <string>. Necessary in TB.
${string}
${map}
${set}

    // INDEXABLE BINARY DIRECT ACYCLIC WORD GRAPH

    constructor (param1, sPath="") {
        // param1 can be a filename or a object with all the necessary data.
        try {
            let oData = null;
            if (typeof(param1) == "string") {
                let sURL;
                if(typeof(process) !== 'undefined') {
                    sURL = (sPath !== "") ? sPath + "/" + param1 : __dirname + "/_dictionaries/"+param1;
                } else {
                let sURL = (sPath !== "") ? sPath + "/" + param1 : "resource://grammalecte/graphspell/_dictionaries/"+param1;
                    sURL = (sPath !== "") ? sPath + "/" + param1 : "resource://grammalecte/graphspell/_dictionaries/"+param1;
                }
                oData = JSON.parse(helpers.loadFile(sURL));
            } else {
                oData = param1;
            }
            Object.assign(this, oData);
        }
        catch (e) {

            "l2grams": this.l2grams
        };
        return oJSON;
    }

    isValidToken (sToken) {
        // checks if sToken is valid (if there is hyphens in sToken, sToken is split, each part is checked)
        sToken = char_player.spellingNormalization(sToken)
        sToken = char_player.spellingNormalization(sToken);
        if (this.isValid(sToken)) {
            return true;
        }
        if (sToken.includes("-")) {
            if (sToken.gl_count("-") > 4) {
                return true;
            }

            }
        }
        return Boolean(this._convBytesToInteger(this.byDic.slice(iAddr, iAddr+this.nBytesArc)) & this._finalNodeMask);
    }

    getMorph (sWord) {
        // retrieves morphologies list, different casing allowed
        sWord = char_player.spellingNormalization(sWord)
        sWord = char_player.spellingNormalization(sWord);
        let l = this.morph(sWord);
        if (sWord[0].gl_isUpperCase()) {
            l.push(...this.morph(sWord.toLowerCase()));
            if (sWord.gl_isUpperCase() && sWord.length > 1) {
                l.push(...this.morph(sWord.gl_toCapitalize()));
            }
        }
        return l;
    }

    suggest (sWord, nSuggLimit=10) {
        // returns a array of suggestions for <sWord>
        //const t0 = Date.now();
        sWord = char_player.spellingNormalization(sWord)
        //console.time("Suggestions for " + sWord);
        sWord = char_player.spellingNormalization(sWord);
        let sPfx = "";
        let sSfx = "";
        [sPfx, sWord, sSfx] = char_player.cut(sWord);
        let nMaxSwitch = Math.max(Math.floor(sWord.length / 3), 1);
        let nMaxDel = Math.floor(sWord.length / 5);
        let nMaxHardRepl = Math.max(Math.floor((sWord.length - 5) / 4), 1);
        let nMaxJump = Math.max(Math.floor(sWord.length / 4), 1);
        let oSuggResult = new SuggResult(sWord);
        this._suggest(oSuggResult, sWord, nMaxSwitch, nMaxDel, nMaxHardRepl, nMaxJump);
        let aSugg = oSuggResult.getSuggestions(nSuggLimit);
        if (sSfx || sPfx) {
            // we add what we removed
            return aSugg.map( (sSugg) => { return sPfx + sSugg + sSfx } );
            return aSugg.map( (sSugg) => { return sPfx + sSugg + sSfx; } );
        }
        //const t1 = Date.now();
        //console.log("Suggestions for " + sWord + " in " + ((t1-t0)/1000).toString() + " s");
        //console.timeEnd("Suggestions for " + sWord);
        return aSugg;
    }

    _suggest (oSuggResult, sRemain, nMaxSwitch=0, nMaxDel=0, nMaxHardRepl=0, nMaxJump=0, nDist=0, nDeep=0, iAddr=0, sNewWord="", bAvoidLoop=false) {
        // returns a set of suggestions
        // recursive function
        if (sRemain == "") {

                }
                iAddr = iEndArcAddr + this.nBytesNodeAddress;
            }
        }
    }

    * _getArcs1 (iAddr) {
        "generator: return all arcs at <iAddr> as tuples of (nVal, iAddr)"
        // generator: return all arcs at <iAddr> as tuples of (nVal, iAddr)
        while (true) {
            let iEndArcAddr = iAddr+this.nBytesArc;
            let nRawArc = this._convBytesToInteger(this.byDic.slice(iAddr, iEndArcAddr));
            yield [nRawArc & this._arcMask, this._convBytesToInteger(this.byDic.slice(iEndArcAddr, iEndArcAddr+this.nBytesNodeAddress))];
            if (nRawArc & this._lastArcMask) {
                break;
            }

// Spellchecker
// Wrapper for the IBDAWG class.
// Useful to check several dictionaries at once.

// To avoid iterating over a pile of dictionaries, it is assumed that 3 are enough:
// - the main dictionary, bundled with the package
// - the extended dictionary
// - the community dictionary, added by an organization
// - the personal dictionary, created by the user for its own convenience

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global require, exports, console, IBDAWG, Tokenizer */

"use strict";


if (typeof(require) !== 'undefined') {
if(typeof(process) !== 'undefined') {
    var ibdawg = require("./ibdawg.js");
    var tokenizer = require("./tokenizer.js");
} else if (typeof(require) !== 'undefined') {
    var ibdawg = require("resource://grammalecte/graphspell/ibdawg.js");
    var tokenizer = require("resource://grammalecte/graphspell/tokenizer.js");
}


${map}


const dDefaultDictionaries = new Map([
    ["fr", "fr-allvars.json"],
    ["en", "en.json"]

            }
        }
        catch (e) {
            let sfDictionary = (typeof(dictionary) == "string") ? dictionary : dictionary.sLangName + "/" + dictionary.sFileName;
            if (bNecessary) {
                throw "Error: <" + sfDictionary + "> not loaded. " + e.message;
            }
            console.log("Error: <" + sfDictionary + "> not loaded.")
            console.log("Error: <" + sfDictionary + "> not loaded.");
            console.log(e.message);
            return null;
        }
    }

    loadTokenizer () {
        if (typeof(tokenizer) !== 'undefined') {

        // checks if sWord is valid (different casing tested if the first letter is a capital)
        if (this.oMainDic.isValid(sWord)) {
            return true;
        }
        if (this.bExtendedDic && this.oExtendedDic.isValid(sWord)) {
            return true;
        }
        if (this.bCommunityDic && this.oCommunityDic.isValid(sToken)) {
        if (this.bCommunityDic && this.oCommunityDic.isValid(sWord)) {
            return true;
        }
        if (this.bPersonalDic && this.oPersonalDic.isValid(sWord)) {
            return true;
        }
        return false;
    }

    lookup (sWord) {
        // checks if sWord is in dictionary as is (strict verification)
        if (this.oMainDic.lookup(sWord)) {
            return true;
        }
        if (this.bExtendedDic && this.oExtendedDic.lookup(sWord)) {
            return true;
        }
        if (this.bCommunityDic && this.oCommunityDic.lookup(sToken)) {
        if (this.bCommunityDic && this.oCommunityDic.lookup(sWord)) {
            return true;
        }
        if (this.bPersonalDic && this.oPersonalDic.lookup(sWord)) {
            return true;
        }
        return false;
    }

        if (this.bPersonalDic) {
            yield this.oPersonalDic.suggest(sWord, nSuggLimit);
        }
    }

    * select (sFlexPattern="", sTagsPattern="") {
        // generator: returns all entries which flexion fits <sFlexPattern> and morphology fits <sTagsPattern>
        yield* this.oMainDic.select(sFlexPattern, sTagsPattern)
        yield* this.oMainDic.select(sFlexPattern, sTagsPattern);
        if (this.bExtendedDic) {
            yield* this.oExtendedDic.select(sFlexPattern, sTagsPattern);
        }
        if (this.bCommunityDic) {
            yield* this.oCommunityDic.select(sFlexPattern, sTagsPattern);
        }
        if (this.bPersonalDic) {

//// STRING TRANSFORMATION
/*jslint esversion: 6*/
// STRING TRANSFORMATION

/* jshint esversion:6, -W097 */
/* jslint esversion:6 */
/* global exports, console */

"use strict";


// Note: 48 is the ASCII code for "0"

var str_transform = {

// JavaScript
// Very simple tokenizer

/* jshint esversion:6, -W097 */
/*jslint esversion: 6*/
/*global require,exports*/
/* jslint esversion:6 */
/*global require, exports, console*/

"use strict";


const aTkzPatterns = {
    // All regexps must start with ^.
    "default":

        while (sText) {
            let iCut = 1;
            for (let [zRegex, sType] of this.aRules) {
                if (sType !== "SPACE"  ||  bWithSpaces) {
                    try {
                        if ((m = zRegex.exec(sText)) !== null) {
                            iToken += 1;
                            yield { "i": iToken, "sType": sType, "sValue": m[0], "nStart": iNext, "nEnd": iNext + m[0].length }
                            yield { "i": iToken, "sType": sType, "sValue": m[0], "nStart": iNext, "nEnd": iNext + m[0].length };
                            iCut = m[0].length;
                            break;
                        }
                    }
                    catch (e) {
                        console.error(e);
                    }

        if not os.path.isdir("grammalecte/graphspell/_dictionaries/"+sf):
            hZip.write("grammalecte/graphspell/_dictionaries/"+sf, sAddPath+"grammalecte/graphspell/_dictionaries/"+sf)
    for sf in os.listdir(spLangPack):
        if not os.path.isdir(spLangPack+"/"+sf):
            hZip.write(spLangPack+"/"+sf, sAddPath+spLangPack+"/"+sf)


def create (sLang, xConfig, bInstallOXT, bJavaScript):
def create (sLang, xConfig, bInstallOXT, bJavaScript, bUseCache):
    "make Grammalecte for project <sLang>"
    oNow = datetime.datetime.now()
    print("============== MAKE GRAMMALECTE [{0}] at {1.hour:>2} h {1.minute:>2} min {1.second:>2} s ==============".format(sLang, oNow))

    #### READ CONFIGURATION
    print("> read configuration...")
    spLang = "gc_lang/" + sLang

    dVars = xConfig._sections['args']
    dVars['locales'] = dVars["locales"].replace("_", "-")
    dVars['loc'] = str(dict([ [s, [s[0:2], s[3:5], ""]] for s in dVars["locales"].split(" ") ]))

    ## COMPILE RULES
    dResult = compile_rules.make(spLang, dVars['lang'], bJavaScript)
    dResult = compile_rules.make(spLang, dVars['lang'], bUseCache)
    dVars.update(dResult)

    ## READ GRAMMAR CHECKER PLUGINS
    print("PYTHON:")
    print("+ Plugins: ", end="")
    sCodePlugins = ""
    for sf in os.listdir(spLang+"/modules"):

def main ():
    "build Grammalecte with requested options"
    print("Python: " + sys.version)
    xParser = argparse.ArgumentParser()
    xParser.add_argument("lang", type=str, nargs='+', help="lang project to generate (name of folder in /lang)")
    xParser.add_argument("-uc", "--use_cache", help="use data cache instead of rebuilding rules", action="store_true")
    xParser.add_argument("-b", "--build_data", help="launch build_data.py (part 1 and 2)", action="store_true")
    xParser.add_argument("-bb", "--build_data_before", help="launch build_data.py (only part 1: before dictionary building)", action="store_true")
    xParser.add_argument("-ba", "--build_data_after", help="launch build_data.py (only part 2: before dictionary building)", action="store_true")
    xParser.add_argument("-d", "--dict", help="generate FSA dictionary", action="store_true")
    xParser.add_argument("-t", "--tests", help="run unit tests", action="store_true")
    xParser.add_argument("-p", "--perf", help="run performance tests", action="store_true")
    xParser.add_argument("-pm", "--perf_memo", help="run performance tests and store results in perf_memo.txt", action="store_true")

            if databuild and xArgs.build_data_after:
                databuild.after('gc_lang/'+sLang, dVars, xArgs.javascript)

            # copy dictionaries from Graphspell
            copyGraphspellDictionaries(dVars, xArgs.javascript, xArgs.add_extended_dictionary, xArgs.add_community_dictionary, xArgs.add_personal_dictionary)

            # make
            sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, )
            sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, xArgs.use_cache)

            # tests
            if xArgs.tests or xArgs.perf or xArgs.perf_memo:
                print("> Running tests")
                try:
                    tests = importlib.import_module("grammalecte."+sLang+".tests")
                    print(tests.__file__)