Index: compile_rules.py ================================================================== --- compile_rules.py +++ compile_rules.py @@ -1,13 +1,15 @@ """ Grammalecte: compile rules """ import re +import os import traceback import json import colorsys +import time import compile_rules_js_convert as jsconv import compile_rules_graph as crg @@ -417,11 +419,11 @@ dOptPriority = {} for sLine in lOptionLines: sLine = sLine.strip() if sLine.startswith("OPTGROUP/"): m = re.match("OPTGROUP/([a-z0-9]+):(.+)$", sLine) - lStructOpt.append( (m.group(1), list(map(str.split, m.group(2).split(",")))) ) + lStructOpt.append( [m.group(1), list(map(str.split, m.group(2).split(",")))] ) elif sLine.startswith("OPTSOFTWARE:"): lOpt = [ [s, {}] for s in sLine[12:].strip().split() ] # don’t use tuples (s, {}), because unknown to JS elif sLine.startswith("OPT/"): m = re.match("OPT/([a-z0-9]+):(.+)$", sLine) for i, sOpt in enumerate(m.group(2).split()): @@ -463,13 +465,22 @@ def printBookmark (nLevel, sComment, nLine): "print bookmark within the rules file" print(" {:>6}: {}".format(nLine, " " * nLevel + sComment)) -def make (spLang, sLang, bJavaScript): +def make (spLang, sLang, bUseCache=False): "compile rules, returns a dictionary of values" # for clarity purpose, don’t create any file here + + if bUseCache and os.path.isfile("_build/data_cache.json"): + print("> don’t rebuild rules, use cache...") + sJSON = open("_build/data_cache.json", "r", encoding="utf-8").read() + dCacheVars = json.loads(sJSON) + print(" build made at: " + time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(dCacheVars.get("fBuildTime", 0)))) + return dCacheVars + + fBuildTime = time.time() print("> read rules file...") try: lRules = open(spLang + "/rules.grx", 'r', encoding="utf-8").readlines() except: @@ -610,20 +621,25 @@ displayStats(lParagraphRules, lSentenceRules) print("Unnamed rules: " + str(nRULEWITHOUTNAME)) - dVars = { "callables": sPyCallables, - "callablesJS": sJSCallables, - "gctests": sGCTests, - "gctestsJS": sGCTestsJS, - "paragraph_rules": mergeRulesByOption(lParagraphRules), - "sentence_rules": mergeRulesByOption(lSentenceRules), - "paragraph_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lParagraphRulesJS)), - "sentence_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lSentenceRulesJS)) } + dVars = { + "fBuildTime": fBuildTime, + "callables": sPyCallables, + "callablesJS": sJSCallables, + "gctests": sGCTests, + "gctestsJS": sGCTestsJS, + "paragraph_rules": mergeRulesByOption(lParagraphRules), + "sentence_rules": mergeRulesByOption(lSentenceRules), + "paragraph_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lParagraphRulesJS)), + "sentence_rules_JS": jsconv.writeRulesToJSArray(mergeRulesByOption(lSentenceRulesJS)) + } dVars.update(dOptions) # compile graph rules - dVars2 = crg.make(lGraphRule, dDEF, sLang, dOptPriority, bJavaScript) + dVars2 = crg.make(lGraphRule, dDEF, sLang, dOptPriority) dVars.update(dVars2) + with open("_build/data_cache.json", "w", encoding="utf-8") as hDst: + hDst.write(json.dumps(dVars, ensure_ascii=False)) return dVars Index: compile_rules_graph.py ================================================================== --- compile_rules_graph.py +++ compile_rules_graph.py @@ -316,11 +316,11 @@ else: print(" # Unknown action.", sActionId) return None -def make (lRule, dDef, sLang, dOptPriority, bJavaScript): +def make (lRule, dDef, sLang, dOptPriority): "compile rules, returns a dictionary of values" # for clarity purpose, don’t create any file here # removing comments, zeroing empty lines, creating definitions, storing tests, merging rule lines print(" parsing rules...") @@ -452,10 +452,10 @@ # Result return { "graph_callables": sPyCallables, "graph_callablesJS": sJSCallables, - "rules_graphs": dAllGraph, + "rules_graphs": str(dAllGraph), "rules_graphsJS": str(dAllGraph).replace("True", "true").replace("False", "false"), - "rules_actions": dACTIONS, + "rules_actions": str(dACTIONS), "rules_actionsJS": str(dACTIONS).replace("True", "true").replace("False", "false") } ADDED gc_core/js/exemple-node.js Index: gc_core/js/exemple-node.js ================================================================== --- /dev/null +++ gc_core/js/exemple-node.js @@ -0,0 +1,22 @@ +// JavaScript + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, console */ + +"use strict"; + +//console.log('\x1B[2J\x1B[0f'); //Clear the console (cmd win) + +var oGrammalecte = require('./fr/gc_engine.js'); + +oGrammalecte.load(); + +var sPhrase = 'Le silences s’amasses Étoile brillantesss'; +console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Test de vérification de:', sPhrase); +console.log( oGrammalecte.parse(sPhrase) ); + +var sWord = 'toutt'; +console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Test de suggestion:', sWord); +var oSpellCheckGramma = oGrammalecte.getSpellChecker(); +console.log( Array.from( oSpellCheckGramma.suggest(sWord) ) ); Index: gc_core/js/lang_core/gc_engine.js ================================================================== --- gc_core/js/lang_core/gc_engine.js +++ gc_core/js/lang_core/gc_engine.js @@ -1,17 +1,25 @@ // Grammar checker engine -/*jslint esversion: 6*/ -/*global console,require,exports*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console */ "use strict"; ${string} ${regex} ${map} -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var gc_options = require("./gc_options.js"); + var gc_rules = require("./gc_rules.js"); + var gc_rules_graph = require("./gc_rules_graph.js"); + var cregex = require("./cregex.js"); + var text = require("../text.js"); +} else if (typeof(require) !== 'undefined') { var gc_options = require("resource://grammalecte/${lang}/gc_options.js"); var gc_rules = require("resource://grammalecte/${lang}/gc_rules.js"); var gc_rules_graph = require("resource://grammalecte/${lang}/gc_rules_graph.js"); var cregex = require("resource://grammalecte/${lang}/cregex.js"); var text = require("resource://grammalecte/text.js"); @@ -51,11 +59,14 @@ //// Initialization load: function (sContext="JavaScript", sColorType="aRGB", sPath="") { try { - if (typeof(require) !== 'undefined') { + if(typeof(process) !== 'undefined') { + var spellchecker = require("../graphspell/spellchecker.js"); + _oSpellChecker = new spellchecker.SpellChecker("${lang}", "", "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}"); + } else if (typeof(require) !== 'undefined') { var spellchecker = require("resource://grammalecte/graphspell/spellchecker.js"); _oSpellChecker = new spellchecker.SpellChecker("${lang}", "", "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}"); } else { _oSpellChecker = new SpellChecker("${lang}", sPath, "${dic_main_filename_js}", "${dic_extended_filename_js}", "${dic_community_filename_js}", "${dic_personal_filename_js}"); } @@ -179,11 +190,11 @@ this.dError = new Map(); this.dErrorPriority = new Map(); // Key = position; value = priority } asString () { - let s = "===== TEXT =====\n" + let s = "===== TEXT =====\n"; s += "sentence: " + this.sSentence0 + "\n"; s += "now: " + this.sSentence + "\n"; for (let dToken of this.lToken) { s += `#${dToken["i"]}\t${dToken["nStart"]}:${dToken["nEnd"]}\t${dToken["sValue"]}\t${dToken["sType"]}`; if (dToken.hasOwnProperty("lMorph")) { Index: gc_core/js/lang_core/gc_options.js ================================================================== --- gc_core/js/lang_core/gc_options.js +++ gc_core/js/lang_core/gc_options.js @@ -1,8 +1,10 @@ // Options for Grammalecte -/*jslint esversion: 6*/ -/*global exports*/ + +/* jshint esversion:6 */ +/* jslint esversion:6 */ +/* global exports */ ${map} var gc_options = { @@ -23,11 +25,11 @@ } return dColor; } catch (e) { console.error(e); - return {} + return {}; } }, lStructOpt: ${lStructOpt}, @@ -40,17 +42,17 @@ dColorType: ${dColorType}, dOptColor: ${dOptColor}, dOptLabel: ${dOptLabel} -} +}; if (typeof(exports) !== 'undefined') { - exports.getOptions = gc_options.getOptions; + exports.getOptions = gc_options.getOptions; exports.getOptionsColors = gc_options.getOptionsColors; - exports.lStructOpt = gc_options.lStructOpt; + exports.lStructOpt = gc_options.lStructOpt; exports.dOpt = gc_options.dOpt; exports.dColorType = gc_options.dColorType; exports.dOptColor = gc_options.dOptColor; - exports.dOptLabel = gc_options.dOptLabel; + exports.dOptLabel = gc_options.dOptLabel; } Index: gc_core/js/lang_core/gc_rules.js ================================================================== --- gc_core/js/lang_core/gc_rules.js +++ gc_core/js/lang_core/gc_rules.js @@ -1,7 +1,9 @@ // Grammar checker rules -/*jslint esversion: 6*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ /*global exports*/ "use strict"; ${string} @@ -9,12 +11,12 @@ var gc_rules = { lParagraphRules: ${paragraph_rules_JS}, lSentenceRules: ${sentence_rules_JS} -} +}; if (typeof(exports) !== 'undefined') { exports.lParagraphRules = gc_rules.lParagraphRules; exports.lSentenceRules = gc_rules.lSentenceRules; } Index: gc_core/js/lang_core/gc_rules_graph.js ================================================================== --- gc_core/js/lang_core/gc_rules_graph.js +++ gc_core/js/lang_core/gc_rules_graph.js @@ -1,8 +1,10 @@ // Grammar checker graph rules -/*jslint esversion: 6*/ -/*global exports*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global exports */ "use strict"; ${string} @@ -9,12 +11,12 @@ var gc_rules_graph = { dAllGraph: ${rules_graphsJS}, dRule: ${rules_actionsJS} -} +}; if (typeof(exports) !== 'undefined') { exports.dAllGraph = gc_rules_graph.dAllGraph; exports.dRule = gc_rules_graph.dRule; } Index: gc_core/js/tests.js ================================================================== --- gc_core/js/tests.js +++ gc_core/js/tests.js @@ -1,13 +1,17 @@ // JavaScript -/*jslint esversion: 6*/ -/*global console,require,exports*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console */ "use strict"; -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var helpers = require("./graphspell/helpers.js"); +} else if (typeof(require) !== 'undefined') { var helpers = require("resource://grammalecte/graphspell/helpers.js"); } class TestGrammarChecking { @@ -18,11 +22,16 @@ this._aRuleTested = new Set(); } * testParse (bDebug=false) { const t0 = Date.now(); - let sURL = (this.spfTests !== "") ? this.spfTests : "resource://grammalecte/"+this.gce.lang+"/tests_data.json"; + let sURL; + if(typeof(process) !== 'undefined') { + sURL = (this.spfTests !== "") ? this.spfTests : "./"+this.gce.lang+"/tests_data.json"; + } else { + sURL = (this.spfTests !== "") ? this.spfTests : "resource://grammalecte/"+this.gce.lang+"/tests_data.json"; + } const aData = JSON.parse(helpers.loadFile(sURL)).aData; let nInvalid = 0; let nTotal = 0; let sErrorText; let sSugg; Index: gc_core/js/text.js ================================================================== --- gc_core/js/text.js +++ gc_core/js/text.js @@ -1,8 +1,10 @@ // JavaScript -/*jslint esversion: 6*/ -/*global require,exports*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console */ "use strict"; var text = { Index: gc_lang/fr/modules-js/conj.js ================================================================== --- gc_lang/fr/modules-js/conj.js +++ gc_lang/fr/modules-js/conj.js @@ -1,16 +1,20 @@ // Grammalecte - Conjugueur // License: GPL 3 -/*jslint esversion: 6*/ -/*global console,require,exports,self,browser*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console, self, browser, chrome, __dirname */ "use strict"; ${map} -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var helpers = require("../graphspell/helpers.js"); +} else if (typeof(require) !== 'undefined') { var helpers = require("resource://grammalecte/graphspell/helpers.js"); } var conj = { _lVtyp: [], @@ -494,25 +498,28 @@ } } // Initialization -if (!conj.bInit && typeof(browser) !== 'undefined') { +if(!conj.bInit && typeof(process) !== 'undefined') { + // Work with nodejs + conj.init(helpers.loadFile(__dirname+"/conj_data.json")); +} else if (!conj.bInit && typeof(browser) !== 'undefined') { // WebExtension Standard (but not in Worker) conj.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/conj_data.json"))); } else if (!conj.bInit && typeof(chrome) !== 'undefined') { // WebExtension Chrome (but not in Worker) conj.init(helpers.loadFile(chrome.extension.getURL("grammalecte/fr/conj_data.json"))); } else if (!conj.bInit && typeof(require) !== 'undefined') { // Add-on SDK and Thunderbird conj.init(helpers.loadFile("resource://grammalecte/fr/conj_data.json")); -} else if (!conj.bInit && typeof(self) !== 'undefined' && typeof(self.port) !== 'undefined' && typeof(self.port.on) !== "undefined") { +} else if (!conj.bInit && typeof(self) !== 'undefined' && typeof(self.port) !== 'undefined' && typeof(self.port.on) !== 'undefined') { // used within Firefox content script (conjugation panel). // can’t load JSON from here, so we do it in ui.js and send it here. self.port.on("provideConjData", function (sJSONData) { conj.init(sJSONData); - }); + }); } else if (conj.bInit){ console.log("Module conj déjà initialisé"); } else { //console.log("Module conj non initialisé"); } Index: gc_lang/fr/modules-js/conj_generator.js ================================================================== --- gc_lang/fr/modules-js/conj_generator.js +++ gc_lang/fr/modules-js/conj_generator.js @@ -1,11 +1,13 @@ // JavaScript -/* - Conjugation generator - beta stage, unfinished, the root for a new way to generate flexions… -*/ +// Conjugation generator +// beta stage, unfinished, the root for a new way to generate flexions… + + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ "use strict"; var conj_generator = { @@ -137,11 +139,11 @@ [2, "ît", ":Sq:3s/*", false], [2, "is", ":E:2s/*", false], [2, "issons", ":E:1p/*", false], [2, "issez", ":E:2p/*", false] ], - + // premier groupe (bien plus irrégulier que prétendu) "V1": { // a // verbes en -er, -ger, -yer, -cer "er": [ Index: gc_lang/fr/modules-js/cregex.js ================================================================== --- gc_lang/fr/modules-js/cregex.js +++ gc_lang/fr/modules-js/cregex.js @@ -1,7 +1,9 @@ -//// Grammalecte - Compiled regular expressions -/*jslint esversion: 6*/ +// Grammalecte - Compiled regular expressions + +/* jshint esversion:6 */ +/* jslint esversion:6 */ var cregex = { ///// Lemme _zLemma: new RegExp(">([a-zà-öø-ÿ0-9Ā-ʯ][a-zà-öø-ÿ0-9Ā-ʯ-]+)"), Index: gc_lang/fr/modules-js/gce_analyseur.js ================================================================== --- gc_lang/fr/modules-js/gce_analyseur.js +++ gc_lang/fr/modules-js/gce_analyseur.js @@ -1,7 +1,9 @@ -//// GRAMMAR CHECKING ENGINE PLUGIN: Parsing functions for French language -/*jslint esversion: 6*/ +// GRAMMAR CHECKING ENGINE PLUGIN: Parsing functions for French language + +/* jshint esversion:6 */ +/* jslint esversion:6 */ function g_morphVC (dToken, sPattern, sNegPattern="") { let nEnd = dToken["sValue"].lastIndexOf("-"); if (dToken["sValue"].includes("-t-")) { nEnd = nEnd - 2; @@ -128,11 +130,11 @@ } return false; } -//// Exceptions +// Exceptions const aREGULARPLURAL = new Set(["abricot", "amarante", "aubergine", "acajou", "anthracite", "brique", "caca", "café", "carotte", "cerise", "chataigne", "corail", "citron", "crème", "grave", "groseille", "jonquille", "marron", "olive", "pervenche", "prune", "sable"]); const aSHOULDBEVERB = new Set(["aller", "manger"]); Index: gc_lang/fr/modules-js/gce_date_verif.js ================================================================== --- gc_lang/fr/modules-js/gce_date_verif.js +++ gc_lang/fr/modules-js/gce_date_verif.js @@ -1,11 +1,12 @@ -//// GRAMMAR CHECKING ENGINE PLUGIN -/*jslint esversion: 6*/ +// GRAMMAR CHECKING ENGINE PLUGIN // Check date validity - // WARNING: when creating a Date, month must be between 0 and 11 + +/* jshint esversion:6 */ +/* jslint esversion:6 */ const _lDay = ["dimanche", "lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi"]; const _dMonth = new Map ([ ["janvier", 1], ["février", 2], ["mars", 3], ["avril", 4], ["mai", 5], ["juin", 6], ["juillet", 7], Index: gc_lang/fr/modules-js/gce_suggestions.js ================================================================== --- gc_lang/fr/modules-js/gce_suggestions.js +++ gc_lang/fr/modules-js/gce_suggestions.js @@ -1,10 +1,16 @@ -//// GRAMMAR CHECKING ENGINE PLUGIN: Suggestion mechanisms -/*jslint esversion: 6*/ -/*global require*/ +// GRAMMAR CHECKING ENGINE PLUGIN: Suggestion mechanisms + +/* jshint esversion:6 */ +/* jslint esversion:6 */ +/* global require */ -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var conj = require("./conj.js"); + var mfsp = require("./mfsp.js"); + var phonet = require("./phonet.js"); +} else if (typeof(require) !== 'undefined') { var conj = require("resource://grammalecte/fr/conj.js"); var mfsp = require("resource://grammalecte/fr/mfsp.js"); var phonet = require("resource://grammalecte/fr/phonet.js"); } Index: gc_lang/fr/modules-js/lexicographe.js ================================================================== --- gc_lang/fr/modules-js/lexicographe.js +++ gc_lang/fr/modules-js/lexicographe.js @@ -1,9 +1,11 @@ // Grammalecte - Lexicographe // License: MPL 2 -/*jslint esversion: 6*/ -/*global require,exports*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console */ "use strict"; ${string} ${map} Index: gc_lang/fr/modules-js/mfsp.js ================================================================== --- gc_lang/fr/modules-js/mfsp.js +++ gc_lang/fr/modules-js/mfsp.js @@ -1,13 +1,17 @@ // Grammalecte -/*jslint esversion: 6*/ -/*global console,require,exports,browser*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console, browser,__dirname */ "use strict"; -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var helpers = require("../graphspell/helpers.js"); +} else if (typeof(require) !== 'undefined') { var helpers = require("resource://grammalecte/graphspell/helpers.js"); } var mfsp = { @@ -102,11 +106,14 @@ } }; // Initialization -if (!mfsp.bInit && typeof(browser) !== 'undefined') { +if(!mfsp.bInit && typeof(process) !== 'undefined') { + //Nodejs + mfsp.init(helpers.loadFile(__dirname+"/mfsp_data.json")); +} else if (!mfsp.bInit && typeof(browser) !== 'undefined') { // WebExtension mfsp.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/mfsp_data.json"))); } else if (!mfsp.bInit && typeof(require) !== 'undefined') { // Add-on SDK and Thunderbird mfsp.init(helpers.loadFile("resource://grammalecte/fr/mfsp_data.json")); Index: gc_lang/fr/modules-js/phonet.js ================================================================== --- gc_lang/fr/modules-js/phonet.js +++ gc_lang/fr/modules-js/phonet.js @@ -1,9 +1,14 @@ // Grammalecte - Suggestion phonétique -/*jslint esversion: 6*/ + +/* jshint esversion:6 */ +/* jslint esversion:6 */ +/* global __dirname */ -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var helpers = require("../graphspell/helpers.js"); +} else if (typeof(require) !== 'undefined') { var helpers = require("resource://grammalecte/graphspell/helpers.js"); } var phonet = { @@ -82,11 +87,14 @@ } }; // Initialization -if (!phonet.bInit && typeof(browser) !== 'undefined') { +if (!phonet.bInit && typeof(process) !== 'undefined') { + //Nodejs + phonet.init(helpers.loadFile(__dirname+"/phonet_data.json")); +} else if (!phonet.bInit && typeof(browser) !== 'undefined') { // WebExtension phonet.init(helpers.loadFile(browser.extension.getURL("grammalecte/fr/phonet_data.json"))); } else if (!phonet.bInit && typeof(require) !== 'undefined') { // Add-on SDK and Thunderbird phonet.init(helpers.loadFile("resource://grammalecte/fr/phonet_data.json")); Index: gc_lang/fr/modules-js/textformatter.js ================================================================== --- gc_lang/fr/modules-js/textformatter.js +++ gc_lang/fr/modules-js/textformatter.js @@ -1,12 +1,14 @@ // Grammalecte - text formatter -/*jslint esversion: 6*/ -/*global exports*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global exports, console */ "use strict"; -${map} +//!${map} // Latin letters: http://unicode-table.com/fr/ // 0-9 // A-Z @@ -83,11 +85,11 @@ "ts_apostrophe": [ [/\b([ldnjmtscç])['´‘′`](?=[a-zA-Zà-ö0-9À-Öø-ÿØ-ßĀ-ʯ])/ig, "$1’"], [/\b(qu|jusqu|lorsqu|puisqu|quoiqu|quelqu|presqu|entr|aujourd|prud)['´‘′`]/ig, "$1’"] ], "ts_ellipsis": [ [/\.\.\./g, "…"], [/…\.\./g, "……"], [/…\.(?!\.)/g, "…"] ], - "ts_n_dash_middle": [ [/ [-—] /g, " – "], + "ts_n_dash_middle": [ [/ [-—] /g, " – "], [/ [-—],/g, " –,"] ], "ts_m_dash_middle": [ [/ [-–] /g, " — "], [/ [-–],/g, " —,"] ], "ts_n_dash_start": [ [/^[-—][  ]/gm, "– "], [/^– /gm, "– "], @@ -256,38 +258,97 @@ ["ma_word", true], ["ma_1letter_lowercase", false], ["ma_1letter_uppercase", false] ]); -const dTFOptions = dTFDefaultOptions.gl_shallowCopy(); - class TextFormatter { - constructor () { + constructor (bDebug=false) { this.sLang = "fr"; + this.bDebug = bDebug; + //don't change this in external ;) + this._optionsUsed = dTFDefaultOptions.gl_shallowCopy(); } formatText (sText, dOpt=null) { if (dOpt !== null) { - dTFOptions.gl_updateOnlyExistingKeys(dOpt); + this._optionsUsed.gl_updateOnlyExistingKeys(dOpt); + } + for (let [sOptName, bVal] of this._optionsUsed) { + //console.log(oReplTable); + if (bVal && oReplTable[sOptName]) { + for (let [zRgx, sRep] of oReplTable[sOptName]) { + sText = sText.replace(zRgx, sRep); + } + } + } + return sText; + } + + formatTextCount (sText, dOpt=null) { + let nCount = 0; + if (dOpt !== null) { + this._optionsUsed.gl_updateOnlyExistingKeys(dOpt); } - for (let [sOptName, bVal] of dTFOptions) { - if (bVal && oReplTable.has(sOptName)) { + for (let [sOptName, bVal] of this._optionsUsed) { + if (bVal && oReplTable[sOptName]) { for (let [zRgx, sRep] of oReplTable[sOptName]) { + nCount += (sText.match(zRgx) || []).length; sText = sText.replace(zRgx, sRep); } } } + return [sText, nCount]; + } + + formatTextRule (sText, sRuleName) { + if (oReplTable[sRuleName]) { + for (let [zRgx, sRep] of oReplTable[sRuleName]) { + sText = sText.replace(zRgx, sRep); + } + } else if (this.bDebug){ + console.log("# Error. TF: there is no option “" + sRuleName+ "”."); + } return sText; } + + formatTextRuleCount (sText, sRuleName) { + let nCount = 0; + if (oReplTable[sRuleName]) { + for (let [zRgx, sRep] of oReplTable[sRuleName]) { + nCount += (sText.match(zRgx) || []).length; + sText = sText.replace(zRgx, sRep); + } + } else if (this.bDebug){ + console.log("# Error. TF: there is no option “" + sRuleName+ "”."); + } + return [sText, nCount]; + } getDefaultOptions () { - return dTFDefaultOptions; + //we return a copy to make sure they are no modification in external + return dTFDefaultOptions.gl_shallowCopy(); + } + + getUsedOptions () { + //we return a copy to make sure they are no modification in external + return this._optionsUsed.gl_shallowCopy(); + } + + setUsedOptions (dOpt=null) { + if (dOpt !== null) { + this._optionsUsed.gl_updateOnlyExistingKeys(dOpt); + } else if (this.bDebug){ + console.log("# Error. TF: no option to change."); + } + } + + getReplTable(){ + return oReplTable; } } if (typeof(exports) !== 'undefined') { exports.TextFormatter = TextFormatter; - exports.oReplTable = oReplTable; } Index: gc_lang/fr/webext/content_scripts/panel_tf.js ================================================================== --- gc_lang/fr/webext/content_scripts/panel_tf.js +++ gc_lang/fr/webext/content_scripts/panel_tf.js @@ -9,10 +9,13 @@ constructor (...args) { super(...args); this.xTFNode = this._createTextFormatter(); this.xPanelContent.appendChild(this.xTFNode); this.xTextArea = null; + + this.TextFormatter = new TextFormatter(); + this.formatText = this.TextFormatter.formatTextRuleCount; } _createTextFormatter () { let xTFNode = document.createElement("div"); try { @@ -157,11 +160,11 @@ xLine.appendChild(this._createOption("o_ordinals_no_exponant", true, "Ordinaux (15e, XXIe…)")); xLine.appendChild(this._createOption("o_ordinals_exponant", true, "e → ᵉ")); xLine.appendChild(oGrammalecte.createNode("div", {id: "res_"+"o_ordinals_no_exponant", className: "grammalecte_tf_result", textContent: "·"})); return xLine; } - + /* Actions */ start (xNode) { if (xNode !== null && xNode.tagName == "TEXTAREA") { @@ -258,11 +261,11 @@ //window.setCursor("wait"); // change pointer this.resetProgressBar(); let sText = this.xTextArea.value.normalize("NFC"); document.getElementById('grammalecte_tf_progressbar').max = 7; let n1 = 0, n2 = 0, n3 = 0, n4 = 0, n5 = 0, n6 = 0, n7 = 0; - + // Restructuration if (this.isSelected("o_group_struct")) { if (this.isSelected("o_remove_hyphens_at_end_of_paragraphs")) { [sText, n1] = this.removeHyphenAtEndOfParagraphs(sText); document.getElementById('res_o_remove_hyphens_at_end_of_paragraphs').textContent = n1; @@ -508,28 +511,10 @@ catch (e) { showError(e); } } - formatText (sText, sOptName) { - let nCount = 0; - try { - if (!oReplTable.hasOwnProperty(sOptName)) { - console.log("# Error. TF: there is no option “" + sOptName+ "”."); - return [sText, nCount]; - } - for (let [zRgx, sRep] of oReplTable[sOptName]) { - nCount += (sText.match(zRgx) || []).length; - sText = sText.replace(zRgx, sRep); - } - } - catch (e) { - showError(e); - } - return [sText, nCount]; - } - removeHyphenAtEndOfParagraphs (sText) { let nCount = (sText.match(/-[  ]*\n/gm) || []).length; sText = sText.replace(/-[  ]*\n/gm, ""); return [sText, nCount]; } Index: graphspell-js/char_player.js ================================================================== --- graphspell-js/char_player.js +++ graphspell-js/char_player.js @@ -1,8 +1,11 @@ // list of similar chars // useful for suggestion mechanism +/* jshint esversion:6 */ +/* jslint esversion:6 */ + ${map} var char_player = { @@ -23,11 +26,11 @@ ['â', 'a'], ['è', 'e'], ['ï', 'i'], ['ö', 'o'], ['ù', 'u'], ['ŷ', 'i'], ['ä', 'a'], ['ê', 'e'], ['í', 'i'], ['ó', 'o'], ['ü', 'u'], ['ý', 'i'], ['á', 'a'], ['ë', 'e'], ['ì', 'i'], ['ò', 'o'], ['ú', 'u'], ['ỳ', 'i'], ['ā', 'a'], ['ē', 'e'], ['ī', 'i'], ['ō', 'o'], ['ū', 'u'], ['ȳ', 'i'], ['ç', 'c'], ['ñ', 'n'], ['k', 'q'], ['w', 'v'], - ['œ', 'oe'], ['æ', 'ae'], + ['œ', 'oe'], ['æ', 'ae'], ['ſ', 's'], ['ffi', 'ffi'], ['ffl', 'ffl'], ['ff', 'ff'], ['ſt', 'ft'], ['fi', 'fi'], ['fl', 'fl'], ['st', 'st'] ]), simplifyWord: function (sWord) { // word simplication before calculating distance between words @@ -102,11 +105,11 @@ ["f", "fF"], ["F", "Ff"], ["g", "gGjJĵĴ"], ["G", "GgJjĴĵ"], - + ["h", "hH"], ["H", "Hh"], ["i", "iIîÎïÏyYíÍìÌīĪÿŸ"], ["I", "IiÎîÏïYyÍíÌìĪīŸÿ"], @@ -380,11 +383,11 @@ // Other functions filterSugg: function (aSugg) { return aSugg.filter((sSugg) => { return !sSugg.endsWith("è") && !sSugg.endsWith("È"); }); } -} +}; if (typeof(exports) !== 'undefined') { exports._xTransCharsForSpelling = char_player._xTransCharsForSpelling; exports.spellingNormalization = char_player.spellingNormalization; Index: graphspell-js/dawg.js ================================================================== --- graphspell-js/dawg.js +++ graphspell-js/dawg.js @@ -6,17 +6,21 @@ // License: MPL 2 // // This tool encodes lexicon into an indexable binary dictionary // Input files MUST be encoded in UTF-8. +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console, helpers */ + "use strict"; - -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var str_transform = require("./str_transform.js"); +} else if (typeof(require) !== 'undefined') { var str_transform = require("resource://grammalecte/graphspell/str_transform.js"); } - ${map} class DAWG { @@ -98,11 +102,11 @@ let lTemp = []; for (let c of sFlex) { lTemp.push(dChar.get(c)); } lTemp.push(iAff+nChar); - lTemp.push(iTag+nChar+nAff) + lTemp.push(iTag+nChar+nAff); lWord.push(lTemp); } lEntry.length = 0; // clear the array // Dictionary of arc values occurrency, to sort arcs of each node @@ -428,11 +432,11 @@ }, reset: function () { this.nNextId = 0; } -} +}; class DawgNode { constructor () { @@ -542,11 +546,11 @@ let sHexVal = nVal.toString(16); // conversion to hexadecimal string //console.log(`value: ${nVal} in ${nByte} bytes`); if (sHexVal.length < (nByte*2)) { return "0".repeat((nByte*2) - sHexVal.length) + sHexVal; } else if (sHexVal.length == (nByte*2)) { - return sHexVal + return sHexVal; } else { throw "Conversion to byte string: value bigger than allowed."; } } } Index: graphspell-js/helpers.js ================================================================== --- graphspell-js/helpers.js +++ graphspell-js/helpers.js @@ -1,9 +1,10 @@ - // HELPERS -/*jslint esversion: 6*/ -/*global console,require,exports,XMLHttpRequest*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console, XMLHttpRequest */ "use strict"; var helpers = { @@ -20,17 +21,23 @@ loadFile: function (spf) { // load ressources in workers (suggested by Mozilla extensions reviewers) // for more options have a look here: https://gist.github.com/Noitidart/ec1e6b9a593ec7e3efed // if not in workers, use sdk/data.load() instead try { - console.log("loadFile: " + spf); - let xRequest; - xRequest = new XMLHttpRequest(); - xRequest.open('GET', spf, false); // 3rd arg is false for synchronous, sync is acceptable in workers - xRequest.overrideMimeType('text/json'); - xRequest.send(); - return xRequest.responseText; + if(typeof(process) !== 'undefined') { + //console.log('loadFile(disque): ' + spf); + let fs = require("fs"); + return fs.readFileSync(spf, "utf8"); + } else { + console.log("loadFile: " + spf); + let xRequest; + xRequest = new XMLHttpRequest(); + xRequest.open('GET', spf, false); // 3rd arg is false for synchronous, sync is acceptable in workers + xRequest.overrideMimeType('text/json'); + xRequest.send(); + return xRequest.responseText; + } } catch (e) { console.error(e); return null; } Index: graphspell-js/ibdawg.js ================================================================== --- graphspell-js/ibdawg.js +++ graphspell-js/ibdawg.js @@ -1,18 +1,22 @@ -//// IBDAWG -/*jslint esversion: 6*/ -/*global console,require,exports*/ +// IBDAWG + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console, __dirname */ "use strict"; - -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var str_transform = require("./str_transform.js"); + var helpers = require("./helpers.js"); + var char_player = require("./char_player.js"); +} else if (typeof(require) !== 'undefined') { var str_transform = require("resource://grammalecte/graphspell/str_transform.js"); var helpers = require("resource://grammalecte/graphspell/helpers.js"); var char_player = require("resource://grammalecte/graphspell/char_player.js"); } - // Don’t remove . Necessary in TB. ${string} ${map} ${set} @@ -95,11 +99,16 @@ constructor (param1, sPath="") { // param1 can be a filename or a object with all the necessary data. try { let oData = null; if (typeof(param1) == "string") { - let sURL = (sPath !== "") ? sPath + "/" + param1 : "resource://grammalecte/graphspell/_dictionaries/"+param1; + let sURL; + if(typeof(process) !== 'undefined') { + sURL = (sPath !== "") ? sPath + "/" + param1 : __dirname + "/_dictionaries/"+param1; + } else { + sURL = (sPath !== "") ? sPath + "/" + param1 : "resource://grammalecte/graphspell/_dictionaries/"+param1; + } oData = JSON.parse(helpers.loadFile(sURL)); } else { oData = param1; } Object.assign(this, oData); @@ -221,11 +230,11 @@ return oJSON; } isValidToken (sToken) { // checks if sToken is valid (if there is hyphens in sToken, sToken is split, each part is checked) - sToken = char_player.spellingNormalization(sToken) + sToken = char_player.spellingNormalization(sToken); if (this.isValid(sToken)) { return true; } if (sToken.includes("-")) { if (sToken.gl_count("-") > 4) { @@ -298,11 +307,11 @@ return Boolean(this._convBytesToInteger(this.byDic.slice(iAddr, iAddr+this.nBytesArc)) & this._finalNodeMask); } getMorph (sWord) { // retrieves morphologies list, different casing allowed - sWord = char_player.spellingNormalization(sWord) + sWord = char_player.spellingNormalization(sWord); let l = this.morph(sWord); if (sWord[0].gl_isUpperCase()) { l.push(...this.morph(sWord.toLowerCase())); if (sWord.gl_isUpperCase() && sWord.length > 1) { l.push(...this.morph(sWord.gl_toCapitalize())); @@ -311,12 +320,12 @@ return l; } suggest (sWord, nSuggLimit=10) { // returns a array of suggestions for - //const t0 = Date.now(); - sWord = char_player.spellingNormalization(sWord) + //console.time("Suggestions for " + sWord + "); + sWord = char_player.spellingNormalization(sWord); let sPfx = ""; let sSfx = ""; [sPfx, sWord, sSfx] = char_player.cut(sWord); let nMaxSwitch = Math.max(Math.floor(sWord.length / 3), 1); let nMaxDel = Math.floor(sWord.length / 5); @@ -325,14 +334,13 @@ let oSuggResult = new SuggResult(sWord); this._suggest(oSuggResult, sWord, nMaxSwitch, nMaxDel, nMaxHardRepl, nMaxJump); let aSugg = oSuggResult.getSuggestions(nSuggLimit); if (sSfx || sPfx) { // we add what we removed - return aSugg.map( (sSugg) => { return sPfx + sSugg + sSfx } ); + return aSugg.map( (sSugg) => { return sPfx + sSugg + sSfx; } ); } - //const t1 = Date.now(); - //console.log("Suggestions for " + sWord + " in " + ((t1-t0)/1000).toString() + " s"); + //console.timeEnd("Suggestions for " + sWord + "); return aSugg; } _suggest (oSuggResult, sRemain, nMaxSwitch=0, nMaxDel=0, nMaxHardRepl=0, nMaxJump=0, nDist=0, nDeep=0, iAddr=0, sNewWord="", bAvoidLoop=false) { // returns a set of suggestions @@ -589,11 +597,11 @@ } } } * _getArcs1 (iAddr) { - "generator: return all arcs at as tuples of (nVal, iAddr)" + // generator: return all arcs at as tuples of (nVal, iAddr) while (true) { let iEndArcAddr = iAddr+this.nBytesArc; let nRawArc = this._convBytesToInteger(this.byDic.slice(iAddr, iEndArcAddr)); yield [nRawArc & this._arcMask, this._convBytesToInteger(this.byDic.slice(iEndArcAddr, iEndArcAddr+this.nBytesNodeAddress))]; if (nRawArc & this._lastArcMask) { Index: graphspell-js/spellchecker.js ================================================================== --- graphspell-js/spellchecker.js +++ graphspell-js/spellchecker.js @@ -6,19 +6,23 @@ // - the main dictionary, bundled with the package // - the extended dictionary // - the community dictionary, added by an organization // - the personal dictionary, created by the user for its own convenience +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, exports, console, IBDAWG, Tokenizer */ "use strict"; - -if (typeof(require) !== 'undefined') { +if(typeof(process) !== 'undefined') { + var ibdawg = require("./ibdawg.js"); + var tokenizer = require("./tokenizer.js"); +} else if (typeof(require) !== 'undefined') { var ibdawg = require("resource://grammalecte/graphspell/ibdawg.js"); var tokenizer = require("resource://grammalecte/graphspell/tokenizer.js"); } - ${map} const dDefaultDictionaries = new Map([ @@ -64,11 +68,11 @@ catch (e) { let sfDictionary = (typeof(dictionary) == "string") ? dictionary : dictionary.sLangName + "/" + dictionary.sFileName; if (bNecessary) { throw "Error: <" + sfDictionary + "> not loaded. " + e.message; } - console.log("Error: <" + sfDictionary + "> not loaded.") + console.log("Error: <" + sfDictionary + "> not loaded."); console.log(e.message); return null; } } @@ -195,11 +199,11 @@ return true; } if (this.bExtendedDic && this.oExtendedDic.isValid(sWord)) { return true; } - if (this.bCommunityDic && this.oCommunityDic.isValid(sToken)) { + if (this.bCommunityDic && this.oCommunityDic.isValid(sWord)) { return true; } if (this.bPersonalDic && this.oPersonalDic.isValid(sWord)) { return true; } @@ -212,11 +216,11 @@ return true; } if (this.bExtendedDic && this.oExtendedDic.lookup(sWord)) { return true; } - if (this.bCommunityDic && this.oCommunityDic.lookup(sToken)) { + if (this.bCommunityDic && this.oCommunityDic.lookup(sWord)) { return true; } if (this.bPersonalDic && this.oPersonalDic.lookup(sWord)) { return true; } @@ -271,11 +275,11 @@ } } * select (sFlexPattern="", sTagsPattern="") { // generator: returns all entries which flexion fits and morphology fits - yield* this.oMainDic.select(sFlexPattern, sTagsPattern) + yield* this.oMainDic.select(sFlexPattern, sTagsPattern); if (this.bExtendedDic) { yield* this.oExtendedDic.select(sFlexPattern, sTagsPattern); } if (this.bCommunityDic) { yield* this.oCommunityDic.select(sFlexPattern, sTagsPattern); Index: graphspell-js/str_transform.js ================================================================== --- graphspell-js/str_transform.js +++ graphspell-js/str_transform.js @@ -1,7 +1,10 @@ -//// STRING TRANSFORMATION -/*jslint esversion: 6*/ +// STRING TRANSFORMATION + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global exports, console */ "use strict"; // Note: 48 is the ASCII code for "0" ADDED graphspell-js/test/minimal.js Index: graphspell-js/test/minimal.js ================================================================== --- /dev/null +++ graphspell-js/test/minimal.js @@ -0,0 +1,57 @@ +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/* global require, console */ + +"use strict"; + +/* +Reset = "\x1b[0m" +Bright = "\x1b[1m" +Dim = "\x1b[2m" +Underscore = "\x1b[4m" +Blink = "\x1b[5m" +Reverse = "\x1b[7m" +Hidden = "\x1b[8m" + +FgBlack = "\x1b[30m" +FgRed = "\x1b[31m" +FgGreen = "\x1b[32m" +FgYellow = "\x1b[33m" +FgBlue = "\x1b[34m" +FgMagenta = "\x1b[35m" +FgCyan = "\x1b[36m" +FgWhite = "\x1b[37m" + +BgBlack = "\x1b[40m" +BgRed = "\x1b[41m" +BgGreen = "\x1b[42m" +BgYellow = "\x1b[43m" +BgBlue = "\x1b[44m" +BgMagenta = "\x1b[45m" +BgCyan = "\x1b[46m" +BgWhite = "\x1b[47m" +*/ + +//console.log('\x1B[2J\x1B[0f'); //Clear the console (cmd win) + +var spellCheck = require("../spellchecker.js"); +var checker = new spellCheck.SpellChecker('fr', '../_dictionaries'); + +function perf(sWord){ + console.log('\x1b[1m\x1b[31m%s\x1b[0m', '--------------------------------'); + + console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Vérification de:', sWord); + console.time('Valid:'+sWord); + console.log(sWord, checker.isValid(sWord) ); + console.timeEnd('Valid:'+sWord); + + console.log('\x1b[36m%s \x1b[32m%s\x1b[0m', 'Suggestion de:', sWord); + console.time('Suggestion:'+sWord); + console.log( JSON.stringify( Array.from(checker.suggest(sWord)) ) ); + console.timeEnd('Suggestion:'+sWord); +} + +perf('binjour'); +perf('saluté'); +perf('graphspell'); +perf('dicollecte'); Index: graphspell-js/tokenizer.js ================================================================== --- graphspell-js/tokenizer.js +++ graphspell-js/tokenizer.js @@ -1,9 +1,11 @@ // JavaScript // Very simple tokenizer -/*jslint esversion: 6*/ -/*global require,exports*/ + +/* jshint esversion:6, -W097 */ +/* jslint esversion:6 */ +/*global require, exports, console*/ "use strict"; const aTkzPatterns = { @@ -70,11 +72,11 @@ for (let [zRegex, sType] of this.aRules) { if (sType !== "SPACE" || bWithSpaces) { try { if ((m = zRegex.exec(sText)) !== null) { iToken += 1; - yield { "i": iToken, "sType": sType, "sValue": m[0], "nStart": iNext, "nEnd": iNext + m[0].length } + yield { "i": iToken, "sType": sType, "sValue": m[0], "nStart": iNext, "nEnd": iNext + m[0].length }; iCut = m[0].length; break; } } catch (e) { Index: make.py ================================================================== --- make.py +++ make.py @@ -184,11 +184,11 @@ for sf in os.listdir(spLangPack): if not os.path.isdir(spLangPack+"/"+sf): hZip.write(spLangPack+"/"+sf, sAddPath+spLangPack+"/"+sf) -def create (sLang, xConfig, bInstallOXT, bJavaScript): +def create (sLang, xConfig, bInstallOXT, bJavaScript, bUseCache): "make Grammalecte for project " oNow = datetime.datetime.now() print("============== MAKE GRAMMALECTE [{0}] at {1.hour:>2} h {1.minute:>2} min {1.second:>2} s ==============".format(sLang, oNow)) #### READ CONFIGURATION @@ -198,11 +198,11 @@ dVars = xConfig._sections['args'] dVars['locales'] = dVars["locales"].replace("_", "-") dVars['loc'] = str(dict([ [s, [s[0:2], s[3:5], ""]] for s in dVars["locales"].split(" ") ])) ## COMPILE RULES - dResult = compile_rules.make(spLang, dVars['lang'], bJavaScript) + dResult = compile_rules.make(spLang, dVars['lang'], bUseCache) dVars.update(dResult) ## READ GRAMMAR CHECKER PLUGINS print("PYTHON:") print("+ Plugins: ", end="") @@ -373,10 +373,11 @@ def main (): "build Grammalecte with requested options" print("Python: " + sys.version) xParser = argparse.ArgumentParser() xParser.add_argument("lang", type=str, nargs='+', help="lang project to generate (name of folder in /lang)") + xParser.add_argument("-uc", "--use_cache", help="use data cache instead of rebuilding rules", action="store_true") xParser.add_argument("-b", "--build_data", help="launch build_data.py (part 1 and 2)", action="store_true") xParser.add_argument("-bb", "--build_data_before", help="launch build_data.py (only part 1: before dictionary building)", action="store_true") xParser.add_argument("-ba", "--build_data_after", help="launch build_data.py (only part 2: before dictionary building)", action="store_true") xParser.add_argument("-d", "--dict", help="generate FSA dictionary", action="store_true") xParser.add_argument("-t", "--tests", help="run unit tests", action="store_true") @@ -440,11 +441,11 @@ # copy dictionaries from Graphspell copyGraphspellDictionaries(dVars, xArgs.javascript, xArgs.add_extended_dictionary, xArgs.add_community_dictionary, xArgs.add_personal_dictionary) # make - sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, ) + sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, xArgs.use_cache) # tests if xArgs.tests or xArgs.perf or xArgs.perf_memo: print("> Running tests") try: