#!/usr/bin/env python3
# coding: UTF-8
import sys
import os
import subprocess
import re
import zipfile
import traceback
import configparser
import datetime
import argparse
import importlib
import unittest
import json
import platform
from distutils import dir_util, file_util
import dialog_bundled
import compile_rules
import helpers
import lex_build
sWarningMessage = "The content of this folder is generated by code and replaced at each build.\n"
def getConfig (sLang):
xConfig = configparser.SafeConfigParser()
xConfig.optionxform = str
try:
xConfig.read("gc_lang/" + sLang + "/config.ini", encoding="utf-8")
except:
print("# Error. Can’t read config file [" + sLang + "]")
exit()
return xConfig
def createOptionsLabelProperties (dOptLbl):
sContent = ""
for sOpt, tLabel in dOptLbl.items():
sContent += sOpt + "=" + tLabel[0] + "\n"
if tLabel[1]:
sContent += "hlp_" + sOpt + "=" + tLabel[1] + "\n"
return sContent
def createDialogOptionsXDL (dVars):
sFixedline = '<dlg:fixedline dlg:id="{0}" dlg:tab-index="{1}" dlg:top="{2}" dlg:left="5" dlg:width="{3}" dlg:height="10" dlg:value="&{0}" />\n'
sCheckbox = '<dlg:checkbox dlg:id="{0}" dlg:tab-index="{1}" dlg:top="{2}" dlg:left="{3}" dlg:width="{4}" dlg:height="10" dlg:value="&{0}" dlg:checked="{5}" {6} />\n'
iTabIndex = 1
nPosY = 5
nWidth = 240
sContent = ""
dOpt = dVars["dOptPython"]
dOptLabel = dVars["dOptLabel"][dVars["lang"]]
for sGroup, lGroupOptions in dVars["lStructOpt"]:
sContent += sFixedline.format(sGroup, iTabIndex, nPosY, nWidth)
iTabIndex += 1
for lLineOptions in lGroupOptions:
nElemWidth = nWidth // len(lLineOptions)
nPosY += 10
nPosX = 10
for sOpt in lLineOptions:
sHelp = 'dlg:help-text="&hlp_%s"'%sOpt if dOptLabel[sOpt][1] else ""
sContent += sCheckbox.format(sOpt, iTabIndex, nPosY, nPosX, nElemWidth, "true" if dOpt[sOpt] else "false", sHelp)
iTabIndex += 1
nPosX += nElemWidth
nPosY += 10
return sContent
def createOXT (spLang, dVars, dOxt, spLangPack, bInstall):
"create extension for Writer"
print("Building extension for Writer")
spfZip = "_build/" + dVars['name'] + "-"+ dVars['lang'] +"-v" + dVars['version'] + '.oxt'
hZip = zipfile.ZipFile(spfZip, mode='w', compression=zipfile.ZIP_DEFLATED)
# Package and parser
copyGrammalectePyPackageInZipFile(hZip, spLangPack, "pythonpath/")
hZip.write("grammalecte-cli.py", "pythonpath/grammalecte-cli.py")
# Extension files
hZip.writestr("META-INF/manifest.xml", helpers.fileFile("gc_core/py/oxt/manifest.xml", dVars))
hZip.writestr("description.xml", helpers.fileFile("gc_core/py/oxt/description.xml", dVars))
hZip.writestr("Linguistic.xcu", helpers.fileFile("gc_core/py/oxt/Linguistic.xcu", dVars))
hZip.writestr("Grammalecte.py", helpers.fileFile("gc_core/py/oxt/Grammalecte.py", dVars))
hZip.writestr("pythonpath/helpers.py", helpers.fileFile("gc_core/py/oxt/helpers.py", dVars))
for sf in dVars["extras"].split(","):
hZip.writestr(sf.strip(), helpers.fileFile(spLang + '/' + sf.strip(), dVars))
if "logo" in dVars.keys() and dVars["logo"].strip():
hZip.write(spLang + '/' + dVars["logo"].strip(), dVars["logo"].strip())
## OPTIONS
# options dialog within LO/OO options panel (legacy)
#hZip.writestr("pythonpath/lightproof_handler_grammalecte.py", helpers.fileFile("gc_core/py/oxt/lightproof_handler_grammalecte.py", dVars))
#lLineOptions = open(spLang + "/options.txt", "r", encoding="utf-8").readlines()
#dialog_bundled.c(dVars["implname"], lLineOptions, hZip, dVars["lang"])
# options dialog
hZip.writestr("pythonpath/Options.py", helpers.fileFile("gc_core/py/oxt/Options.py", dVars))
hZip.write("gc_core/py/oxt/op_strings.py", "pythonpath/op_strings.py")
# options dialog within Writer options panel
dVars["xdl_dialog_options"] = createDialogOptionsXDL(dVars)
dVars["xcs_options"] = "\n".join([ '<prop oor:name="'+sOpt+'" oor:type="xs:string"><value></value></prop>' for sOpt in dVars["dOptPython"] ])
dVars["xcu_label_values"] = "\n".join([ '<value xml:lang="'+sLang+'">' + dVars["dOptLabel"][sLang]["__optiontitle__"] + '</value>' for sLang in dVars["dOptLabel"] ])
hZip.writestr("dialog/options_page.xdl", helpers.fileFile("gc_core/py/oxt/options_page.xdl", dVars))
hZip.writestr("dialog/OptionsDialog.xcs", helpers.fileFile("gc_core/py/oxt/OptionsDialog.xcs", dVars))
hZip.writestr("dialog/OptionsDialog.xcu", helpers.fileFile("gc_core/py/oxt/OptionsDialog.xcu", dVars))
hZip.writestr("dialog/" + dVars['lang'] + "_en.default", "")
for sLangLbl, dOptLbl in dVars['dOptLabel'].items():
hZip.writestr("dialog/" + dVars['lang'] + "_" + sLangLbl + ".properties", createOptionsLabelProperties(dOptLbl))
## ADDONS OXT
print("+ OXT: ", end="")
for spfSrc, spfDst in dOxt.items():
print(spfSrc, end=", ")
if os.path.isdir(spLang+'/'+spfSrc):
for sf in os.listdir(spLang+'/'+spfSrc):
hZip.write(spLang+'/'+spfSrc+"/"+sf, spfDst+"/"+sf)
else:
if spfSrc.endswith(('.txt', '.py')):
hZip.writestr(spfDst, helpers.fileFile(spLang+'/'+spfSrc, dVars))
else:
hZip.write(spLang+'/'+spfSrc, spfDst)
print()
hZip.close()
# Installation in Writer profile
if bInstall:
print("> installation in Writer")
if dVars.get('unopkg', False):
cmd = '"'+os.path.abspath(dVars.get('unopkg')+'" add -f '+spfZip)
print(cmd)
#subprocess.run(cmd)
os.system(cmd)
else:
print("# Error: path and filename of unopkg not set in config.ini")
def createServerOptions (sLang, dOptData):
with open("grammalecte-server-options."+sLang+".ini", "w", encoding="utf-8", newline="\n") as hDst:
hDst.write("# Server options. Lang: " + sLang + "\n\n[gc_options]\n")
for sSection, lOpt in dOptData["lStructOpt"]:
hDst.write("\n########## " + dOptData["dOptLabel"][sLang].get(sSection, sSection + "[no label found]")[0] + " ##########\n")
for lLineOpt in lOpt:
for sOpt in lLineOpt:
hDst.write("# " + dOptData["dOptLabel"][sLang].get(sOpt, "[no label found]")[0] + "\n")
hDst.write(sOpt + " = " + ("1" if dOptData["dOptServer"].get(sOpt, None) else "0") + "\n")
hDst.write("html = 1\n")
def createPackageZip (sLang, dVars, spLangPack):
"create server zip"
spfZip = "_build/" + dVars['name'] + "-"+ dVars['lang'] +"-v" + dVars['version'] + '.zip'
hZip = zipfile.ZipFile(spfZip, mode='w', compression=zipfile.ZIP_DEFLATED)
copyGrammalectePyPackageInZipFile(hZip, spLangPack)
for spf in ["grammalecte-cli.py", "grammalecte-server.py", "bottle.py", \
"grammalecte-server-options._global.ini", "grammalecte-server-options."+sLang+".ini", \
"README.txt", "LICENSE.txt", "LICENSE.fr.txt"]:
hZip.write(spf)
hZip.writestr("setup.py", helpers.fileFile("gc_lang/fr/setup.py", dVars))
def copyGrammalectePyPackageInZipFile (hZip, spLangPack, sAddPath=""):
for sf in os.listdir("grammalecte"):
if not os.path.isdir("grammalecte/"+sf):
hZip.write("grammalecte/"+sf, sAddPath+"grammalecte/"+sf)
for sf in os.listdir("grammalecte/graphspell"):
if not os.path.isdir("grammalecte/graphspell/"+sf):
hZip.write("grammalecte/graphspell/"+sf, sAddPath+"grammalecte/graphspell/"+sf)
for sf in os.listdir("grammalecte/graphspell/_dictionaries"):
if not os.path.isdir("grammalecte/graphspell/_dictionaries/"+sf):
hZip.write("grammalecte/graphspell/_dictionaries/"+sf, sAddPath+"grammalecte/graphspell/_dictionaries/"+sf)
for sf in os.listdir(spLangPack):
if not os.path.isdir(spLangPack+"/"+sf):
hZip.write(spLangPack+"/"+sf, sAddPath+spLangPack+"/"+sf)
def create (sLang, xConfig, bInstallOXT, bJavaScript):
oNow = datetime.datetime.now()
print("============== MAKE GRAMMALECTE [{0}] at {1.hour:>2} h {1.minute:>2} min {1.second:>2} s ==============".format(sLang, oNow))
#### READ CONFIGURATION
print("> read configuration...")
spLang = "gc_lang/" + sLang
dVars = xConfig._sections['args']
dVars['locales'] = dVars["locales"].replace("_", "-")
dVars['loc'] = str(dict([ [s, [s[0:2], s[3:5], ""]] for s in dVars["locales"].split(" ") ]))
## COMPILE RULES
dResult = compile_rules.make(spLang, dVars['lang'], bJavaScript)
dVars.update(dResult)
## READ GRAMMAR CHECKER PLUGINS
print("PYTHON:")
print("+ Plugins: ", end="")
sCodePlugins = ""
for sf in os.listdir(spLang+"/modules"):
if re.match(r"gce_\w+[.]py$", sf):
sCodePlugins += "\n\n" + open(spLang+'/modules/'+sf, "r", encoding="utf-8").read()
print(sf, end=", ")
print()
dVars["plugins"] = sCodePlugins
## COPY GC_CORE COMMON FILES
for sf in os.listdir("gc_core/py"):
if not os.path.isdir("gc_core/py/"+sf):
helpers.copyAndFileTemplate("gc_core/py/"+sf, "grammalecte/"+sf, dVars)
open("grammalecte/WARNING.txt", "w", encoding="utf-8", newline="\n").write(sWarningMessage)
## CREATE GRAMMAR CHECKER PACKAGE
spLangPack = "grammalecte/"+sLang
helpers.createCleanFolder(spLangPack)
for sf in os.listdir("gc_core/py/lang_core"):
if not os.path.isdir("gc_core/py/lang_core/"+sf):
helpers.copyAndFileTemplate("gc_core/py/lang_core/"+sf, spLangPack+"/"+sf, dVars)
print("+ Modules: ", end="")
for sf in os.listdir(spLang+"/modules"):
if not sf.startswith(("gce_", "__pycache__")):
helpers.copyAndFileTemplate(spLang+"/modules/"+sf, spLangPack+"/"+sf, dVars)
print(sf, end=", ")
print()
# TEST FILES
with open("grammalecte/"+sLang+"/gc_test.txt", "w", encoding="utf-8", newline="\n") as hDstPy:
hDstPy.write("# TESTS FOR LANG [" + sLang + "]\n\n")
hDstPy.write(dVars['gctests'])
createOXT(spLang, dVars, xConfig._sections['oxt'], spLangPack, bInstallOXT)
createServerOptions(sLang, dVars)
createPackageZip(sLang, dVars, spLangPack)
#### JAVASCRIPT
if bJavaScript:
print("JAVASCRIPT:")
print("+ Plugins: ", end="")
sCodePlugins = ""
for sf in os.listdir(spLang+"/modules-js"):
if re.match(r"gce_\w+[.]js$", sf):
sCodePlugins += "\n\n" + open(spLang+'/modules-js/'+sf, "r", encoding="utf-8").read()
print(sf, end=", ")
print()
dVars["pluginsJS"] = sCodePlugins
# options data struct
dVars["dOptJavaScript"] = json.dumps(list(dVars["dOptJavaScript"].items()))
dVars["dOptFirefox"] = json.dumps(list(dVars["dOptFirefox"].items()))
dVars["dOptThunderbird"] = json.dumps(list(dVars["dOptThunderbird"].items()))
# create folder
spLangPack = "grammalecte-js/"+sLang
helpers.createCleanFolder(spLangPack)
# create files
for sf in os.listdir("js_extension"):
dVars[sf[:-3]] = open("js_extension/"+sf, "r", encoding="utf-8").read()
for sf in os.listdir("gc_core/js"):
if not os.path.isdir("gc_core/js/"+sf) and not sf.startswith("jsex_"):
helpers.copyAndFileTemplate("gc_core/js/"+sf, "grammalecte-js/"+sf, dVars)
open("grammalecte-js/WARNING.txt", "w", encoding="utf-8", newline="\n").write(sWarningMessage)
for sf in os.listdir("gc_core/js/lang_core"):
if not os.path.isdir("gc_core/js/lang_core/"+sf) and sf.startswith("gc_"):
helpers.copyAndFileTemplate("gc_core/js/lang_core/"+sf, spLangPack+"/"+sf, dVars)
print("+ Modules: ", end="")
for sf in os.listdir(spLang+"/modules-js"):
if not sf.startswith("gce_"):
helpers.copyAndFileTemplate(spLang+"/modules-js/"+sf, spLangPack+"/"+sf, dVars)
print(sf, end=", ")
print()
try:
build_module = importlib.import_module("gc_lang."+sLang+".build")
except ImportError:
print("# No complementary builder <build.py> in folder gc_lang/"+sLang)
else:
build_module.build(sLang, dVars, spLangPack)
return dVars['version']
def copyGraphspellCore (bJavaScript=False):
helpers.createCleanFolder("grammalecte/graphspell")
dir_util.mkpath("grammalecte/graphspell/_dictionaries")
for sf in os.listdir("graphspell"):
if not os.path.isdir("graphspell/"+sf):
file_util.copy_file("graphspell/"+sf, "grammalecte/graphspell")
if bJavaScript:
helpers.createCleanFolder("grammalecte-js/graphspell")
dir_util.mkpath("grammalecte-js/graphspell/_dictionaries")
dVars = {}
for sf in os.listdir("js_extension"):
dVars[sf[:-3]] = open("js_extension/"+sf, "r", encoding="utf-8").read()
for sf in os.listdir("graphspell-js"):
if not os.path.isdir("graphspell-js/"+sf):
file_util.copy_file("graphspell-js/"+sf, "grammalecte-js/graphspell")
helpers.copyAndFileTemplate("graphspell-js/"+sf, "grammalecte-js/graphspell/"+sf, dVars)
def copyGraphspellDictionaries (dVars, bJavaScript=False, bExtendedDict=False, bCommunityDict=False, bPersonalDict=False):
dVars["dic_main_filename_py"] = ""
dVars["dic_main_filename_js"] = ""
dVars["dic_extended_filename_py"] = ""
dVars["dic_extended_filename_js"] = ""
dVars["dic_community_filename_py"] = ""
dVars["dic_community_filename_js"] = ""
dVars["dic_personal_filename_py"] = ""
dVars["dic_personal_filename_js"] = ""
lDict = [ ("main", s) for s in dVars['dic_filenames'].split(",") ]
if bExtendedDict:
lDict.append(("extended", dVars['dic_extended_filename']))
if bCommunityDict:
lDict.append(("community", dVars['dic_community_filename']))
if bPersonalDict:
lDict.append(("personal", dVars['dic_personal_filename']))
for sType, sFileName in lDict:
spfPyDic = "graphspell/_dictionaries/" + sFileName + ".bdic"
spfJSDic = "graphspell-js/_dictionaries/" + sFileName + ".json"
if not os.path.isfile(spfPyDic) or (bJavaScript and not os.path.isfile(spfJSDic)):
buildDictionary(dVars, sType, bJavaScript)
print(spfPyDic)
file_util.copy_file(spfPyDic, "grammalecte/graphspell/_dictionaries")
dVars['dic_'+sType+'_filename_py'] = sFileName + '.bdic'
if bJavaScript:
print(spfJSDic)
file_util.copy_file(spfJSDic, "grammalecte-js/graphspell/_dictionaries")
dVars['dic_'+sType+'_filename_js'] = sFileName + '.json'
dVars['dic_main_filename_py'] = dVars['dic_default_filename_py'] + ".bdic"
dVars['dic_main_filename_js'] = dVars['dic_default_filename_js'] + ".json"
def buildDictionary (dVars, sType, bJavaScript=False):
if sType == "main":
spfLexSrc = dVars['lexicon_src']
l_sfDictDst = dVars['dic_filenames'].split(",")
l_sDicName = dVars['dic_name'].split(",")
l_sFilter = dVars['dic_filter'].split(",")
for sfDictDst, sDicName, sFilter in zip(l_sfDictDst, l_sDicName, l_sFilter):
lex_build.build(spfLexSrc, dVars['lang'], dVars['lang_name'], sfDictDst, bJavaScript, sDicName, sFilter, dVars['stemming_method'], int(dVars['fsa_method']))
else:
if sType == "extended":
spfLexSrc = dVars['lexicon_extended_src']
sfDictDst = dVars['dic_extended_filename']
sDicName = dVars['dic_extended_name']
elif sType == "community":
spfLexSrc = dVars['lexicon_community_src']
sfDictDst = dVars['dic_community_filename']
sDicName = dVars['dic_community_name']
elif sType == "personal":
spfLexSrc = dVars['lexicon_personal_src']
sfDictDst = dVars['dic_personal_filename']
sDicName = dVars['dic_personal_name']
lex_build.build(spfLexSrc, dVars['lang'], dVars['lang_name'], sfDictDst, bJavaScript, sDicName, "", dVars['stemming_method'], int(dVars['fsa_method']))
def main ():
print("Python: " + sys.version)
xParser = argparse.ArgumentParser()
xParser.add_argument("lang", type=str, nargs='+', help="lang project to generate (name of folder in /lang)")
xParser.add_argument("-b", "--build_data", help="launch build_data.py (part 1 and 2)", action="store_true")
xParser.add_argument("-bb", "--build_data_before", help="launch build_data.py (only part 1: before dictionary building)", action="store_true")
xParser.add_argument("-ba", "--build_data_after", help="launch build_data.py (only part 2: before dictionary building)", action="store_true")
xParser.add_argument("-d", "--dict", help="generate FSA dictionary", action="store_true")
xParser.add_argument("-t", "--tests", help="run unit tests", action="store_true")
xParser.add_argument("-p", "--perf", help="run performance tests", action="store_true")
xParser.add_argument("-pm", "--perf_memo", help="run performance tests and store results in perf_memo.txt", action="store_true")
xParser.add_argument("-js", "--javascript", help="JavaScript build for Firefox", action="store_true")
xParser.add_argument("-aed", "--add_extended_dictionary", help="add extended dictionary to the build", action="store_true")
xParser.add_argument("-acd", "--add_community_dictionary", help="add community dictionary to the build", action="store_true")
xParser.add_argument("-apd", "--add_personal_dictionary", help="add personal dictionary to the build", action="store_true")
xParser.add_argument("-fx", "--firefox", help="Launch Firefox Developper for WebExtension testing", action="store_true")
xParser.add_argument("-we", "--web_ext", help="Launch Firefox Nightly for WebExtension testing", action="store_true")
xParser.add_argument("-l", "--lint_web_ext", help="web-ext lint on the WebExtension", action="store_true")
xParser.add_argument("-tb", "--thunderbird", help="Launch Thunderbird", action="store_true")
xParser.add_argument("-i", "--install", help="install the extension in Writer (path of unopkg must be set in config.ini)", action="store_true")
xArgs = xParser.parse_args()
if xArgs.build_data:
xArgs.build_data_before = True
xArgs.build_data_after = True
dir_util.mkpath("_build")
dir_util.mkpath("grammalecte")
if xArgs.javascript:
dir_util.mkpath("grammalecte-js")
copyGraphspellCore(xArgs.javascript)
for sLang in xArgs.lang:
if os.path.exists("gc_lang/"+sLang) and os.path.isdir("gc_lang/"+sLang):
xConfig = getConfig(sLang)
dVars = xConfig._sections['args']
if not dVars["lexicon_extended_src"]:
xArgs.add_extended_dictionary = False
if not dVars["lexicon_community_src"]:
xArgs.add_community_dictionary = False
if not dVars["lexicon_personal_src"]:
xArgs.add_personal_dictionary = False
# build data
build_data_module = None
if xArgs.build_data_before or xArgs.build_data_after:
# lang data
try:
build_data_module = importlib.import_module("gc_lang."+sLang+".build_data")
except ImportError:
print("# Error. Couldn’t import file build_data.py in folder gc_lang/"+sLang)
if build_data_module and xArgs.build_data_before:
build_data_module.before('gc_lang/'+sLang, dVars, xArgs.javascript)
if xArgs.dict:
buildDictionary(dVars, "main", xArgs.javascript)
if xArgs.add_extended_dictionary:
buildDictionary(dVars, "extended", xArgs.javascript)
if xArgs.add_community_dictionary:
buildDictionary(dVars, "community", xArgs.javascript)
if xArgs.add_personal_dictionary:
buildDictionary(dVars, "personal", xArgs.javascript)
if build_data_module and xArgs.build_data_after:
build_data_module.after('gc_lang/'+sLang, dVars, xArgs.javascript)
# copy dictionaries from Graphspell
copyGraphspellDictionaries(dVars, xArgs.javascript, xArgs.add_extended_dictionary, xArgs.add_community_dictionary, xArgs.add_personal_dictionary)
# make
sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, )
# tests
if xArgs.tests or xArgs.perf or xArgs.perf_memo:
print("> Running tests")
try:
tests = importlib.import_module("grammalecte."+sLang+".tests")
print(tests.__file__)
except ImportError:
print("# Error. Import failed:" + "grammalecte."+sLang+".tests")
traceback.print_exc()
else:
if xArgs.tests:
xTestSuite = unittest.TestLoader().loadTestsFromModule(tests)
unittest.TextTestRunner().run(xTestSuite)
if xArgs.perf or xArgs.perf_memo:
hDst = open("./gc_lang/"+sLang+"/perf_memo.txt", "a", encoding="utf-8", newline="\n") if xArgs.perf_memo else None
tests.perf(sVersion, hDst)
# Firefox
if False:
# obsolete
with helpers.cd("_build/xpi/"+sLang):
spfFirefox = dVars['win_fx_dev_path'] if platform.system() == "Windows" else dVars['linux_fx_dev_path']
os.system('jpm run -b "' + spfFirefox + '"')
if xArgs.web_ext or xArgs.firefox:
with helpers.cd("_build/webext/"+sLang):
if xArgs.lint_web_ext:
os.system(r'web-ext lint -o text')
if xArgs.firefox:
# Firefox Developper edition
spfFirefox = dVars['win_fx_dev_path'] if platform.system() == "Windows" else dVars['linux_fx_dev_path']
else:
# Firefox Nightly edition
spfFirefox = dVars['win_fx_nightly_path'] if platform.system() == "Windows" else dVars['linux_fx_nightly_path']
os.system(r'web-ext run --firefox="' + spfFirefox + '" --browser-console --firefox-profile=debug')
# https://github.com/mozilla/web-ext/issues/932
# os.system(r'web-ext run --firefox="' + spfFirefox + r'" --browser-console --firefox-profile=C:\Users\EAK\AppData\Roaming\Mozilla\Firefox\Profiles\e26559tw.debug --keep-profile-changes')
# Thunderbird
if xArgs.thunderbird:
os.system("thunderbird -jsconsole -P debug")
else:
print("Folder not found: gc_lang/"+sLang)
oNow = datetime.datetime.now()
print("============== MAKE GRAMMALECTE [finished] at {0.hour:>2} h {0.minute:>2} min {0.second:>2} s ==============".format(oNow))
if __name__ == '__main__':
main()