#!/usr/bin/env python3
# coding: UTF-8
"""
Grammalecte builder
"""
import sys
import os
import re
import zipfile
import traceback
import configparser
import datetime
import argparse
import importlib
import unittest
import json
import platform
from distutils import dir_util, file_util
#import dialog_bundled
import compile_rules
import helpers
import lex_build
sWarningMessage = "The content of this folder is generated by code and replaced at each build.\n"
def getConfig (sLang):
"load config.ini in <sLang> at gc_lang/<sLang>, returns xConfigParser object"
xConfig = configparser.ConfigParser()
xConfig.optionxform = str
try:
xConfig.read_file(open(f"gc_lang/{sLang}/config.ini", "r", encoding="utf-8"))
except FileNotFoundError:
print(f"# Error. Can’t read config file <{sLang}>")
exit()
return xConfig
def createOptionsLabelProperties (dOptLbl):
"create content for .properties files (LibreOffice)"
sContent = ""
for sOpt, tLabel in dOptLbl.items():
sContent += f"{sOpt}={tLabel[0]}\n"
if tLabel[1]:
sContent += f"hlp_{sOpt}={tLabel[1]}\n"
return sContent
def createDialogOptionsXDL (dVars):
"create bundled dialog options file .xdl (LibreOffice)"
iTab = 1
nPosY = 5
nWidth = 240
sContent = ""
dOpt = dVars["dOptWriter"]
dOptLabel = dVars["dOptLabel"][dVars["lang"]]
for sGroup, lGroupOptions in dVars["lStructOpt"]:
sContent += f'<dlg:fixedline dlg:id="{sGroup}" dlg:tab-index="{iTab}" dlg:top="{nPosY}" dlg:left="5" dlg:width="{nWidth}" dlg:height="10" dlg:value="&{sGroup}" />\n'
iTab += 1
for lLineOptions in lGroupOptions:
nElemWidth = nWidth // len(lLineOptions)
nPosY += 10
nPosX = 10
for sOpt in lLineOptions:
sHelp = f'dlg:help-text="&hlp_{sOpt}"' if dOptLabel[sOpt][1] else ""
sChecked = "true" if dOpt[sOpt] else "false"
sContent += f'<dlg:checkbox dlg:id="{sOpt}" dlg:tab-index="{iTab}" dlg:top="{nPosY}" dlg:left="{nPosX}" dlg:width="{nElemWidth}" dlg:height="10" dlg:value="&{sOpt}" dlg:checked="{sChecked}" {sHelp} />\n'
iTab += 1
nPosX += nElemWidth
nPosY += 10
return sContent
def createOXT (spLang, dVars, dOxt, spLangPack, bInstall):
"create extension for Writer"
print("Building extension for Writer")
spfZip = f"_build/{dVars['name']}-{dVars['lang']}-v{dVars['version']}.oxt"
hZip = zipfile.ZipFile(spfZip, mode='w', compression=zipfile.ZIP_DEFLATED)
# Package and parser
copyGrammalectePyPackageInZipFile(hZip, spLangPack, "pythonpath/")
hZip.write("grammalecte-cli.py", "pythonpath/grammalecte-cli.py")
# Extension files
hZip.writestr("META-INF/manifest.xml", helpers.fileFile("gc_core/py/oxt/manifest.xml", dVars))
hZip.writestr("description.xml", helpers.fileFile("gc_core/py/oxt/description.xml", dVars))
hZip.writestr("Linguistic.xcu", helpers.fileFile("gc_core/py/oxt/Linguistic.xcu", dVars))
hZip.writestr("Grammalecte.py", helpers.fileFile("gc_core/py/oxt/Grammalecte.py", dVars))
hZip.writestr("pythonpath/helpers.py", helpers.fileFile("gc_core/py/oxt/helpers.py", dVars))
for sf in dVars["extras"].split(","):
hZip.writestr(sf.strip(), helpers.fileFile(spLang + '/' + sf.strip(), dVars))
if "logo" in dVars.keys() and dVars["logo"].strip():
hZip.write(spLang + '/' + dVars["logo"].strip(), dVars["logo"].strip())
## OPTIONS
# options dialog within LO/OO options panel (legacy)
#hZip.writestr("pythonpath/lightproof_handler_grammalecte.py", helpers.fileFile("gc_core/py/oxt/lightproof_handler_grammalecte.py", dVars))
#lLineOptions = open(spLang + "/options.txt", "r", encoding="utf-8").readlines()
#dialog_bundled.c(dVars["implname"], lLineOptions, hZip, dVars["lang"])
# options dialog
hZip.writestr("pythonpath/Options.py", helpers.fileFile("gc_core/py/oxt/Options.py", dVars))
hZip.write("gc_core/py/oxt/op_strings.py", "pythonpath/op_strings.py")
# options dialog within Writer options panel
dVars["xdl_dialog_options"] = createDialogOptionsXDL(dVars)
dVars["xcs_options"] = "\n".join([ '<prop oor:name="'+sOpt+'" oor:type="xs:string"><value></value></prop>' for sOpt in dVars["dOptPython"] ])
dVars["xcu_label_values"] = "\n".join([ '<value xml:lang="'+sLang+'">' + dVars["dOptLabel"][sLang]["__optiontitle__"] + '</value>' for sLang in dVars["dOptLabel"] ])
hZip.writestr("dialog/options_page.xdl", helpers.fileFile("gc_core/py/oxt/options_page.xdl", dVars))
hZip.writestr("dialog/OptionsDialog.xcs", helpers.fileFile("gc_core/py/oxt/OptionsDialog.xcs", dVars))
hZip.writestr("dialog/OptionsDialog.xcu", helpers.fileFile("gc_core/py/oxt/OptionsDialog.xcu", dVars))
hZip.writestr("dialog/" + dVars['lang'] + "_en.default", "")
for sLangLbl, dOptLbl in dVars['dOptLabel'].items():
hZip.writestr(f"dialog/{dVars['lang']}_{sLangLbl}.properties", createOptionsLabelProperties(dOptLbl))
## ADDONS OXT
print("+ OXT: ", end="")
for spfSrc, spfDst in dOxt.items():
print(spfSrc, end=", ")
if os.path.isdir(spLang+'/'+spfSrc):
for sf in os.listdir(spLang+'/'+spfSrc):
if sf.endswith(('.txt', '.py')):
hZip.writestr(spfDst+"/"+sf, helpers.fileFile(spLang+'/'+spfSrc+"/"+sf, dVars))
else:
hZip.write(spLang+'/'+spfSrc+"/"+sf, spfDst+"/"+sf)
else:
if spfSrc.endswith(('.txt', '.py')):
hZip.writestr(spfDst, helpers.fileFile(spLang+'/'+spfSrc, dVars))
else:
hZip.write(spLang+'/'+spfSrc, spfDst)
print()
hZip.close()
# Installation in Writer profile
if bInstall:
print("> installation in Writer")
if dVars.get('unopkg', False):
cmd = '"'+os.path.abspath(dVars.get('unopkg')+'" add -f '+spfZip)
print(cmd)
os.system(cmd)
else:
print("# Error: path and filename of unopkg not set in config.ini")
def createPackageZip (dVars, spLangPack):
"create server zip"
spfZip = f"_build/{dVars['name']}-{dVars['lang']}-v{dVars['version']}.zip"
hZip = zipfile.ZipFile(spfZip, mode='w', compression=zipfile.ZIP_DEFLATED)
copyGrammalectePyPackageInZipFile(hZip, spLangPack)
for spf in ["grammalecte-cli.py", "grammalecte-server.py", \
"README.txt", "LICENSE.txt", "LICENSE.fr.txt"]:
hZip.write(spf)
hZip.writestr("setup.py", helpers.fileFile("gc_lang/fr/setup.py", dVars))
def copyGrammalectePyPackageInZipFile (hZip, spLangPack, sAddPath=""):
"copy Grammalecte Python package in zip file"
for sf in os.listdir("grammalecte"):
if not os.path.isdir("grammalecte/"+sf):
hZip.write("grammalecte/"+sf, sAddPath+"grammalecte/"+sf)
for sf in os.listdir("grammalecte/graphspell"):
if not os.path.isdir("grammalecte/graphspell/"+sf):
hZip.write("grammalecte/graphspell/"+sf, sAddPath+"grammalecte/graphspell/"+sf)
for sf in os.listdir("grammalecte/graphspell/_dictionaries"):
if not os.path.isdir("grammalecte/graphspell/_dictionaries/"+sf):
hZip.write("grammalecte/graphspell/_dictionaries/"+sf, sAddPath+"grammalecte/graphspell/_dictionaries/"+sf)
for sf in os.listdir(spLangPack):
if not os.path.isdir(spLangPack+"/"+sf):
hZip.write(spLangPack+"/"+sf, sAddPath+spLangPack+"/"+sf)
def create (sLang, xConfig, bInstallOXT, bJavaScript, bUseCache):
"make Grammalecte for project <sLang>"
oNow = datetime.datetime.now()
print("============== MAKE GRAMMALECTE [{0}] at {1.hour:>2} h {1.minute:>2} min {1.second:>2} s ==============".format(sLang, oNow))
#### READ CONFIGURATION
print("> read configuration...")
spLang = "gc_lang/" + sLang
dVars = xConfig._sections['args']
dVars['locales'] = dVars["locales"].replace("_", "-")
dVars['loc'] = str({ s: [s[0:2], s[3:5], ""] for s in dVars["locales"].split(" ") })
## COMPILE RULES
dResult = compile_rules.make(spLang, dVars['lang'], bUseCache)
dVars.update(dResult)
## READ GRAMMAR CHECKER PLUGINS
print("PYTHON:")
print("+ Plugins: ", end="")
sCodePlugins = ""
for sf in os.listdir(spLang+"/modules"):
if re.match(r"gce_\w+[.]py$", sf):
sCodePlugins += "\n\n" + open(spLang+'/modules/'+sf, "r", encoding="utf-8").read()
print(sf, end=", ")
print()
dVars["plugins"] = sCodePlugins
## COPY GC_CORE COMMON FILES
for sf in os.listdir("gc_core/py"):
if not os.path.isdir("gc_core/py/"+sf):
helpers.copyAndFileTemplate("gc_core/py/"+sf, "grammalecte/"+sf, dVars)
file_util.copy_file("3rd/bottle.py", "grammalecte/bottle.py")
open("grammalecte/WARNING.txt", "w", encoding="utf-8", newline="\n").write(sWarningMessage)
## CREATE GRAMMAR CHECKER PACKAGE
spLangPack = "grammalecte/"+sLang
helpers.createCleanFolder(spLangPack)
for sf in os.listdir("gc_core/py/lang_core"):
if not os.path.isdir("gc_core/py/lang_core/"+sf):
helpers.copyAndFileTemplate("gc_core/py/lang_core/"+sf, spLangPack+"/"+sf, dVars)
print("+ Modules: ", end="")
for sf in os.listdir(spLang+"/modules"):
if not sf.startswith(("gce_", "__pycache__")):
helpers.copyAndFileTemplate(spLang+"/modules/"+sf, spLangPack+"/"+sf, dVars)
print(sf, end=", ")
print()
# TEST FILES
with open("grammalecte/"+sLang+"/gc_test.txt", "w", encoding="utf-8", newline="\n") as hDstPy:
hDstPy.write("# TESTS FOR LANG [" + sLang + "]\n\n")
hDstPy.write(dVars['gctests'])
hDstPy.write("\n")
createOXT(spLang, dVars, xConfig._sections['oxt'], spLangPack, bInstallOXT)
createPackageZip(dVars, spLangPack)
#### JAVASCRIPT
if bJavaScript:
print("JAVASCRIPT:")
print("+ Plugins: ", end="")
sCodePlugins = ""
for sf in os.listdir(spLang+"/modules-js"):
if re.match(r"gce_\w+[.]js$", sf):
sCodePlugins += "\n\n" + open(spLang+'/modules-js/'+sf, "r", encoding="utf-8").read()
print(sf, end=", ")
print()
dVars["pluginsJS"] = sCodePlugins
# options data struct
dVars["dOptJavaScript"] = json.dumps(list(dVars["dOptJavaScript"].items()))
dVars["dOptFirefox"] = json.dumps(list(dVars["dOptFirefox"].items()))
dVars["dOptThunderbird"] = json.dumps(list(dVars["dOptThunderbird"].items()))
# create folder
spLangPack = "grammalecte-js/"+sLang
helpers.createCleanFolder(spLangPack)
# create files
for sf in os.listdir("js_extension"):
dVars[sf[:-3]] = open("js_extension/"+sf, "r", encoding="utf-8").read()
for sf in os.listdir("gc_core/js"):
if not os.path.isdir("gc_core/js/"+sf) and not sf.startswith("jsex_"):
helpers.copyAndFileTemplate("gc_core/js/"+sf, "grammalecte-js/"+sf, dVars)
open("grammalecte-js/WARNING.txt", "w", encoding="utf-8", newline="\n").write(sWarningMessage)
for sf in os.listdir("gc_core/js/lang_core"):
if not os.path.isdir("gc_core/js/lang_core/"+sf) and sf.startswith("gc_"):
helpers.copyAndFileTemplate("gc_core/js/lang_core/"+sf, spLangPack+"/"+sf, dVars)
print("+ Modules: ", end="")
for sf in os.listdir(spLang+"/modules-js"):
if not sf.startswith("gce_"):
helpers.copyAndFileTemplate(spLang+"/modules-js/"+sf, spLangPack+"/"+sf, dVars)
print(sf, end=", ")
print()
try:
buildjs = importlib.import_module("gc_lang."+sLang+".build")
except ImportError:
print("# No complementary builder <build.py> in folder gc_lang/"+sLang)
else:
buildjs.build(sLang, dVars)
return dVars['version']
def copyGraphspellCore (bJavaScript=False):
"copy Graphspell package in Grammalecte package"
helpers.createCleanFolder("grammalecte/graphspell")
dir_util.mkpath("grammalecte/graphspell/_dictionaries")
for sf in os.listdir("graphspell"):
if not os.path.isdir("graphspell/"+sf):
file_util.copy_file("graphspell/"+sf, "grammalecte/graphspell")
if bJavaScript:
helpers.createCleanFolder("grammalecte-js/graphspell")
dir_util.mkpath("grammalecte-js/graphspell/_dictionaries")
dVars = {}
for sf in os.listdir("js_extension"):
dVars[sf[:-3]] = open("js_extension/"+sf, "r", encoding="utf-8").read()
for sf in os.listdir("graphspell-js"):
if not os.path.isdir("graphspell-js/"+sf):
file_util.copy_file("graphspell-js/"+sf, "grammalecte-js/graphspell")
helpers.copyAndFileTemplate("graphspell-js/"+sf, "grammalecte-js/graphspell/"+sf, dVars)
def copyGraphspellDictionaries (dVars, bJavaScript=False, bCommunityDict=False, bPersonalDict=False):
"copy requested Graphspell dictionaries in Grammalecte package"
dVars["dic_main_filename_py"] = ""
dVars["dic_main_filename_js"] = ""
dVars["dic_community_filename_py"] = ""
dVars["dic_community_filename_js"] = ""
dVars["dic_personal_filename_py"] = ""
dVars["dic_personal_filename_js"] = ""
lDict = [ ("main", s) for s in dVars['dic_filenames'].split(",") ]
if bCommunityDict:
lDict.append(("community", dVars['dic_community_filename']))
if bPersonalDict:
lDict.append(("personal", dVars['dic_personal_filename']))
for sType, sFileName in lDict:
spfPyDic = f"graphspell/_dictionaries/{sFileName}.bdic"
spfJSDic = f"graphspell-js/_dictionaries/{sFileName}.json"
if not os.path.isfile(spfPyDic) or (bJavaScript and not os.path.isfile(spfJSDic)):
buildDictionary(dVars, sType, bJavaScript)
print(spfPyDic)
file_util.copy_file(spfPyDic, "grammalecte/graphspell/_dictionaries")
dVars['dic_'+sType+'_filename_py'] = sFileName + '.bdic'
if bJavaScript:
print(spfJSDic)
file_util.copy_file(spfJSDic, "grammalecte-js/graphspell/_dictionaries")
dVars['dic_'+sType+'_filename_js'] = sFileName + '.json'
dVars['dic_main_filename_py'] = dVars['dic_default_filename_py'] + ".bdic"
dVars['dic_main_filename_js'] = dVars['dic_default_filename_js'] + ".json"
def buildDictionary (dVars, sType, bJavaScript=False):
"build binary dictionary for Graphspell from lexicons"
if sType == "main":
spfLexSrc = dVars['lexicon_src']
lSfDictDst = dVars['dic_filenames'].split(",")
lDicName = dVars['dic_name'].split(",")
lDescription = dVars['dic_description'].split(",")
lFilter = dVars['dic_filter'].split(",")
for sfDictDst, sDicName, sDescription, sFilter in zip(lSfDictDst, lDicName, lDescription, lFilter):
lex_build.build(spfLexSrc, dVars['lang'], dVars['lang_name'], sfDictDst, bJavaScript, sDicName, sDescription, sFilter, dVars['stemming_method'], int(dVars['fsa_method']))
else:
if sType == "community":
spfLexSrc = dVars['lexicon_community_src']
sfDictDst = dVars['dic_community_filename']
sDicName = dVars['dic_community_name']
sDescription = dVars['dic_community_description']
elif sType == "personal":
spfLexSrc = dVars['lexicon_personal_src']
sfDictDst = dVars['dic_personal_filename']
sDicName = dVars['dic_personal_name']
sDescription = dVars['dic_personal_description']
lex_build.build(spfLexSrc, dVars['lang'], dVars['lang_name'], sfDictDst, bJavaScript, sDicName, sDescription, "", dVars['stemming_method'], int(dVars['fsa_method']))
def main ():
"build Grammalecte with requested options"
print("Python: " + sys.version)
if sys.version < "3.7":
print("Python 3.7+ required")
return
xParser = argparse.ArgumentParser()
xParser.add_argument("lang", type=str, nargs='+', help="lang project to generate (name of folder in /lang)")
xParser.add_argument("-uc", "--use_cache", help="use data cache instead of rebuilding rules", action="store_true")
xParser.add_argument("-frb", "--force_rebuild", help="force rebuilding rules", action="store_true")
xParser.add_argument("-b", "--build_data", help="launch build_data.py (part 1 and 2)", action="store_true")
xParser.add_argument("-bb", "--build_data_before", help="launch build_data.py (only part 1: before dictionary building)", action="store_true")
xParser.add_argument("-ba", "--build_data_after", help="launch build_data.py (only part 2: before dictionary building)", action="store_true")
xParser.add_argument("-d", "--dict", help="generate FSA dictionary", action="store_true")
xParser.add_argument("-t", "--tests", help="run unit tests", action="store_true")
xParser.add_argument("-p", "--perf", help="run performance tests", action="store_true")
xParser.add_argument("-pm", "--perf_memo", help="run performance tests and store results in perf_memo.txt", action="store_true")
xParser.add_argument("-js", "--javascript", help="JavaScript build for Firefox", action="store_true")
xParser.add_argument("-acd", "--add_community_dictionary", help="add community dictionary to the build", action="store_true")
xParser.add_argument("-apd", "--add_personal_dictionary", help="add personal dictionary to the build", action="store_true")
xParser.add_argument("-fx", "--firefox", help="Launch Firefox Developper for WebExtension testing", action="store_true")
xParser.add_argument("-fxn", "--firefox_nightly", help="Launch Firefox Nightly for WebExtension testing", action="store_true")
xParser.add_argument("-l", "--lint_web_ext", help="web-ext lint on the WebExtension", action="store_true")
xParser.add_argument("-tb", "--thunderbird", help="Launch Thunderbird", action="store_true")
xParser.add_argument("-tbb", "--thunderbird_beta", help="Launch Thunderbird Beta", action="store_true")
xParser.add_argument("-i", "--install", help="install the extension in Writer (path of unopkg must be set in config.ini)", action="store_true")
xArgs = xParser.parse_args()
if xArgs.build_data:
xArgs.build_data_before = True
xArgs.build_data_after = True
dir_util.mkpath("_build")
dir_util.mkpath("grammalecte")
if xArgs.javascript:
dir_util.mkpath("grammalecte-js")
copyGraphspellCore(xArgs.javascript)
for sLang in xArgs.lang:
if os.path.exists("gc_lang/"+sLang) and os.path.isdir("gc_lang/"+sLang):
xConfig = getConfig(sLang)
dVars = xConfig._sections['args']
if not dVars["lexicon_community_src"]:
xArgs.add_community_dictionary = False
if not dVars["lexicon_personal_src"]:
xArgs.add_personal_dictionary = False
# build data
databuild = None
if xArgs.build_data_before or xArgs.build_data_after:
# lang data
try:
databuild = importlib.import_module("gc_lang."+sLang+".build_data")
except ImportError:
print("# Error. Couldn’t import file build_data.py in folder gc_lang/"+sLang)
if databuild and xArgs.build_data_before:
databuild.before('gc_lang/'+sLang, dVars, xArgs.javascript)
if xArgs.dict:
buildDictionary(dVars, "main", xArgs.javascript)
if xArgs.add_community_dictionary:
buildDictionary(dVars, "community", xArgs.javascript)
if xArgs.add_personal_dictionary:
buildDictionary(dVars, "personal", xArgs.javascript)
if databuild and xArgs.build_data_after:
databuild.after('gc_lang/'+sLang, dVars, xArgs.javascript)
# copy dictionaries from Graphspell
copyGraphspellDictionaries(dVars, xArgs.javascript, xArgs.add_community_dictionary, xArgs.add_personal_dictionary)
# make
bUseCache = None # we may rebuild if it’s necessary
if xArgs.use_cache:
bUseCache = True # we use the cache if it exists
if xArgs.force_rebuild:
bUseCache = False # we rebuild
sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, bUseCache)
# tests
if xArgs.tests or xArgs.perf or xArgs.perf_memo:
print("> Running tests")
try:
tests = importlib.import_module("grammalecte."+sLang+".tests")
print(tests.__file__)
except ImportError:
print(f"# Error. Import failed: grammalecte.{sLang}.tests")
traceback.print_exc()
else:
if xArgs.tests:
xTestSuite = unittest.TestLoader().loadTestsFromModule(tests)
unittest.TextTestRunner().run(xTestSuite)
if xArgs.perf or xArgs.perf_memo:
hDst = open("./gc_lang/"+sLang+"/perf_memo.txt", "a", encoding="utf-8", newline="\n") if xArgs.perf_memo else None
tests.perf(sVersion, hDst)
# JavaScript linter
if xArgs.lint_web_ext:
with helpers.CD("_build/webext/"+sLang):
os.system(r'web-ext lint -o text')
# Firefox
if xArgs.firefox: # Firefox Developer edition
with helpers.CD("_build/webext/"+sLang):
spfFirefox = dVars['win_fx_dev_path'] if platform.system() == "Windows" else dVars['linux_fx_dev_path']
os.system(r'web-ext run --firefox="' + spfFirefox + '" --browser-console')
if xArgs.firefox_nightly: # Firefox Nightly edition
with helpers.CD("_build/webext/"+sLang):
spfFirefox = dVars['win_fx_nightly_path'] if platform.system() == "Windows" else dVars['linux_fx_nightly_path']
os.system(r'web-ext run --firefox="' + spfFirefox + '" --browser-console')
# https://github.com/mozilla/web-ext/issues/932
# os.system(r'web-ext run --firefox="' + spfFirefox + r'" --browser-console --firefox-profile=C:\Users\EAK\AppData\Roaming\Mozilla\Firefox\Profiles\e26559tw.debug --keep-profile-changes')
# Thunderbird
if xArgs.thunderbird:
spfThunderbird = '"'+dVars['win_tb_path']+'"' if platform.system() == "Windows" else dVars['linux_tb_path']
print(spfThunderbird)
os.system(spfThunderbird + ' -jsconsole -P debug')
if xArgs.thunderbird_beta:
spfThunderbird = '"'+dVars['win_tb_beta_path']+'"' if platform.system() == "Windows" else dVars['linux_tb_beta_path']
print(spfThunderbird)
os.system(spfThunderbird + ' -jsconsole -P beta')
else:
print("Folder not found: gc_lang/"+sLang)
oNow = datetime.datetime.now()
print("============== MAKE GRAMMALECTE [finished] at {0.hour:>2} h {0.minute:>2} min {0.second:>2} s ==============".format(oNow))
if __name__ == '__main__':
main()