Index: compile_rules.py
==================================================================
--- compile_rules.py
+++ compile_rules.py
@@ -137,11 +137,11 @@
def createRule (s, nIdLine, sLang, bParagraph, dOptPriority):
"returns rule as list [option name, regex, bCaseInsensitive, identifier, list of actions]"
global dJSREGEXES
- sLineId = "#" + str(nIdLine) + ("p" if bParagraph else "s")
+ sLineId = f"#{nIdLine}" + ("p" if bParagraph else "s")
sRuleId = sLineId
#### GRAPH CALL
if s.startswith("@@@@"):
if bParagraph:
@@ -162,25 +162,25 @@
cCaseMode = m.group('borders_and_case')[1]
cWordLimitRight = m.group('borders_and_case')[2]
sOption = m.group('option')[1:] if m.group('option') else False
sRuleId = m.group('ruleid')[1:-1]
if sRuleId in aRULESET:
- print("# Error. Several rules have the same id: " + sRuleId)
+ print(f"# Error. Several rules have the same id: {sRuleId}")
exit()
aRULESET.add(sRuleId)
nPriority = dOptPriority.get(sOption, 4)
if m.group('priority'):
nPriority = int(m.group('priority')[1:])
s = s[m.end(0):]
else:
- print("# Warning. Rule wrongly shaped at line: " + sLineId)
+ print(f"# Warning. Rule wrongly shaped at line: {sLineId}")
exit()
#### REGEX TRIGGER
i = s.find(" <<-")
if i == -1:
- print("# Error: no condition at line " + sLineId)
+ print(f"# Error: no condition at line {sLineId}")
return None
sRegex = s[:i].strip()
s = s[i+4:]
# JS groups positioning codes
@@ -192,11 +192,11 @@
m = re.search(".+i?", sRegex)
if m:
dJSREGEXES[sLineId] = m.group(0)
sRegex = sRegex[:m.start()].strip()
if "" in sRegex or "" in sRegex:
- print("# Error: JavaScript regex not delimited at line " + sLineId)
+ print(f"# Error: JavaScript regex not delimited at line {sLineId}")
return None
# quotes ?
if sRegex.startswith('"') and sRegex.endswith('"'):
sRegex = sRegex[1:-1]
@@ -207,14 +207,14 @@
## count number of groups (must be done before modifying the regex)
nGroup = countGroupInRegex(sRegex)
if nGroup > 0:
if not tGroups:
- print("# Warning: groups positioning code for JavaScript should be defined at line " + sLineId)
+ print(f"# Warning: groups positioning code for JavaScript should be defined at line {sLineId}")
else:
if nGroup != len(tGroups):
- print("# Error: groups positioning code irrelevant at line " + sLineId)
+ print(f"# Error: groups positioning code irrelevant at line {sLineId}")
## word limit
if cWordLimitLeft == '[' and not sRegex.startswith(("^", '’', "'", ",")):
sRegex = sWORDLIMITLEFT + sRegex
if cWordLimitRight == ']' and not sRegex.endswith(("$", '’', "'", ",")):
@@ -231,22 +231,22 @@
elif cCaseMode == "u":
bCaseInsensitive = False
sRegex = sRegex.replace("(?i)", "")
sRegex = uppercase(sRegex, sLang)
else:
- print("# Unknown case mode [" + cCaseMode + "] at line " + sLineId)
+ print(f"# Unknown case mode [{cCaseMode}] at line {sLineId}")
## check regex
try:
re.compile(sRegex)
except re.error:
- print("# Regex error at line ", nIdLine)
+ print(f"# Regex error at line {sLineId}")
print(sRegex)
return None
## groups in non grouping parenthesis
for _ in re.finditer(r"\(\?:[^)]*\([\[\w -]", sRegex):
- print("# Warning: groups inside non grouping parenthesis in regex at line " + sLineId)
+ print(f"# Warning: groups inside non grouping parenthesis in regex at line {sLineId}")
#### PARSE ACTIONS
lActions = []
nAction = 1
for sAction in s.split(" <<- "):
@@ -262,26 +262,26 @@
def checkReferenceNumbers (sText, sActionId, nToken):
"check if token references in greater than (debugging)"
for x in re.finditer(r"\\(\d+)", sText):
if int(x.group(1)) > nToken:
- print("# Error in token index at line " + sActionId + " ("+str(nToken)+" tokens only)")
+ print(f"# Error in token index at line {sActionId} ({nToken} tokens only)")
print(sText)
def checkIfThereIsCode (sText, sActionId):
"check if there is code in (debugging)"
if re.search("[.]\\w+[(]|sugg\\w+[(]|\\([0-9]|\\[[0-9]", sText):
- print("# Warning at line " + sActionId + ": This message looks like code. Line should probably begin with =")
+ print(f"# Warning at line {sActionId}: This message looks like code. Line should probably begin with =")
print(sText)
def createAction (sIdAction, sAction, nGroup):
"returns an action to perform as a tuple (condition, action type, action[, iGroup [, message, URL ]])"
m = re.search(r"([-~=>])(\d*|)>>", sAction)
if not m:
- print("# No action at line " + sIdAction)
+ print(f"# No action at line {sIdAction}")
return None
#### CONDITION
sCondition = sAction[:m.start()].strip()
if sCondition:
@@ -295,11 +295,11 @@
sCondition = None
#### iGroup / positioning
iGroup = int(m.group(2)) if m.group(2) else 0
if iGroup > nGroup:
- print("# Selected group > group number in regex at line " + sIdAction)
+ print(f"# Selected group > group number in regex at line {sIdAction}")
#### ACTION
sAction = sAction[m.end():].strip()
cAction = m.group(1)
if cAction == "-":
@@ -306,11 +306,11 @@
## error
iMsg = sAction.find(" # ")
if iMsg == -1:
sMsg = "# Error. Error message not found."
sURL = ""
- print(sMsg + " Action id: " + sIdAction)
+ print(f"# No message. Action id: {sIdAction}")
else:
sMsg = sAction[iMsg+3:].strip()
sAction = sAction[:iMsg].strip()
sURL = ""
mURL = re.search("[|] *(https?://.*)", sMsg)
@@ -335,11 +335,11 @@
if cAction == ">":
## no action, break loop if condition is False
return [sCondition, cAction, ""]
if not sAction:
- print("# Error in action at line " + sIdAction + ": This action is empty.")
+ print(f"# Error in action at line {sIdAction}: This action is empty.")
return None
if cAction == "-":
## error detected --> suggestion
if sAction[0:1] == "=":
@@ -346,11 +346,11 @@
lFUNCTIONS.append(("_s_"+sIdAction, sAction[1:]))
sAction = "=_s_"+sIdAction
elif sAction.startswith('"') and sAction.endswith('"'):
sAction = sAction[1:-1]
if not sMsg:
- print("# Error in action at line " + sIdAction + ": the message is empty.")
+ print(f"# Error in action at line {sIdAction}: the message is empty.")
return [sCondition, cAction, sAction, iGroup, sMsg, sURL]
if cAction == "~":
## text processor
if sAction[0:1] == "=":
lFUNCTIONS.append(("_p_"+sIdAction, sAction[1:]))
@@ -361,16 +361,16 @@
if cAction == "=":
## disambiguator
if sAction[0:1] == "=":
sAction = sAction[1:]
if "define" in sAction and not re.search(r"define\(dTokenPos, *m\.start.*, \[.*\] *\)", sAction):
- print("# Error in action at line " + sIdAction + ": second argument for define must be a list of strings")
+ print(f"# Error in action at line {sIdAction}: second argument for define must be a list of strings")
print(sAction)
lFUNCTIONS.append(("_d_"+sIdAction, sAction))
sAction = "_d_"+sIdAction
return [sCondition, cAction, sAction]
- print("# Unknown action at line " + sIdAction)
+ print(f"# Unknown action at line {sIdAction}")
return None
def _calcRulesStats (lRules):
"count rules and actions"
@@ -466,13 +466,13 @@
def printBookmark (nLevel, sComment, nLine):
"print bookmark within the rules file"
print(" {:>6}: {}".format(nLine, " " * nLevel + sComment))
-def make (spLang, sLang, bUseCache=False):
+def make (spLang, sLang, bUseCache=None):
"compile rules, returns a dictionary of values"
- # for clarity purpose, don’t create any file here
+ # for clarity purpose, don’t create any file here (except cache)
dCacheVars = None
if os.path.isfile("_build/data_cache.json"):
print("> data cache found")
@@ -486,18 +486,20 @@
print("> read rules file...")
try:
sFileContent = open(spLang + "/rules.grx", 'r', encoding="utf-8").read()
except OSError:
- print("Error. Rules file in project [" + sLang + "] not found.")
+ print(f"# Error. Rules file in project <{sLang}> not found.")
exit()
+ # calculate hash of loaded file
xHasher = hashlib.new("sha3_512")
xHasher.update(sFileContent.encode("utf-8"))
sFileHash = xHasher.hexdigest()
- if dCacheVars and sFileHash == dCacheVars.get("sFileHash", ""):
+ if dCacheVars and bUseCache != False and sFileHash == dCacheVars.get("sFileHash", ""):
+ # if is None or True, we can use the cache
print("> cache hash identical to file hash, use cache")
print(" build made at: " + sBuildDate)
return dCacheVars
# removing comments, zeroing empty lines, creating definitions, storing tests, merging rule lines
@@ -521,11 +523,11 @@
# definition
m = re.match("DEF: +([a-zA-Z_][a-zA-Z_0-9]*) +(.+)$", sLine.strip())
if m:
dDEFINITIONS["{"+m.group(1)+"}"] = m.group(2)
else:
- print("Error in definition: ", end="")
+ print("# Error in definition: ", end="")
print(sLine.strip())
elif sLine.startswith("DECL:"):
# declensions
m = re.match(r"DECL: +(\+\w+) (.+)$", sLine.strip())
if m:
@@ -626,17 +628,17 @@
elif sFuncName.startswith("_p_"): # preprocessor
sParams = "sSentence, m"
elif sFuncName.startswith("_d_"): # disambiguator
sParams = "sSentence, m, dTokenPos"
else:
- print("# Unknown function type in [" + sFuncName + "]")
+ print(f"# Unknown function type in <{sFuncName}>")
continue
# Python
- sPyCallables += "def {} ({}):\n".format(sFuncName, sParams)
- sPyCallables += " return " + sReturn + "\n"
+ sPyCallables += f"def {sFuncName} ({sParams}):\n"
+ sPyCallables += f" return {sReturn}\n"
# JavaScript
- sJSCallables += " {}: function ({})".format(sFuncName, sParams) + " {\n"
+ sJSCallables += f" {sFuncName}: function ({sParams}) {{\n"
sJSCallables += " return " + jsconv.py2js(sReturn) + ";\n"
sJSCallables += " },\n"
displayStats(lParagraphRules, lSentenceRules)
Index: compile_rules_graph.py
==================================================================
--- compile_rules_graph.py
+++ compile_rules_graph.py
@@ -99,11 +99,10 @@
self.dOptPriority = dOptPriority
self.dAntiPatterns = {}
self.dActions = {}
self.dFuncName = {}
self.dFunctions = {}
- self.dURL = {}
def _genTokenLines (self, sTokenLine):
"tokenize a string and return a list of lines of tokens"
lTokenLines = []
for sTokBlock in sTokenLine.split():
@@ -322,11 +321,11 @@
## error
iMsg = sAction.find(" # ")
if iMsg == -1:
sMsg = "# Error. Error message not found."
sURL = ""
- print("\n" + sMsg + " at: ", sLineId, sActionId)
+ print("\n# Error. No message at: ", sLineId, sActionId)
else:
sMsg = sAction[iMsg+3:].strip()
sAction = sAction[:iMsg].strip()
sURL = ""
mURL = re.search("[|] *(https?://.*)", sMsg)
@@ -476,39 +475,39 @@
for iLine, sLine in lRule:
sLine = sLine.rstrip()
if "\t" in sLine:
# tabulation not allowed
- print("Error. Tabulation at line: ", iLine)
+ print("# Error. Tabulation at line: ", iLine)
exit()
elif sLine.startswith("@@@@GRAPH: "):
# rules graph call
m = re.match(r"@@@@GRAPH: *(\w+) *[|] *(\w+)", sLine.strip())
if m:
sGraphName = m.group(1)
sGraphCode = m.group(2)
if sGraphName in dAllGraph or sGraphCode in dGraphCode:
- print(f"Error at line {iLine}. Graph name <{sGraphName}> or graph code <{sGraphCode}> already exists.")
+ print(f"# Error at line {iLine}. Graph name <{sGraphName}> or graph code <{sGraphCode}> already exists.")
exit()
dAllGraph[sGraphName] = []
dGraphCode[sGraphName] = sGraphCode
else:
- print("Error. Graph name not found at line", iLine)
+ print("# Error. Graph name not found at line", iLine)
exit()
elif sLine.startswith("__") and sLine.endswith("__"):
# new rule group
m = re.match("__(\\w+)(!\\d|)__", sLine)
if m:
sRuleName = m.group(1)
if sRuleName in aRuleName:
- print(f"Error at line {iLine}. Rule name <{sRuleName}> already exists.")
+ print(f"# Error at line {iLine}. Rule name <{sRuleName}> already exists.")
exit()
aRuleName.add(sRuleName)
iActionBlock = 1
nPriority = int(m.group(2)[1:]) if m.group(2) else -1
else:
- print("Syntax error in rule group: ", sLine, " -- line:", iLine)
+ print("# Syntax error in rule group: ", sLine, " -- line:", iLine)
exit()
elif re.match(" \\S", sLine):
# tokens line
lTokenLine.append([iLine, sLine.strip()])
elif sLine.startswith(" ||"):
@@ -533,23 +532,23 @@
elif re.match("[ ]*$", sLine):
# empty line to end merging
if not lTokenLine:
continue
if bActionBlock or not lActions:
- print("Error. No action found at line:", iLine)
+ print("# Error. No action found at line:", iLine)
print(bActionBlock, lActions)
exit()
if not sGraphName:
- print("Error. All rules must belong to a named graph. Line: ", iLine)
+ print("# Error. All rules must belong to a named graph. Line: ", iLine)
exit()
for j, sTokenLine in lTokenLine:
dAllGraph[sGraphName].append((j, sRuleName, sTokenLine, iActionBlock, list(lActions), nPriority))
lTokenLine.clear()
lActions.clear()
iActionBlock += 1
else:
- print("Unknown line at:", iLine)
+ print("# Unknown line at:", iLine)
print(sLine)
# processing rules
print(" processing graph rules...")
initProcessPoolExecutor(len(dAllGraph))
Index: make.py
==================================================================
--- make.py
+++ make.py
@@ -32,57 +32,56 @@
def getConfig (sLang):
"load config.ini in at gc_lang/, returns xConfigParser object"
xConfig = configparser.ConfigParser()
xConfig.optionxform = str
try:
- xConfig.read_file(open("gc_lang/" + sLang + "/config.ini", "r", encoding="utf-8"))
+ xConfig.read_file(open(f"gc_lang/{sLang}/config.ini", "r", encoding="utf-8"))
except FileNotFoundError:
- print("# Error. Can’t read config file [" + sLang + "]")
+ print(f"# Error. Can’t read config file <{sLang}>")
exit()
return xConfig
def createOptionsLabelProperties (dOptLbl):
"create content for .properties files (LibreOffice)"
sContent = ""
for sOpt, tLabel in dOptLbl.items():
- sContent += sOpt + "=" + tLabel[0] + "\n"
+ sContent += f"{sOpt}={tLabel[0]}\n"
if tLabel[1]:
- sContent += "hlp_" + sOpt + "=" + tLabel[1] + "\n"
+ sContent += f"hlp_{sOpt}={tLabel[1]}\n"
return sContent
def createDialogOptionsXDL (dVars):
"create bundled dialog options file .xdl (LibreOffice)"
- sFixedline = '\n'
- sCheckbox = '\n'
- iTabIndex = 1
+ iTab = 1
nPosY = 5
nWidth = 240
sContent = ""
- dOpt = dVars["dOptPython"]
+ dOpt = dVars["dOptWriter"]
dOptLabel = dVars["dOptLabel"][dVars["lang"]]
for sGroup, lGroupOptions in dVars["lStructOpt"]:
- sContent += sFixedline.format(sGroup, iTabIndex, nPosY, nWidth)
- iTabIndex += 1
+ sContent += f'\n'
+ iTab += 1
for lLineOptions in lGroupOptions:
nElemWidth = nWidth // len(lLineOptions)
nPosY += 10
nPosX = 10
for sOpt in lLineOptions:
- sHelp = 'dlg:help-text="&hlp_%s"'%sOpt if dOptLabel[sOpt][1] else ""
- sContent += sCheckbox.format(sOpt, iTabIndex, nPosY, nPosX, nElemWidth, "true" if dOpt[sOpt] else "false", sHelp)
- iTabIndex += 1
+ sHelp = f'dlg:help-text="&hlp_{sOpt}"' if dOptLabel[sOpt][1] else ""
+ sChecked = "true" if dOpt[sOpt] else "false"
+ sContent += f'\n'
+ iTab += 1
nPosX += nElemWidth
nPosY += 10
return sContent
def createOXT (spLang, dVars, dOxt, spLangPack, bInstall):
"create extension for Writer"
print("Building extension for Writer")
- spfZip = "_build/" + dVars['name'] + "-"+ dVars['lang'] +"-v" + dVars['version'] + '.oxt'
+ spfZip = f"_build/{dVars['name']}-{dVars['lang']}-v{dVars['version']}.oxt"
hZip = zipfile.ZipFile(spfZip, mode='w', compression=zipfile.ZIP_DEFLATED)
# Package and parser
copyGrammalectePyPackageInZipFile(hZip, spLangPack, "pythonpath/")
hZip.write("grammalecte-cli.py", "pythonpath/grammalecte-cli.py")
@@ -116,11 +115,11 @@
hZip.writestr("dialog/options_page.xdl", helpers.fileFile("gc_core/py/oxt/options_page.xdl", dVars))
hZip.writestr("dialog/OptionsDialog.xcs", helpers.fileFile("gc_core/py/oxt/OptionsDialog.xcs", dVars))
hZip.writestr("dialog/OptionsDialog.xcu", helpers.fileFile("gc_core/py/oxt/OptionsDialog.xcu", dVars))
hZip.writestr("dialog/" + dVars['lang'] + "_en.default", "")
for sLangLbl, dOptLbl in dVars['dOptLabel'].items():
- hZip.writestr("dialog/" + dVars['lang'] + "_" + sLangLbl + ".properties", createOptionsLabelProperties(dOptLbl))
+ hZip.writestr(f"dialog/{dVars['lang']}_{sLangLbl}.properties", createOptionsLabelProperties(dOptLbl))
## ADDONS OXT
print("+ OXT: ", end="")
for spfSrc, spfDst in dOxt.items():
print(spfSrc, end=", ")
@@ -146,27 +145,14 @@
print(cmd)
os.system(cmd)
else:
print("# Error: path and filename of unopkg not set in config.ini")
-
-def createServerOptions (sLang, dOptData):
- "create file options for Grammalecte server"
- with open("grammalecte-server-options."+sLang+".ini", "w", encoding="utf-8", newline="\n") as hDst:
- hDst.write("# Server options. Lang: " + sLang + "\n\n[gc_options]\n")
- for sSection, lOpt in dOptData["lStructOpt"]:
- hDst.write("\n########## " + dOptData["dOptLabel"][sLang].get(sSection, sSection + "[no label found]")[0] + " ##########\n")
- for lLineOpt in lOpt:
- for sOpt in lLineOpt:
- hDst.write("# " + dOptData["dOptLabel"][sLang].get(sOpt, "[no label found]")[0] + "\n")
- hDst.write(sOpt + " = " + ("1" if dOptData["dOptServer"].get(sOpt, None) else "0") + "\n")
- hDst.write("html = 1\n")
-
def createPackageZip (dVars, spLangPack):
"create server zip"
- spfZip = "_build/" + dVars['name'] + "-"+ dVars['lang'] +"-v" + dVars['version'] + '.zip'
+ spfZip = f"_build/{dVars['name']}-{dVars['lang']}-v{dVars['version']}.zip"
hZip = zipfile.ZipFile(spfZip, mode='w', compression=zipfile.ZIP_DEFLATED)
copyGrammalectePyPackageInZipFile(hZip, spLangPack)
for spf in ["grammalecte-cli.py", "grammalecte-server.py", \
"README.txt", "LICENSE.txt", "LICENSE.fr.txt"]:
hZip.write(spf)
@@ -326,12 +312,12 @@
if bCommunityDict:
lDict.append(("community", dVars['dic_community_filename']))
if bPersonalDict:
lDict.append(("personal", dVars['dic_personal_filename']))
for sType, sFileName in lDict:
- spfPyDic = "graphspell/_dictionaries/" + sFileName + ".bdic"
- spfJSDic = "graphspell-js/_dictionaries/" + sFileName + ".json"
+ spfPyDic = f"graphspell/_dictionaries/{sFileName}.bdic"
+ spfJSDic = f"graphspell-js/_dictionaries/{sFileName}.json"
if not os.path.isfile(spfPyDic) or (bJavaScript and not os.path.isfile(spfJSDic)):
buildDictionary(dVars, sType, bJavaScript)
print(spfPyDic)
file_util.copy_file(spfPyDic, "grammalecte/graphspell/_dictionaries")
dVars['dic_'+sType+'_filename_py'] = sFileName + '.bdic'
@@ -374,10 +360,11 @@
print("Python 3.7+ required")
return
xParser = argparse.ArgumentParser()
xParser.add_argument("lang", type=str, nargs='+', help="lang project to generate (name of folder in /lang)")
xParser.add_argument("-uc", "--use_cache", help="use data cache instead of rebuilding rules", action="store_true")
+ xParser.add_argument("-frb", "--force_rebuild", help="force rebuilding rules", action="store_true")
xParser.add_argument("-b", "--build_data", help="launch build_data.py (part 1 and 2)", action="store_true")
xParser.add_argument("-bb", "--build_data_before", help="launch build_data.py (only part 1: before dictionary building)", action="store_true")
xParser.add_argument("-ba", "--build_data_after", help="launch build_data.py (only part 2: before dictionary building)", action="store_true")
xParser.add_argument("-d", "--dict", help="generate FSA dictionary", action="store_true")
xParser.add_argument("-t", "--tests", help="run unit tests", action="store_true")
@@ -436,20 +423,25 @@
# copy dictionaries from Graphspell
copyGraphspellDictionaries(dVars, xArgs.javascript, xArgs.add_community_dictionary, xArgs.add_personal_dictionary)
# make
- sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, xArgs.use_cache)
+ bUseCache = None # we may rebuild if it’s necessary
+ if xArgs.use_cache:
+ bUseCache = True # we use the cache if it exists
+ if xArgs.force_rebuild:
+ bUseCache = False # we rebuild
+ sVersion = create(sLang, xConfig, xArgs.install, xArgs.javascript, bUseCache)
# tests
if xArgs.tests or xArgs.perf or xArgs.perf_memo:
print("> Running tests")
try:
tests = importlib.import_module("grammalecte."+sLang+".tests")
print(tests.__file__)
except ImportError:
- print("# Error. Import failed:" + "grammalecte."+sLang+".tests")
+ print(f"# Error. Import failed: grammalecte.{sLang}.tests")
traceback.print_exc()
else:
if xArgs.tests:
xTestSuite = unittest.TestLoader().loadTestsFromModule(tests)
unittest.TextTestRunner().run(xTestSuite)