Grammalecte  Check-in [5cdb3649d7]

"""
Grammalecte: compile rules
Create a Direct Acyclic Rule Graphs (DARGs)
"""

import re
import os
import time
import concurrent.futures

import darg
import compile_rules_js_convert as jsconv


#### PROCESS POOL EXECUTOR ####
xProcessPoolExecutor = None

def initProcessPoolExecutor (nMultiCPU=None):
    "process pool executor initialisation"
    global xProcessPoolExecutor
    if xProcessPoolExecutor:
        # we shutdown the ProcessPoolExecutor which may have been launched previously
        print("  ProcessPoolExecutor shutdown.")
        xProcessPoolExecutor.shutdown(wait=False)
    nMaxCPU = max(os.cpu_count()-1, 1)
    if nMultiCPU is None or not (1 <= nMultiCPU <= nMaxCPU):
        nMultiCPU = nMaxCPU
    print("  CPU processes used for workers: ", nMultiCPU)
    xProcessPoolExecutor = concurrent.futures.ProcessPoolExecutor(max_workers=nMultiCPU)


def rewriteCode (sCode):
    "convert simple rule syntax to a string of Python code"
    "convert simple code syntax to a string of Python code"
    if sCode[0:1] == "=":
        sCode = sCode[1:]
    sCode = sCode.replace("__also__", "bCondMemo")
    sCode = sCode.replace("__else__", "not bCondMemo")
    sCode = sCode.replace("sContext", "_sAppContext")
    sCode = re.sub(r"\b(morph|morphVC|analyse|value|tag|displayInfo)[(]\\(\d+)", 'g_\\1(lToken[nTokenOffset+\\2]', sCode)
    sCode = re.sub(r"\b(morph|morphVC|analyse|value|tag|displayInfo)[(]\\-(\d+)", 'g_\\1(lToken[nLastToken-\\2+1]', sCode)

        print("# Warning at line " + sActionId + ":  This message looks like code. Line should probably begin with =")
        print(sText)



class GraphBuilder:

    def __init__ (self, dDef, dDecl, dOptPriority):
    def __init__ (self, sGraphName, sGraphCode, sLang, dDef, dDecl, dOptPriority):
        self.sGraphName = sGraphName
        self.sGraphCode = sGraphCode
        self.sLang = sLang
        self.dDef = dDef
        self.dDecl = dDecl
        self.dOptPriority = dOptPriority
        self.dAntiPatterns = {}
        self.dActions = {}
        self.dFuncName = {}
        self.dFunctions = {}

                        for sSuffix in self.dDecl[sCode]:
                            lToken.append(sToken+sSuffix)
                        break
            else:
                lToken.append(sToken)
        return lToken

    def createGraphAndActions (self, sGraphName, lRuleLine, sLang):
    def createGraphAndActions (self, lRuleLine):
        "create a graph as a dictionary with <lRuleLine>"
        fStartTimer = time.time()
        print("{:>8,} rules in {:<24} ".format(len(lRuleLine), "<"+sGraphName+">"), end="")
        print("{:>8,} rules in {:<30} ".format(len(lRuleLine), "<"+self.sGraphName+"|"+self.sGraphCode+">"), end="")
        lPreparedRule = []
        for i, sRuleName, sTokenLine, iActionBlock, lActions, nPriority in lRuleLine:
            for aRule in self.createRule(i, sRuleName, sTokenLine, iActionBlock, lActions, nPriority):
                lPreparedRule.append(aRule)
        # Debugging
        if False:
            print("\nRULES:")
            for e in lPreparedRule:
                if e[-2] == "##2211":
                    print(e)
        # Graph creation
        oDARG = darg.DARG(lPreparedRule, sLang)
        oDARG = darg.DARG(lPreparedRule, self.sLang)
        dGraph = oDARG.createGraph()
        print(oDARG, end="")
        # debugging
        if False:
            print("\nGRAPH:", sGraphName)
            print("\nGRAPH:", self.sGraphName)
            for k, v in dGraph.items():
                print(k, "\t", v)
        print("\tin {:>8.2f} s".format(time.time()-fStartTimer))
        sPyCallables, sJSCallables = self.createCallables()
        return dGraph
        return dGraph, self.dActions, sPyCallables, sJSCallables

    def createRule (self, iLine, sRuleName, sTokenLine, iActionBlock, lActions, nPriority):
        "generator: create rule as list"
        # print(iLine, "//", sRuleName, "//", sTokenLine, "//", lActions, "//", nPriority)
        if sTokenLine.startswith("!!") and sTokenLine.endswith("¡¡"):
            # antipattern
            sTokenLine = sTokenLine[2:-2].strip()

                        iGroup += 1
                        dPos[iGroup] = i + 1    # we add 1, for we count tokens from 1 to n (not from 0)

                # Parse actions
                for iAction, (iActionLine, sAction) in enumerate(lActions):
                    sAction = sAction.strip()
                    if sAction:
                        sActionId = sRuleName + "__b" + str(iActionBlock) + "_a" + str(iAction)
                        sActionId = self.sGraphCode + "__" + sRuleName + "__b" + str(iActionBlock) + "_a" + str(iAction)
                        aAction = self.createAction(sActionId, sAction, nPriority, len(lToken), dPos, iActionLine)
                        if aAction:
                            sActionName = self.storeAction(sActionId, aAction)
                            lResult = list(lToken)
                            lResult.extend(["##"+str(iLine), sActionName])
                            #if iLine == 13341:
                            #    print("  ".join(lToken))

    def _getNameForCode (self, sType, sCode):
        "create and get a name for a code"
        if sType not in self.dFuncName:
            self.dFuncName[sType] = {}
        if sCode not in self.dFuncName[sType]:
            self.dFuncName[sType][sCode] = len(self.dFuncName[sType])+1
        return "_g_" + sType + "_" + str(self.dFuncName[sType][sCode])
        return "_g_" + sType + "_" + self.sGraphCode + "_" + str(self.dFuncName[sType][sCode])

    def createCallables (self):
        "return callables for Python and JavaScript"
        print("  creating callables for graph rules...")
        sPyCallables = ""
        sJSCallables = ""
        for sFuncName, sReturn in self.dFunctions.items():
            if sFuncName.startswith("_g_cond_"): # condition
                sParams = "lToken, nTokenOffset, nLastToken, sCountry, bCondMemo, dTags, sSentence, sSentence0"
            elif sFuncName.startswith("_g_msg_"): # message
                sParams = "lToken, nTokenOffset, nLastToken"

            sPyCallables += "    return " + sReturn + "\n"
            # JavaScript
            sJSCallables += "    {}: function ({})".format(sFuncName, sParams) + " {\n"
            sJSCallables += "        return " + jsconv.py2js(sReturn) + ";\n"
            sJSCallables += "    },\n"
        return sPyCallables, sJSCallables


def processing (sGraphName, sGraphCode, sLang, lRuleLine, dDef, dDecl, dOptPriority):
    "to be run in a separate process"
    oGraphBuilder = GraphBuilder(sGraphName, sGraphCode, sLang, dDef, dDecl, dOptPriority)
    dGraph, dActions, sPy, sJS = oGraphBuilder.createGraphAndActions(lRuleLine)
    return (sGraphName, dGraph, dActions, sPy, sJS)


def make (lRule, sLang, dDef, dDecl, dOptPriority):
    "compile rules, returns a dictionary of values"
    # for clarity purpose, don’t create any file here

    # removing comments, zeroing empty lines, creating definitions, storing tests, merging rule lines
    print("  parsing rules...")
    lTokenLine = []
    lActions = []
    bActionBlock = False
    nPriority = -1
    dAllGraph = {}
    dGraphCode = {}
    sGraphName = ""
    iActionBlock = 0
    aRuleName = set()

    for iLine, sLine in lRule:
        sLine = sLine.rstrip()
        if "\t" in sLine:
            # tabulation not allowed
            print("Error. Tabulation at line: ", iLine)
            exit()
        elif sLine.startswith("@@@@GRAPH: "):
            # rules graph call
            m = re.match(r"@@@@GRAPH: *(\w+)", sLine.strip())
            m = re.match(r"@@@@GRAPH: *(\w+) *[|] *(\w+)", sLine.strip())
            if m:
                sGraphName = m.group(1)
                sGraphCode = m.group(2)
                if sGraphName in dAllGraph:
                    print("Error at line " + iLine + ". Graph name <" + sGraphName + "> already exists.")
                if sGraphName in dAllGraph or sGraphCode in dGraphCode:
                    print("Error at line " + iLine + ". Graph name <" + sGraphName + "> or graph code <" + sGraphCode + "> already exists.")
                    exit()
                dAllGraph[sGraphName] = []
                dGraphCode[sGraphName] = sGraphCode
            else:
                print("Error. Graph name not found at line", iLine)
                exit()
        elif sLine.startswith("__") and sLine.endswith("__"):
            # new rule group
            m = re.match("__(\\w+)(!\\d|)__", sLine)
            if m:

            iActionBlock += 1
        else:
            print("Unknown line at:", iLine)
            print(sLine)

    # processing rules
    print("  processing rules...")
    initProcessPoolExecutor()
    fStartTimer = time.time()
    nRule = 0
    dAllActions = {}
    sPyCallables = ""
    sJSCallables = ""
    lResult = []
    oGraphBuilder = GraphBuilder(dDef, dDecl, dOptPriority)
    # buid graph
    for sGraphName, lRuleLine in dAllGraph.items():
        #dGraph, dActions, sPy, sJS = processing(sGraphName, dGraphCode[sGraphName], sLang, lRuleLine, dDef, dDecl, dOptPriority)
        nRule += len(lRuleLine)
        try:
            xFuture = xProcessPoolExecutor.submit(processing, sGraphName, dGraphCode[sGraphName], sLang, lRuleLine, dDef, dDecl, dOptPriority)
            lResult.append(xFuture)
        except (concurrent.futures.TimeoutError, concurrent.futures.CancelledError):
            return "Analysis aborted (time out or cancelled)"
        except concurrent.futures.BrokenExecutor:
            return "Executor broken. The server failed."
    # merging results
    xProcessPoolExecutor.shutdown(wait=True)
    for xFuture in lResult:
        dGraph = oGraphBuilder.createGraphAndActions(sGraphName, lRuleLine, sLang)
        sGraphName, dGraph, dActions, sPy, sJS = xFuture.result()
        #print(dGraph)
        #for k, v in dActions.items():
        #    print(k, ":", v)
        #input()
        dAllGraph[sGraphName] = dGraph
        dAllActions.update(dActions)
        sPyCallables += sPy
        sJSCallables += sJS
    print("  Total: ", nRule, "rules, ", len(oGraphBuilder.dActions), "actions")
    print("  Total: ", nRule, "rules, ", len(dAllActions), "actions")
    print("  Build time: {:.2f} s".format(time.time() - fStartTimer))

    sPyCallables, sJSCallables = oGraphBuilder.createCallables()
    #print(sPyCallables)

    return {
        # the graphs describe paths of tokens to actions which eventually execute callables
        "rules_graphs": str(dAllGraph),
        "rules_graphsJS": str(dAllGraph),
        "rules_actions": str(oGraphBuilder.dActions),
        "rules_actionsJS": jsconv.pyActionsToString(oGraphBuilder.dActions),
        "rules_actions": str(dAllActions),
        "rules_actionsJS": jsconv.pyActionsToString(dAllActions),
        "graph_callables": sPyCallables,
        "graph_callablesJS": sJSCallables
    }

@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: purge_tag_disambiguate                                                                  _
@@@@GRAPH: purge_tag_disambiguate|ptd                                                              _
@@@@
@@@@
@@@@
@@@@

!!!
!!!

@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: ocr                                                                                     _
@@@@GRAPH: ocr|ocr                                                                                 _
@@@@
@@@@
@@@@
@@@@

# This graph is parsed only if option <ocr> is activated.

@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: graphe1                                                                                 _
@@@@GRAPH: graphe1|g1                                                                              _
@@@@
@@@@
@@@@
@@@@

!!!! Doublons (casse identique)                                                                   !!

@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: purge_passe2                                                                            _
@@@@GRAPH: purge_passe2|pp2                                                                        _
@@@@
@@@@
@@@@
@@@@

## Seconde passe (il faut réorganiser tout le cycle de simplification)

@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: purge_passe3                                                                            _
@@@@GRAPH: purge_passe3|pp3                                                                        _
@@@@
@@@@
@@@@
@@@@

__da_le_la_les_leur2__
    [<start>|,|(]  c’                                                 ?[ne|n’]¿  >être   [le|la|l’|les]  @:[NA]¬:G

@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: groupes_nominaux                                                                        _
@@@@GRAPH: groupes_nominaux|gn                                                                     _
@@@@
@@@@
@@@@
@@@@

!!
!!

@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: purge_ponctuations2                                                                     _
@@@@GRAPH: purge_ponctuations2|ppc2                                                                _
@@@@
@@@@
@@@@
@@@@

__da_été2__
    >avoir été

TEST: En outre, les grosses institutions comme l’Éducation Nationale ne développent pas forcément en interne


@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: verbes1                                                                                 _
@@@@GRAPH: verbes1|gv1                                                                             _
@@@@
@@@@
@@@@
@@@@


!!!! OCR                                                                                          !!

        <<- morph(\1, ":[NAM]") and morph(\2, ":[NAM]") />> enum


@@@@
@@@@
@@@@
@@@@
@@@@GRAPH: verbes2                                                                                 _
@@@@GRAPH: verbes2|gv2                                                                             _
@@@@
@@@@
@@@@
@@@@

!!
!!