Overview
Comment: | [core] gc engine: regex for multi-tokens [fr] tests |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | fr | core | mtok |
Files: | files | file ages | folders |
SHA3-256: |
3e55498f43219e152d38148c59ce0ae7 |
User & Date: | olr on 2021-03-10 19:54:13 |
Original Comment: | [core] regex for multi-tokens [fr] tests |
Other Links: | branch diff | manifest | tags |
Context
2021-03-10
| ||
20:02 | [core] gc engine: better names for vars and functions check-in: 82ff352fc9 user: olr tags: core, mtok | |
19:54 | [core] gc engine: regex for multi-tokens [fr] tests check-in: 3e55498f43 user: olr tags: core, fr, mtok | |
19:39 | [build] darg: regex for multi-token morphologies check-in: 1e120c280b user: olr tags: build, mtok | |
Changes
Modified gc_core/js/lang_core/gc_engine.js from [04cf68e8aa] to [acf5e76688].
︙ | ︙ | |||
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 | for (let [iToken, oToken] of this.lTokens.entries()) { if (bDebug) { console.log("TOKEN: " + oToken["sValue"]); } // check arcs for each existing pointer let lNextPointer = []; for (let oPointer of lPointer) { for (let [cActionType, sMatch, iNode] of this._getMatches(oGraph, oToken, oGraph[oPointer["iNode"]])) { if (cActionType === null) { lNextPointer.push(oPointer); continue; } if (bDebug) { console.log(" MATCH: " + cActionType + sMatch); } | > > > > > > > > > | > | > > > > > > > > | 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 | for (let [iToken, oToken] of this.lTokens.entries()) { if (bDebug) { console.log("TOKEN: " + oToken["sValue"]); } // check arcs for each existing pointer let lNextPointer = []; for (let oPointer of lPointer) { if (oPointer["nMultiEnd"] != -1) { if (oToken["i"] <= oPointer["nMultiEnd"]) { lNextPointer.push(oPointer); } if (oToken["i"] != oPointer["nMultiEnd"]) { continue; } } for (let [cActionType, sMatch, iNode] of this._getMatches(oGraph, oToken, oGraph[oPointer["iNode"]])) { if (cActionType === null) { lNextPointer.push(oPointer); continue; } if (bDebug) { console.log(" MATCH: " + cActionType + sMatch); } let nMultiEnd = (cActionType != "&") ? -1 : dToken["nMultiStartTo"]; lNextPointer.push({ "iToken1": oPointer["iToken1"], "iNode": iNode, "nMultiEnd": nMultiEnd }); } } lPointer = lNextPointer; // check arcs of first nodes for (let [cActionType, sMatch, iNode] of this._getMatches(oGraph, oToken, oGraph[0])) { if (cActionType === null) { continue; } if (bDebug) { console.log(" MATCH: " + cActionType + sMatch); } let nMultiEnd = (cActionType != "&") ? -1 : dToken["nMultiStartTo"]; lPointer.push({ "iToken1": iToken, "iNode": iNode, "nMultiEnd": nMultiEnd }); } // check if there is rules to check for each pointer for (let oPointer of lPointer) { if (oPointer["nMultiEnd"] != -1) { if (oToken["i"] < oPointer["nMultiEnd"]) { continue; } if (oToken["i"] == oPointer["nMultiEnd"]) { oPointer["nMultiEnd"] = -1; } } if (oGraph[oPointer["iNode"]].hasOwnProperty("<rules>")) { let bChange = this._executeActions(oGraph, oGraph[oPointer["iNode"]]["<rules>"], oPointer["iToken1"]-1, iToken, dOptions, sCountry, bShowRuleId, bDebug, bContext); if (bChange) { bTagAndRewrite = true; } } } |
︙ | ︙ | |||
655 656 657 658 659 660 661 662 663 664 665 666 667 668 | } let [_, sOption, sFuncCond, cActionType, sWhat, ...eAct] = gc_rules_graph.dRule[sRuleId]; // Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ] // TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ] // Disambiguator [ option, condition, "=", replacement/suggestion/action ] // Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ] // Immunity [ option, condition, "!", "", iTokenStart, iTokenEnd ] // Test [ option, condition, ">", "" ] if (!sOption || dOptions.gl_get(sOption, false)) { bCondMemo = !sFuncCond || gc_functions[sFuncCond](this.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, this.dTags, this.sSentence, this.sSentence0); if (bCondMemo) { if (cActionType == "-") { // grammar error let [iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL] = eAct; | > | 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 | } let [_, sOption, sFuncCond, cActionType, sWhat, ...eAct] = gc_rules_graph.dRule[sRuleId]; // Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ] // TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ] // Disambiguator [ option, condition, "=", replacement/suggestion/action ] // Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ] // Immunity [ option, condition, "!", "", iTokenStart, iTokenEnd ] // Immunity [ option, condition, "&", "", iTokenStart, iTokenEnd ] // Test [ option, condition, ">", "" ] if (!sOption || dOptions.gl_get(sOption, false)) { bCondMemo = !sFuncCond || gc_functions[sFuncCond](this.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, this.dTags, this.sSentence, this.sSentence0); if (bCondMemo) { if (cActionType == "-") { // grammar error let [iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL] = eAct; |
︙ | ︙ | |||
746 747 748 749 750 751 752 | this.lTokens[i]["sImmunity"] = sImmunity; let nErrorStart = this.nOffsetWithinParagraph + this.lTokens[i]["nStart"]; if (this.dError.has(nErrorStart)) { this.dError.delete(nErrorStart); } } } | > > > > > > > > > > > > > > > > | | 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 | this.lTokens[i]["sImmunity"] = sImmunity; let nErrorStart = this.nOffsetWithinParagraph + this.lTokens[i]["nStart"]; if (this.dError.has(nErrorStart)) { this.dError.delete(nErrorStart); } } } } else if (cActionType == "#") { // multi-tokens let nTokenStart = (eAct[0] > 0) ? nTokenOffset + eAct[0] : nLastToken + eAct[0]; let nTokenEnd = (eAct[1] > 0) ? nTokenOffset + eAct[1] : nLastToken + eAct[1]; let oMultiToken = { "nTokenStart": nTokenStart, "nTokenEnd": nTokenEnd, "lTokens": this.lTokens.slice(nTokenStart, nTokenEnd+1), "lMorph": (sWhat) ? sWhat.split("|") : [":HM"] } this.lTokens[nTokenStart]["nMultiStartTo"] = nTokenEnd this.lTokens[nTokenEnd]["nMultiEndFrom"] = nTokenStart this.lTokens[nTokenStart]["dMultiToken"] = dMultiToken this.lTokens[nTokenEnd]["dMultiToken"] = dMultiToken } else { console.log("# error: unknown action at " + sLineId); } } else if (cActionType == ">") { if (bDebug) { console.log(" COND_BREAK"); } |
︙ | ︙ |
Modified gc_core/py/lang_core/gc_engine.py from [2b36e73536] to [27f95227b1].
︙ | ︙ | |||
505 506 507 508 509 510 511 512 513 514 515 516 517 518 | bTokenFound = True else: if sNegPattern and any(re.search(sNegPattern, sMorph) for sMorph in lMorph): continue if not sPattern or any(re.search(sPattern, sMorph) for sMorph in lMorph): yield ("@", sRegex, dNode["<re_morph>"][sRegex]) bTokenFound = True # token tags if "aTags" in dToken and "<tags>" in dNode: for sTag in dToken["aTags"]: if sTag in dNode["<tags>"]: yield ("/", sTag, dNode["<tags>"][sTag]) bTokenFound = True # meta arc (for token type) | > > > > > > > > > > > > > > > > > > > > > > > > > | 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 | bTokenFound = True else: if sNegPattern and any(re.search(sNegPattern, sMorph) for sMorph in lMorph): continue if not sPattern or any(re.search(sPattern, sMorph) for sMorph in lMorph): yield ("@", sRegex, dNode["<re_morph>"][sRegex]) bTokenFound = True # regex multi morph arcs if "<re_mmorph>" in dNode: if "nMultiStartTo" in dToken: lMorph = dToken["dMultiToken"]["lMorph"] for sRegex in dNode["<re_mmorph>"]: if "¬" not in sRegex: # no anti-pattern if any(re.search(sRegex, sMorph) for sMorph in lMorph): yield ("&", sRegex, dNode["<re_mmorph>"][sRegex]) bTokenFound = True else: # there is an anti-pattern sPattern, sNegPattern = sRegex.split("¬", 1) if sNegPattern == "*": # all morphologies must match with <sPattern> if sPattern: if all(re.search(sPattern, sMorph) for sMorph in lMorph): yield ("&", sRegex, dNode["<re_mmorph>"][sRegex]) bTokenFound = True else: if sNegPattern and any(re.search(sNegPattern, sMorph) for sMorph in lMorph): continue if not sPattern or any(re.search(sPattern, sMorph) for sMorph in lMorph): yield ("&", sRegex, dNode["<re_mmorph>"][sRegex]) bTokenFound = True # token tags if "aTags" in dToken and "<tags>" in dNode: for sTag in dToken["aTags"]: if sTag in dNode["<tags>"]: yield ("/", sTag, dNode["<tags>"][sTag]) bTokenFound = True # meta arc (for token type) |
︙ | ︙ | |||
539 540 541 542 543 544 545 546 547 548 549 550 551 | bTagAndRewrite = False for iToken, dToken in enumerate(self.lTokens): if bDebug: echo("TOKEN: " + dToken["sValue"]) # check arcs for each existing pointer lNextPointer = [] for dPointer in lPointer: for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[dPointer["iNode"]]): if cActionType is None: lNextPointer.append(dPointer) continue if bDebug: echo(" MATCH: " + cActionType + sMatch) | > > > > > > | > | | > > > | | 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 | bTagAndRewrite = False for iToken, dToken in enumerate(self.lTokens): if bDebug: echo("TOKEN: " + dToken["sValue"]) # check arcs for each existing pointer lNextPointer = [] for dPointer in lPointer: if dPointer["nMultiEnd"] != -1: if dToken["i"] <= dPointer["nMultiEnd"]: lNextPointer.append(dPointer) if dToken["i"] != dPointer["nMultiEnd"]: continue for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[dPointer["iNode"]]): if cActionType is None: lNextPointer.append(dPointer) continue if bDebug: echo(" MATCH: " + cActionType + sMatch) nMultiEnd = -1 if cActionType != "&" else dToken["nMultiStartTo"] lNextPointer.append({ "iToken1": dPointer["iToken1"], "iNode": iNode, "nMultiEnd": nMultiEnd }) lPointer = lNextPointer # check arcs of first nodes for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[0]): if cActionType is None: continue if bDebug: echo(" MATCH: " + cActionType + sMatch) nMultiEnd = -1 if cActionType != "&" else dToken["nMultiStartTo"] lPointer.append({ "iToken1": iToken, "iNode": iNode, "nMultiEnd": nMultiEnd }) # check if there is rules to check for each pointer for dPointer in lPointer: if dPointer["nMultiEnd"] != -1: if dToken["i"] < dPointer["nMultiEnd"]: continue if dToken["i"] == dPointer["nMultiEnd"]: dPointer["nMultiEnd"] = -1 if "<rules>" in dGraph[dPointer["iNode"]]: bChange = self._executeActions(dGraph, dGraph[dPointer["iNode"]]["<rules>"], dPointer["iToken1"]-1, iToken, dOptions, sCountry, bShowRuleId, bDebug, bContext) if bChange: bTagAndRewrite = True if bTagAndRewrite: self.rewriteFromTags(bDebug) if bDebug: |
︙ | ︙ | |||
583 584 585 586 587 588 589 590 591 592 593 594 595 596 | echo(" >TRY: " + sRuleId + " " + sLineId) _, sOption, sFuncCond, cActionType, sWhat, *eAct = _rules_graph.dRule[sRuleId] # Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ] # TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ] # Disambiguator [ option, condition, "=", replacement/suggestion/action ] # Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ] # Immunity [ option, condition, "!", option, iTokenStart, iTokenEnd ] # Test [ option, condition, ">", "" ] if not sOption or dOptions.get(sOption, False): bCondMemo = not sFuncCond or getattr(gc_functions, sFuncCond)(self.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, self.dTags, self.sSentence, self.sSentence0) if bCondMemo: if cActionType == "-": # grammar error iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL = eAct | > | 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 | echo(" >TRY: " + sRuleId + " " + sLineId) _, sOption, sFuncCond, cActionType, sWhat, *eAct = _rules_graph.dRule[sRuleId] # Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ] # TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ] # Disambiguator [ option, condition, "=", replacement/suggestion/action ] # Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ] # Immunity [ option, condition, "!", option, iTokenStart, iTokenEnd ] # Multi-token [ option, condition, "&", morphologies, iTokenStart, iTokenEnd ] # Test [ option, condition, ">", "" ] if not sOption or dOptions.get(sOption, False): bCondMemo = not sFuncCond or getattr(gc_functions, sFuncCond)(self.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, self.dTags, self.sSentence, self.sSentence0) if bCondMemo: if cActionType == "-": # grammar error iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL = eAct |
︙ | ︙ | |||
654 655 656 657 658 659 660 661 662 663 664 665 666 667 | del self.dError[nErrorStart] else: for i in range(nTokenStart, nTokenEnd+1): self.lTokens[i]["sImmunity"] = sImmunity nErrorStart = self.nOffsetWithinParagraph + self.lTokens[i]["nStart"] if nErrorStart in self.dError: del self.dError[nErrorStart] else: echo("# error: unknown action at " + sLineId) elif cActionType == ">": if bDebug: echo(" COND_BREAK") break except Exception as e: | > > > > > > > > > > > > > > > | 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 | del self.dError[nErrorStart] else: for i in range(nTokenStart, nTokenEnd+1): self.lTokens[i]["sImmunity"] = sImmunity nErrorStart = self.nOffsetWithinParagraph + self.lTokens[i]["nStart"] if nErrorStart in self.dError: del self.dError[nErrorStart] elif cActionType == "&": # multi-tokens nTokenStart = nTokenOffset + eAct[0] if eAct[0] > 0 else nLastToken + eAct[0] nTokenEnd = nTokenOffset + eAct[1] if eAct[1] > 0 else nLastToken + eAct[1] dMultiToken = { "nTokenStart": nTokenStart, "nTokenEnd": nTokenEnd, "lTokens": self.lTokens[nTokenStart:nTokenEnd+1], "lMorph": sWhat.split("|") if sWhat else [":HM"] } self.lTokens[nTokenStart]["nMultiStartTo"] = nTokenEnd self.lTokens[nTokenEnd]["nMultiEndFrom"] = nTokenStart self.lTokens[nTokenStart]["dMultiToken"] = dMultiToken self.lTokens[nTokenEnd]["dMultiToken"] = dMultiToken print(dMultiToken) else: echo("# error: unknown action at " + sLineId) elif cActionType == ">": if bDebug: echo(" COND_BREAK") break except Exception as e: |
︙ | ︙ |
Modified gc_lang/fr/rules.grx from [9c6bc913d3] to [c7efaf6d46].
︙ | ︙ | |||
428 429 430 431 432 433 434 435 436 437 438 439 440 441 | [)]\b(?![s¹²³⁴⁵⁶⁷⁸⁹⁰]\b) <<- not before("\\((?:[rR][eéEÉ]|[qQ][uU]’|[nNmMtTsSdDlL]’)$") ->> ") " && Il manque un espace après la parenthèse. __<s>/typo(typo_parenthèse_ouvrante_collée)__ \b[(](?=[^)][^)][^)]) <<- ->> " (" && Il manque un espace avant la parenthèse. TEST: C’est au fond du couloir{{(}}celui du deuxième étage{{)}}qu’il se trouve. ->> " (|||) " TEST: (a + b)² TEST: il faut (re)former tout ça. TEST: il (n’)est (qu’)ingénieur # Points et espaces __<s>/typo(typo_point_entre_deux_espaces)__ [ ][.](?=[ ]) <<- ->> . && Pas d’espace avant un point. | > | 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 | [)]\b(?![s¹²³⁴⁵⁶⁷⁸⁹⁰]\b) <<- not before("\\((?:[rR][eéEÉ]|[qQ][uU]’|[nNmMtTsSdDlL]’)$") ->> ") " && Il manque un espace après la parenthèse. __<s>/typo(typo_parenthèse_ouvrante_collée)__ \b[(](?=[^)][^)][^)]) <<- ->> " (" && Il manque un espace avant la parenthèse. TEST: C’est au fond du couloir{{(}}celui du deuxième étage{{)}}qu’il se trouve. ->> " (|||) " TEST: de gain différentiel 𝐴 (𝑉ᵣ = 𝐴·𝑣H{{)}}et associé ->> ") " TEST: (a + b)² TEST: il faut (re)former tout ça. TEST: il (n’)est (qu’)ingénieur # Points et espaces __<s>/typo(typo_point_entre_deux_espaces)__ [ ][.](?=[ ]) <<- ->> . && Pas d’espace avant un point. |
︙ | ︙ | |||
3827 3828 3829 3830 3831 3832 3833 | ~^[A-ZÀÂÉÈÊÎÔ]. Airways <<- ~>> ␣ <<- =>> define(\2, ":MP:e:i") __merge__ | | | 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 | ~^[A-ZÀÂÉÈÊÎÔ]. Airways <<- ~>> ␣ <<- =>> define(\2, ":MP:e:i") __merge__ à la pp <<- &>> :LW __immunités__ il y a il n’ y a <<- !-1>> |
︙ | ︙ | |||
4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 | # <<- \1 == \2 # and not value(\2, "|nous|vous|faire|en|la|lui|donnant|œuvre|ah|oh|eh|hé|ho|ha|hou|olé|joli|Bora|couvent|dément|sapiens|très|vroum|") # and not (value(\1, "|est|une|") and value(<1, "|l’|d’|")) # and not (\2 == "mieux" and value(<1, "|qui|")) # ->> \1 && Doublon. # #TEST: Il y a un {{doublon doublon}}. ->> doublon !! !! !!!! Élisions & euphonie !! !! !! | > > > > > > > > > > > > > | 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 | # <<- \1 == \2 # and not value(\2, "|nous|vous|faire|en|la|lui|donnant|œuvre|ah|oh|eh|hé|ho|ha|hou|olé|joli|Bora|couvent|dément|sapiens|très|vroum|") # and not (value(\1, "|est|une|") and value(<1, "|l’|d’|")) # and not (\2 == "mieux" and value(<1, "|qui|")) # ->> \1 && Doublon. # #TEST: Il y a un {{doublon doublon}}. ->> doublon __test_merge__ &:LW <<- echo("DETECTED") ~>> * je vais &:LW <<- --1>> X && TEST0. je vais &:LW de <<- --1>> X && TEST2. <<- -1>> Z && TEST1. !! !! !!!! Élisions & euphonie !! !! !! |
︙ | ︙ |