Overview
| Comment: | [core] gc engine: regex for multi-tokens [fr] tests |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fr | core | mtok |
| Files: | files | file ages | folders |
| SHA3-256: |
3e55498f43219e152d38148c59ce0ae7 |
| User & Date: | olr on 2021-03-10 19:54:13 |
| Original Comment: | [core] regex for multi-tokens [fr] tests |
| Other Links: | branch diff | manifest | tags |
Context
|
2021-03-10
| ||
| 20:02 | [core] gc engine: better names for vars and functions check-in: 82ff352fc9 user: olr tags: core, mtok | |
| 19:54 | [core] gc engine: regex for multi-tokens [fr] tests check-in: 3e55498f43 user: olr tags: fr, core, mtok | |
| 19:39 | [build] darg: regex for multi-token morphologies check-in: 1e120c280b user: olr tags: build, mtok | |
Changes
Modified gc_core/js/lang_core/gc_engine.js from [04cf68e8aa] to [acf5e76688].
| ︙ | ︙ | |||
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 |
for (let [iToken, oToken] of this.lTokens.entries()) {
if (bDebug) {
console.log("TOKEN: " + oToken["sValue"]);
}
// check arcs for each existing pointer
let lNextPointer = [];
for (let oPointer of lPointer) {
for (let [cActionType, sMatch, iNode] of this._getMatches(oGraph, oToken, oGraph[oPointer["iNode"]])) {
if (cActionType === null) {
lNextPointer.push(oPointer);
continue;
}
if (bDebug) {
console.log(" MATCH: " + cActionType + sMatch);
}
| > > > > > > > > > | > | > > > > > > > > | 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 |
for (let [iToken, oToken] of this.lTokens.entries()) {
if (bDebug) {
console.log("TOKEN: " + oToken["sValue"]);
}
// check arcs for each existing pointer
let lNextPointer = [];
for (let oPointer of lPointer) {
if (oPointer["nMultiEnd"] != -1) {
if (oToken["i"] <= oPointer["nMultiEnd"]) {
lNextPointer.push(oPointer);
}
if (oToken["i"] != oPointer["nMultiEnd"]) {
continue;
}
}
for (let [cActionType, sMatch, iNode] of this._getMatches(oGraph, oToken, oGraph[oPointer["iNode"]])) {
if (cActionType === null) {
lNextPointer.push(oPointer);
continue;
}
if (bDebug) {
console.log(" MATCH: " + cActionType + sMatch);
}
let nMultiEnd = (cActionType != "&") ? -1 : dToken["nMultiStartTo"];
lNextPointer.push({ "iToken1": oPointer["iToken1"], "iNode": iNode, "nMultiEnd": nMultiEnd });
}
}
lPointer = lNextPointer;
// check arcs of first nodes
for (let [cActionType, sMatch, iNode] of this._getMatches(oGraph, oToken, oGraph[0])) {
if (cActionType === null) {
continue;
}
if (bDebug) {
console.log(" MATCH: " + cActionType + sMatch);
}
let nMultiEnd = (cActionType != "&") ? -1 : dToken["nMultiStartTo"];
lPointer.push({ "iToken1": iToken, "iNode": iNode, "nMultiEnd": nMultiEnd });
}
// check if there is rules to check for each pointer
for (let oPointer of lPointer) {
if (oPointer["nMultiEnd"] != -1) {
if (oToken["i"] < oPointer["nMultiEnd"]) {
continue;
}
if (oToken["i"] == oPointer["nMultiEnd"]) {
oPointer["nMultiEnd"] = -1;
}
}
if (oGraph[oPointer["iNode"]].hasOwnProperty("<rules>")) {
let bChange = this._executeActions(oGraph, oGraph[oPointer["iNode"]]["<rules>"], oPointer["iToken1"]-1, iToken, dOptions, sCountry, bShowRuleId, bDebug, bContext);
if (bChange) {
bTagAndRewrite = true;
}
}
}
|
| ︙ | ︙ | |||
655 656 657 658 659 660 661 662 663 664 665 666 667 668 |
}
let [_, sOption, sFuncCond, cActionType, sWhat, ...eAct] = gc_rules_graph.dRule[sRuleId];
// Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ]
// TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ]
// Disambiguator [ option, condition, "=", replacement/suggestion/action ]
// Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ]
// Immunity [ option, condition, "!", "", iTokenStart, iTokenEnd ]
// Test [ option, condition, ">", "" ]
if (!sOption || dOptions.gl_get(sOption, false)) {
bCondMemo = !sFuncCond || gc_functions[sFuncCond](this.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, this.dTags, this.sSentence, this.sSentence0);
if (bCondMemo) {
if (cActionType == "-") {
// grammar error
let [iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL] = eAct;
| > | 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 |
}
let [_, sOption, sFuncCond, cActionType, sWhat, ...eAct] = gc_rules_graph.dRule[sRuleId];
// Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ]
// TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ]
// Disambiguator [ option, condition, "=", replacement/suggestion/action ]
// Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ]
// Immunity [ option, condition, "!", "", iTokenStart, iTokenEnd ]
// Immunity [ option, condition, "&", "", iTokenStart, iTokenEnd ]
// Test [ option, condition, ">", "" ]
if (!sOption || dOptions.gl_get(sOption, false)) {
bCondMemo = !sFuncCond || gc_functions[sFuncCond](this.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, this.dTags, this.sSentence, this.sSentence0);
if (bCondMemo) {
if (cActionType == "-") {
// grammar error
let [iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL] = eAct;
|
| ︙ | ︙ | |||
746 747 748 749 750 751 752 |
this.lTokens[i]["sImmunity"] = sImmunity;
let nErrorStart = this.nOffsetWithinParagraph + this.lTokens[i]["nStart"];
if (this.dError.has(nErrorStart)) {
this.dError.delete(nErrorStart);
}
}
}
| > > > > > > > > > > > > > > > > | | 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 |
this.lTokens[i]["sImmunity"] = sImmunity;
let nErrorStart = this.nOffsetWithinParagraph + this.lTokens[i]["nStart"];
if (this.dError.has(nErrorStart)) {
this.dError.delete(nErrorStart);
}
}
}
}
else if (cActionType == "#") {
// multi-tokens
let nTokenStart = (eAct[0] > 0) ? nTokenOffset + eAct[0] : nLastToken + eAct[0];
let nTokenEnd = (eAct[1] > 0) ? nTokenOffset + eAct[1] : nLastToken + eAct[1];
let oMultiToken = {
"nTokenStart": nTokenStart,
"nTokenEnd": nTokenEnd,
"lTokens": this.lTokens.slice(nTokenStart, nTokenEnd+1),
"lMorph": (sWhat) ? sWhat.split("|") : [":HM"]
}
this.lTokens[nTokenStart]["nMultiStartTo"] = nTokenEnd
this.lTokens[nTokenEnd]["nMultiEndFrom"] = nTokenStart
this.lTokens[nTokenStart]["dMultiToken"] = dMultiToken
this.lTokens[nTokenEnd]["dMultiToken"] = dMultiToken
}
else {
console.log("# error: unknown action at " + sLineId);
}
}
else if (cActionType == ">") {
if (bDebug) {
console.log(" COND_BREAK");
}
|
| ︙ | ︙ |
Modified gc_core/py/lang_core/gc_engine.py from [2b36e73536] to [27f95227b1].
| ︙ | ︙ | |||
505 506 507 508 509 510 511 512 513 514 515 516 517 518 |
bTokenFound = True
else:
if sNegPattern and any(re.search(sNegPattern, sMorph) for sMorph in lMorph):
continue
if not sPattern or any(re.search(sPattern, sMorph) for sMorph in lMorph):
yield ("@", sRegex, dNode["<re_morph>"][sRegex])
bTokenFound = True
# token tags
if "aTags" in dToken and "<tags>" in dNode:
for sTag in dToken["aTags"]:
if sTag in dNode["<tags>"]:
yield ("/", sTag, dNode["<tags>"][sTag])
bTokenFound = True
# meta arc (for token type)
| > > > > > > > > > > > > > > > > > > > > > > > > > | 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 |
bTokenFound = True
else:
if sNegPattern and any(re.search(sNegPattern, sMorph) for sMorph in lMorph):
continue
if not sPattern or any(re.search(sPattern, sMorph) for sMorph in lMorph):
yield ("@", sRegex, dNode["<re_morph>"][sRegex])
bTokenFound = True
# regex multi morph arcs
if "<re_mmorph>" in dNode:
if "nMultiStartTo" in dToken:
lMorph = dToken["dMultiToken"]["lMorph"]
for sRegex in dNode["<re_mmorph>"]:
if "¬" not in sRegex:
# no anti-pattern
if any(re.search(sRegex, sMorph) for sMorph in lMorph):
yield ("&", sRegex, dNode["<re_mmorph>"][sRegex])
bTokenFound = True
else:
# there is an anti-pattern
sPattern, sNegPattern = sRegex.split("¬", 1)
if sNegPattern == "*":
# all morphologies must match with <sPattern>
if sPattern:
if all(re.search(sPattern, sMorph) for sMorph in lMorph):
yield ("&", sRegex, dNode["<re_mmorph>"][sRegex])
bTokenFound = True
else:
if sNegPattern and any(re.search(sNegPattern, sMorph) for sMorph in lMorph):
continue
if not sPattern or any(re.search(sPattern, sMorph) for sMorph in lMorph):
yield ("&", sRegex, dNode["<re_mmorph>"][sRegex])
bTokenFound = True
# token tags
if "aTags" in dToken and "<tags>" in dNode:
for sTag in dToken["aTags"]:
if sTag in dNode["<tags>"]:
yield ("/", sTag, dNode["<tags>"][sTag])
bTokenFound = True
# meta arc (for token type)
|
| ︙ | ︙ | |||
539 540 541 542 543 544 545 546 547 548 549 550 551 |
bTagAndRewrite = False
for iToken, dToken in enumerate(self.lTokens):
if bDebug:
echo("TOKEN: " + dToken["sValue"])
# check arcs for each existing pointer
lNextPointer = []
for dPointer in lPointer:
for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[dPointer["iNode"]]):
if cActionType is None:
lNextPointer.append(dPointer)
continue
if bDebug:
echo(" MATCH: " + cActionType + sMatch)
| > > > > > > | > | | > > > | | 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 |
bTagAndRewrite = False
for iToken, dToken in enumerate(self.lTokens):
if bDebug:
echo("TOKEN: " + dToken["sValue"])
# check arcs for each existing pointer
lNextPointer = []
for dPointer in lPointer:
if dPointer["nMultiEnd"] != -1:
if dToken["i"] <= dPointer["nMultiEnd"]:
lNextPointer.append(dPointer)
if dToken["i"] != dPointer["nMultiEnd"]:
continue
for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[dPointer["iNode"]]):
if cActionType is None:
lNextPointer.append(dPointer)
continue
if bDebug:
echo(" MATCH: " + cActionType + sMatch)
nMultiEnd = -1 if cActionType != "&" else dToken["nMultiStartTo"]
lNextPointer.append({ "iToken1": dPointer["iToken1"], "iNode": iNode, "nMultiEnd": nMultiEnd })
lPointer = lNextPointer
# check arcs of first nodes
for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[0]):
if cActionType is None:
continue
if bDebug:
echo(" MATCH: " + cActionType + sMatch)
nMultiEnd = -1 if cActionType != "&" else dToken["nMultiStartTo"]
lPointer.append({ "iToken1": iToken, "iNode": iNode, "nMultiEnd": nMultiEnd })
# check if there is rules to check for each pointer
for dPointer in lPointer:
if dPointer["nMultiEnd"] != -1:
if dToken["i"] < dPointer["nMultiEnd"]:
continue
if dToken["i"] == dPointer["nMultiEnd"]:
dPointer["nMultiEnd"] = -1
if "<rules>" in dGraph[dPointer["iNode"]]:
bChange = self._executeActions(dGraph, dGraph[dPointer["iNode"]]["<rules>"], dPointer["iToken1"]-1, iToken, dOptions, sCountry, bShowRuleId, bDebug, bContext)
if bChange:
bTagAndRewrite = True
if bTagAndRewrite:
self.rewriteFromTags(bDebug)
if bDebug:
|
| ︙ | ︙ | |||
583 584 585 586 587 588 589 590 591 592 593 594 595 596 |
echo(" >TRY: " + sRuleId + " " + sLineId)
_, sOption, sFuncCond, cActionType, sWhat, *eAct = _rules_graph.dRule[sRuleId]
# Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ]
# TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ]
# Disambiguator [ option, condition, "=", replacement/suggestion/action ]
# Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ]
# Immunity [ option, condition, "!", option, iTokenStart, iTokenEnd ]
# Test [ option, condition, ">", "" ]
if not sOption or dOptions.get(sOption, False):
bCondMemo = not sFuncCond or getattr(gc_functions, sFuncCond)(self.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, self.dTags, self.sSentence, self.sSentence0)
if bCondMemo:
if cActionType == "-":
# grammar error
iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL = eAct
| > | 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 |
echo(" >TRY: " + sRuleId + " " + sLineId)
_, sOption, sFuncCond, cActionType, sWhat, *eAct = _rules_graph.dRule[sRuleId]
# Suggestion [ option, condition, "-", replacement/suggestion/action, iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL ]
# TextProcessor [ option, condition, "~", replacement/suggestion/action, iTokenStart, iTokenEnd, bCaseSvty ]
# Disambiguator [ option, condition, "=", replacement/suggestion/action ]
# Tag [ option, condition, "/", replacement/suggestion/action, iTokenStart, iTokenEnd ]
# Immunity [ option, condition, "!", option, iTokenStart, iTokenEnd ]
# Multi-token [ option, condition, "&", morphologies, iTokenStart, iTokenEnd ]
# Test [ option, condition, ">", "" ]
if not sOption or dOptions.get(sOption, False):
bCondMemo = not sFuncCond or getattr(gc_functions, sFuncCond)(self.lTokens, nTokenOffset, nLastToken, sCountry, bCondMemo, self.dTags, self.sSentence, self.sSentence0)
if bCondMemo:
if cActionType == "-":
# grammar error
iTokenStart, iTokenEnd, cStartLimit, cEndLimit, bCaseSvty, nPriority, sMessage, iURL = eAct
|
| ︙ | ︙ | |||
654 655 656 657 658 659 660 661 662 663 664 665 666 667 |
del self.dError[nErrorStart]
else:
for i in range(nTokenStart, nTokenEnd+1):
self.lTokens[i]["sImmunity"] = sImmunity
nErrorStart = self.nOffsetWithinParagraph + self.lTokens[i]["nStart"]
if nErrorStart in self.dError:
del self.dError[nErrorStart]
else:
echo("# error: unknown action at " + sLineId)
elif cActionType == ">":
if bDebug:
echo(" COND_BREAK")
break
except Exception as e:
| > > > > > > > > > > > > > > > | 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 |
del self.dError[nErrorStart]
else:
for i in range(nTokenStart, nTokenEnd+1):
self.lTokens[i]["sImmunity"] = sImmunity
nErrorStart = self.nOffsetWithinParagraph + self.lTokens[i]["nStart"]
if nErrorStart in self.dError:
del self.dError[nErrorStart]
elif cActionType == "&":
# multi-tokens
nTokenStart = nTokenOffset + eAct[0] if eAct[0] > 0 else nLastToken + eAct[0]
nTokenEnd = nTokenOffset + eAct[1] if eAct[1] > 0 else nLastToken + eAct[1]
dMultiToken = {
"nTokenStart": nTokenStart,
"nTokenEnd": nTokenEnd,
"lTokens": self.lTokens[nTokenStart:nTokenEnd+1],
"lMorph": sWhat.split("|") if sWhat else [":HM"]
}
self.lTokens[nTokenStart]["nMultiStartTo"] = nTokenEnd
self.lTokens[nTokenEnd]["nMultiEndFrom"] = nTokenStart
self.lTokens[nTokenStart]["dMultiToken"] = dMultiToken
self.lTokens[nTokenEnd]["dMultiToken"] = dMultiToken
print(dMultiToken)
else:
echo("# error: unknown action at " + sLineId)
elif cActionType == ">":
if bDebug:
echo(" COND_BREAK")
break
except Exception as e:
|
| ︙ | ︙ |
Modified gc_lang/fr/rules.grx from [9c6bc913d3] to [c7efaf6d46].
| ︙ | ︙ | |||
428 429 430 431 432 433 434 435 436 437 438 439 440 441 |
[)]\b(?![s¹²³⁴⁵⁶⁷⁸⁹⁰]\b)
<<- not before("\\((?:[rR][eéEÉ]|[qQ][uU]’|[nNmMtTsSdDlL]’)$") ->> ") " && Il manque un espace après la parenthèse.
__<s>/typo(typo_parenthèse_ouvrante_collée)__
\b[(](?=[^)][^)][^)])
<<- ->> " (" && Il manque un espace avant la parenthèse.
TEST: C’est au fond du couloir{{(}}celui du deuxième étage{{)}}qu’il se trouve. ->> " (|||) "
TEST: (a + b)²
TEST: il faut (re)former tout ça.
TEST: il (n’)est (qu’)ingénieur
# Points et espaces
__<s>/typo(typo_point_entre_deux_espaces)__ [ ][.](?=[ ]) <<- ->> . && Pas d’espace avant un point.
| > | 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 |
[)]\b(?![s¹²³⁴⁵⁶⁷⁸⁹⁰]\b)
<<- not before("\\((?:[rR][eéEÉ]|[qQ][uU]’|[nNmMtTsSdDlL]’)$") ->> ") " && Il manque un espace après la parenthèse.
__<s>/typo(typo_parenthèse_ouvrante_collée)__
\b[(](?=[^)][^)][^)])
<<- ->> " (" && Il manque un espace avant la parenthèse.
TEST: C’est au fond du couloir{{(}}celui du deuxième étage{{)}}qu’il se trouve. ->> " (|||) "
TEST: de gain différentiel 𝐴 (𝑉ᵣ = 𝐴·𝑣H{{)}}et associé ->> ") "
TEST: (a + b)²
TEST: il faut (re)former tout ça.
TEST: il (n’)est (qu’)ingénieur
# Points et espaces
__<s>/typo(typo_point_entre_deux_espaces)__ [ ][.](?=[ ]) <<- ->> . && Pas d’espace avant un point.
|
| ︙ | ︙ | |||
3827 3828 3829 3830 3831 3832 3833 |
~^[A-ZÀÂÉÈÊÎÔ]. Airways
<<- ~>> ␣
<<- =>> define(\2, ":MP:e:i")
__merge__
| | | 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 |
~^[A-ZÀÂÉÈÊÎÔ]. Airways
<<- ~>> ␣
<<- =>> define(\2, ":MP:e:i")
__merge__
à la pp
<<- &>> :LW
__immunités__
il y a
il n’ y a
<<- !-1>>
|
| ︙ | ︙ | |||
4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 |
# <<- \1 == \2
# and not value(\2, "|nous|vous|faire|en|la|lui|donnant|œuvre|ah|oh|eh|hé|ho|ha|hou|olé|joli|Bora|couvent|dément|sapiens|très|vroum|")
# and not (value(\1, "|est|une|") and value(<1, "|l’|d’|"))
# and not (\2 == "mieux" and value(<1, "|qui|"))
# ->> \1 && Doublon.
#
#TEST: Il y a un {{doublon doublon}}. ->> doublon
!!
!!
!!!! Élisions & euphonie !!
!!
!!
| > > > > > > > > > > > > > | 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 |
# <<- \1 == \2
# and not value(\2, "|nous|vous|faire|en|la|lui|donnant|œuvre|ah|oh|eh|hé|ho|ha|hou|olé|joli|Bora|couvent|dément|sapiens|très|vroum|")
# and not (value(\1, "|est|une|") and value(<1, "|l’|d’|"))
# and not (\2 == "mieux" and value(<1, "|qui|"))
# ->> \1 && Doublon.
#
#TEST: Il y a un {{doublon doublon}}. ->> doublon
__test_merge__
&:LW
<<- echo("DETECTED") ~>> *
je vais &:LW
<<- --1>> X && TEST0.
je vais &:LW de
<<- --1>> X && TEST2.
<<- -1>> Z && TEST1.
!!
!!
!!!! Élisions & euphonie !!
!!
!!
|
| ︙ | ︙ |