From bb5256835b7c16153e1c80d3b17e2676c01718e2 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 27 Oct 2026 18:36:13 -0400 Subject: [PATCH 1/3] Add analysis affix edge-segment prefilter Closes #564 --- .../Morpher.cs | 3 + .../AnalysisAffixProcessAllomorphRuleSpec.cs | 6 + .../AnalysisAffixProcessRule.cs | 37 +- .../AnalysisEdgePrefilterTests.cs | 355 ++++++++++++++++++ .../early-lexicon-agglutinative/generate.py | 158 ++++++++ .../early-lexicon-agglutinative/grammar.xml | 95 +++++ .../early-lexicon-agglutinative/words.txt | 60 +++ 7 files changed, 713 insertions(+), 1 deletion(-) create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/generate.py create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/grammar.xml create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/words.txt diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index d0675509e..6094ece12 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -18,6 +18,9 @@ namespace SIL.Machine.Morphology.HermitCrab { public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator { + internal bool EdgePrefilterEnabled { get; set; } = true; + internal Action EdgePrefilterRejected { get; set; } + private readonly Language _lang; private readonly IRule _analysisRule; private readonly IRule _synthesisRule; diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs index 580b663fb..96d90b35f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs @@ -1,4 +1,5 @@ using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; using SIL.Machine.Matching; using SIL.Machine.Rules; @@ -14,8 +15,13 @@ public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph) _allomorph = allomorph; Pattern.Acceptable = CopyAgreementCheck; Pattern.Freeze(); + LeftEdgeConstraint = (Pattern.Children.First as Constraint)?.FeatureStruct; + RightEdgeConstraint = (Pattern.Children.Last as Constraint)?.FeatureStruct; } + internal FeatureStruct LeftEdgeConstraint { get; } + internal FeatureStruct RightEdgeConstraint { get; } + public override Word ApplyRhs(PatternRule rule, Match match) { Word output = match.Input.Clone(); diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index 7e841b1fb..4a231351a 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -12,6 +12,7 @@ public class AnalysisAffixProcessRule : IRule private readonly Morpher _morpher; private readonly AffixProcessRule _rule; private readonly List> _rules; + private readonly List _ruleSpecs; private readonly FeatureStruct _outputFS; public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) @@ -26,11 +27,14 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) _outputFS.Freeze(); _rules = new List>(); + _ruleSpecs = new List(); foreach (AffixProcessAllomorph allo in rule.Allomorphs) { + var spec = new AnalysisAffixProcessAllomorphRuleSpec(allo) { PruningMorpher = morpher }; + _ruleSpecs.Add(spec); _rules.Add( new MultiplePatternRule( - new AnalysisAffixProcessAllomorphRuleSpec(allo) { PruningMorpher = morpher }, + spec, new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, @@ -44,6 +48,22 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) } } + private static ShapeNode FindEdge(Shape shape, bool first) + { + ShapeNode node = first ? shape.First : shape.Last; + ShapeNode end = first ? shape.End : shape.Begin; + while (node != null && node != end && node.Type() != HCFeatureSystem.Segment) + node = first ? node.Next : node.Prev; + return node == end ? null : node; + } + + // The matcher can skip optional annotations, so only a mandatory edge can reject a match. + private static bool EdgePasses(FeatureStruct constraint, ShapeNode node) => + constraint == null + || node == null + || node.Annotation.Optional + || node.Annotation.FeatureStruct.IsUnifiable(constraint, false, null); + public IEnumerable Apply(Word input) { if (!_morpher.RuleSelector(_rule)) @@ -57,9 +77,24 @@ public IEnumerable Apply(Word input) return Enumerable.Empty(); } + ShapeNode first = _morpher.EdgePrefilterEnabled ? FindEdge(input.Shape, true) : null; + ShapeNode last = _morpher.EdgePrefilterEnabled ? FindEdge(input.Shape, false) : null; var output = new List(); for (int i = 0; i < _rules.Count; i++) { + if ( + _morpher.EdgePrefilterEnabled + && ( + !EdgePasses(_ruleSpecs[i].LeftEdgeConstraint, first) + || !EdgePasses(_ruleSpecs[i].RightEdgeConstraint, last) + ) + ) + { + _morpher.EdgePrefilterRejected?.Invoke(); + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.MorphologicalRuleNotUnapplied(_rule, i, input); + continue; + } bool unapplied = false; foreach (Word outWord in _rules[i].Apply(input).RemoveDuplicates()) { diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs new file mode 100644 index 000000000..4f8ee9c4f --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs @@ -0,0 +1,355 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +[TestFixture] +public class AnalysisEdgePrefilterTests : HermitCrabTestBase +{ + private AffixProcessRule SuffixRuleAppending(string literal) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var rule = new AffixProcessRule + { + Id = "S1", + Name = "s1_suffix", + Gloss = "S1", + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, literal) }, + } + ); + return rule; + } + + private AffixProcessRule PrefixRulePrepending(string literal) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var rule = new AffixProcessRule + { + Id = "P1", + Name = "p1_prefix", + Gloss = "P1", + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new InsertSegments(Table3, literal), new CopyFromInput("1") }, + } + ); + return rule; + } + + private AffixProcessRule CircumfixRule(string prefixLiteral, string suffixLiteral) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var rule = new AffixProcessRule + { + Id = "C1", + Name = "c1_circumfix", + Gloss = "C1", + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = + { + new InsertSegments(Table3, prefixLiteral), + new CopyFromInput("1"), + new InsertSegments(Table3, suffixLiteral), + }, + } + ); + return rule; + } + + private Word FrozenInput(string word, bool optionalLast = false) + { + Shape shape = Morphophonemic.CharacterDefinitionTable.Segment(word); + if (optionalLast) + shape.Last.Annotation.Optional = true; + var input = new Word(Morphophonemic, shape); + input.Freeze(); + return input; + } + + private Word[] Apply(AffixProcessRule rule, string word, bool enabled, bool optionalLast = false) + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) { EdgePrefilterEnabled = enabled }; + return new AnalysisAffixProcessRule(morpher, rule).Apply(FrozenInput(word, optionalLast)).ToArray(); + } + + private static void AssertEquivalent(Word[] enabled, Word[] disabled) + { + Assert.That( + enabled.Select(w => w.ToString()).OrderBy(s => s, StringComparer.Ordinal), + Is.EqualTo(disabled.Select(w => w.ToString()).OrderBy(s => s, StringComparer.Ordinal)) + ); + } + + [Test] + public void MismatchingEdgeSkipsMatcher() + { + int rejected = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + EdgePrefilterRejected = () => rejected++, + }; + var rule = new AnalysisAffixProcessRule(morpher, SuffixRuleAppending("d")); + Assert.That(rule.Apply(FrozenInput("sag")), Is.Empty); + Assert.That(rejected, Is.EqualTo(1)); + morpher.EdgePrefilterEnabled = false; + Assert.That(rule.Apply(FrozenInput("sag")), Is.Empty); + Assert.That(rejected, Is.EqualTo(1)); + } + + [Test] + public void RejectedEdgeEmitsTheSameFailureTraceAsMatcher() + { + var traceManager = new TraceManager { IsTracing = true }; + AffixProcessRule rule = SuffixRuleAppending("d"); + var input = FrozenInput("sag"); + int rejected = 0; + var morpher = new Morpher(traceManager, Language, maxDegreeOfParallelism: 1) + { + EdgePrefilterRejected = () => rejected++, + }; + var analysisRule = new AnalysisAffixProcessRule(morpher, rule); + + Trace[] Run(bool enabled) + { + morpher.EdgePrefilterEnabled = enabled; + traceManager.AnalyzeWord(Language, input); + var root = (Trace)input.CurrentTrace; + Assert.That(analysisRule.Apply(input), Is.Empty); + return root.Children.ToArray(); + } + + Trace[] matcher = Run(false); + Trace[] prefilter = Run(true); + Assert.That(rejected, Is.EqualTo(1)); + Assert.That(matcher, Has.Length.EqualTo(1)); + Assert.That(prefilter, Has.Length.EqualTo(1)); + foreach (Trace trace in matcher.Concat(prefilter)) + { + Assert.That(trace.Type, Is.EqualTo(TraceType.MorphologicalRuleAnalysis)); + Assert.That(trace.Source, Is.SameAs(rule)); + Assert.That(trace.SubruleIndex, Is.Zero); + Assert.That(trace.Input, Is.SameAs(input)); + Assert.That(trace.Output, Is.Null); + } + } + + [TestCase("sagd")] + [TestCase("sast")] + public void AlphaVariableEdgePassesWithUnboundVariable(string word) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var consonant = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons+") + .Value; + var alphaConsonant = FeatureStruct + .New(Language.PhonologicalFeatureSystem, consonant) + .Feature("vd") + .EqualToVariable("a") + .Value; + var rule = new AffixProcessRule { Id = "alpha", Name = "alpha" }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Annotation(alphaConsonant).Value }, + Rhs = { new CopyFromInput("1"), new InsertSimpleContext(consonant, Variable("vd", "a")) }, + } + ); + var spec = new AnalysisAffixProcessAllomorphRuleSpec(rule.Allomorphs[0]); + Assert.That(spec.RightEdgeConstraint.HasVariables, Is.True); + int rejected = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + EdgePrefilterRejected = () => rejected++, + }; + var analysisRule = new AnalysisAffixProcessRule(morpher, rule); + Word input = FrozenInput(word); + Word[] enabled = analysisRule.Apply(input).ToArray(); + morpher.EdgePrefilterEnabled = false; + Word[] disabled = analysisRule.Apply(input).ToArray(); + Assert.That(rejected, Is.Zero); + Assert.That(enabled, Is.Not.Empty); + AssertEquivalent(enabled, disabled); + } + + [Test] + public void GroupFirstWithOptionalLeadingConstraintDoesNotReject() + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var rule = new AffixProcessRule { Id = "optional", Name = "optional" }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern + .New("1") + .Annotation(Character(Table3, "z")) + .Optional.Annotation(any) + .OneOrMore.Value, + }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "d") }, + } + ); + var spec = new AnalysisAffixProcessAllomorphRuleSpec(rule.Allomorphs[0]); + Assert.That(spec.Pattern.Children.First, Is.InstanceOf>()); + Assert.That(spec.LeftEdgeConstraint, Is.Null); + int rejected = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + EdgePrefilterRejected = () => rejected++, + }; + var analysisRule = new AnalysisAffixProcessRule(morpher, rule); + Word input = FrozenInput("sagd"); + Word[] enabled = analysisRule.Apply(input).ToArray(); + morpher.EdgePrefilterEnabled = false; + Word[] disabled = analysisRule.Apply(input).ToArray(); + Assert.That(rejected, Is.Zero); + Assert.That(enabled, Is.Not.Empty); + AssertEquivalent(enabled, disabled); + } + + [Test] + public void DigraphIsOneSegmentAtTheEdge() + { + AffixProcessRule rule = SuffixRuleAppending("ats"); + var spec = new AnalysisAffixProcessAllomorphRuleSpec(rule.Allomorphs[0]); + Assert.That(spec.RightEdgeConstraint, Is.Not.Null); + Assert.That( + Morphophonemic.CharacterDefinitionTable.Segment("sagats").Count(n => n.Type() == HCFeatureSystem.Segment), + Is.EqualTo(5) + ); + int rejected = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + EdgePrefilterRejected = () => rejected++, + }; + var analysisRule = new AnalysisAffixProcessRule(morpher, rule); + Word[] enabled = analysisRule.Apply(FrozenInput("sagats")).ToArray(); + Assert.That(enabled, Is.Not.Empty); + Assert.That(analysisRule.Apply(FrozenInput("sagat")), Is.Empty); + Assert.That(rejected, Is.EqualTo(1)); + morpher.EdgePrefilterEnabled = false; + AssertEquivalent(enabled, analysisRule.Apply(FrozenInput("sagats")).ToArray()); + Assert.That(analysisRule.Apply(FrozenInput("sagat")), Is.Empty); + } + + [Test] + public void SuffixEdgeMismatch_PrefilterMatchesFullMatcher() + { + AffixProcessRule rule = SuffixRuleAppending("d"); + + Word[] enabled = Apply(rule, "sag", true); + Word[] disabled = Apply(rule, "sag", false); + + AssertEquivalent(enabled, disabled); + Assert.That(enabled, Is.Empty); + } + + [Test] + public void SuffixEdgeMatch_PrefilterMatchesFullMatcher() + { + AffixProcessRule rule = SuffixRuleAppending("d"); + + Word[] enabled = Apply(rule, "sagd", true); + Word[] disabled = Apply(rule, "sagd", false); + + AssertEquivalent(enabled, disabled); + Assert.That(enabled, Is.Not.Empty); + } + + [Test] + public void BoundaryNodesDoNotHideSegmentEdges() + { + AffixProcessRule rule = SuffixRuleAppending("d"); + Word[] enabled = Apply(rule, "+sagd+", true); + Word[] disabled = Apply(rule, "+sagd+", false); + AssertEquivalent(enabled, disabled); + Assert.That(enabled, Is.Not.Empty); + } + + [Test] + public void OptionalEdgeNode_PrefilterMatchesFullMatcher() + { + AffixProcessRule rule = SuffixRuleAppending("d"); + + Word[] enabled = Apply(rule, "sadg", true, optionalLast: true); + Word[] disabled = Apply(rule, "sadg", false, optionalLast: true); + AssertEquivalent(enabled, disabled); + Assert.That(enabled, Is.Not.Empty); + } + + [TestCase("sag", false)] + [TestCase("zag", true)] + public void PrefixEdge_PrefilterMatchesFullMatcher(string word, bool matches) + { + AffixProcessRule rule = PrefixRulePrepending("z"); + + Word[] enabled = Apply(rule, word, true); + Word[] disabled = Apply(rule, word, false); + + AssertEquivalent(enabled, disabled); + Assert.That(enabled.Any(), Is.EqualTo(matches)); + } + + [TestCase("sag", false)] + [TestCase("zag", false)] + [TestCase("sad", false)] + [TestCase("zad", true)] + public void CircumfixEdges_PrefilterMatchesFullMatcher(string word, bool matches) + { + AffixProcessRule rule = CircumfixRule("z", "d"); + + Word[] enabled = Apply(rule, word, true); + Word[] disabled = Apply(rule, word, false); + + AssertEquivalent(enabled, disabled); + Assert.That(enabled.Any(), Is.EqualTo(matches)); + } + + [TestCase("sag", true)] + [TestCase("sas", false)] + public void NaturalClassEdge_PrefilterMatchesFullMatcher(string word, bool matches) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + var rule = new AffixProcessRule + { + Id = "NC1", + Name = "nc1_suffix", + Gloss = "NC1", + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSimpleContext(voiced) }, + } + ); + + Word[] enabled = Apply(rule, word, true); + Word[] disabled = Apply(rule, word, false); + + AssertEquivalent(enabled, disabled); + Assert.That(enabled.Any(), Is.EqualTo(matches)); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/generate.py b/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/generate.py new file mode 100644 index 000000000..2bb761573 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/generate.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Generates the early-lexicon-agglutinative stress grammar and its words file. + +A new FieldWorks project in a Quechua/Turkic-type language: one unordered stratum of 25 suffixal rules (pure +concatenation, multipleApplication 1, several with 1-2 segment allomorphs), a vowel-harmony phonological rule in +the same stratum (a surfaces as e after a front vowel and any consonants), 20 roots, and 60 words: 20 with a +lexicon root and 40 whose root is absent from the lexicon, each carrying 3-6 suffixes. +usage: generate.py (writes grammar.xml and words.txt beside the script) +""" +import os, random +from xml.sax.saxutils import escape + +here = os.path.dirname(os.path.abspath(__file__)) +FEATS = ["syl", "cons", "son", "nas", "voice", "cont", "high", "low", "back", "round"] +V = dict(syl="+", cons="-", son="+", nas="-", voice="+", cont="+", high="-", low="-", back="-", round="-") +C = dict(syl="-", cons="+", son="-", nas="-", voice="+", cont="-", high="-", low="-", back="-", round="-") + + +def seg(base, **kw): + d = dict(base) + d.update(kw) + return d + + +SEGS = { + "a": seg(V, low="+", back="+"), + "e": seg(V), + "i": seg(V, high="+"), + "o": seg(V, back="+", round="+"), + "u": seg(V, high="+", back="+", round="+"), + "b": seg(C), + "d": seg(C, high="+"), + "g": seg(C, back="+"), + "k": seg(C, voice="-", back="+"), + "l": seg(C, son="+", cont="+"), + "m": seg(C, son="+", nas="+"), + "n": seg(C, son="+", nas="+", high="+"), + "p": seg(C, voice="-"), + "r": seg(C, son="+", cont="+", high="+"), + "s": seg(C, voice="-", cont="+"), + "t": seg(C, voice="-", high="+"), + "y": seg(C, cons="-", son="+", cont="+", high="+"), +} +assert len({tuple(sorted(d.items())) for d in SEGS.values()}) == len(SEGS) +FRONT = "ei" +CONS = "bdgklmnprsty" +# (gloss, allomorphs by underlying form); 25 rules, 7 with 1-2 segment allomorphs +RULES = [ + ("PL", ["lar"]), ("P1", ["um"]), ("P2", ["un"]), ("P3", ["si", "i"]), ("ACC", ["ni", "yi"]), + ("DAT", ["ka", "a"]), ("LOC", ["da"]), ("ABL", ["dan"]), ("GEN", ["nin"]), ("INST", ["la"]), + ("COM", ["bla"]), ("EQU", ["ga"]), ("NEG", ["ma"]), ("PAST", ["di", "ti"]), ("EVID", ["mis"]), + ("FUT", ["asak"]), ("COND", ["sa"]), ("IMP", ["sin"]), ("ABIL", ["abil"]), ("CAUS", ["tir"]), + ("PASS", ["il", "n"]), ("RECIP", ["is"]), ("REFL", ["in"]), ("DESID", ["gi"]), ("AGENT", ["ki"]), +] +ROOTS_BACK = ["kat", "pol", "mur", "tok", "bas", "gun", "dar", "lop", "sut", "nok"] +ROOTS_FRONT = ["kel", "pir", "sen", "bil", "des", "git", "ner", "tip", "mek", "lis"] +ROOTS = ROOTS_BACK + ROOTS_FRONT +assert len(RULES) == 25 and len(ROOTS) == 20 + + +def harmony(s): + out = [] + for ch in s: + if ch == "a": + j = len(out) - 1 + while j >= 0 and out[j] in CONS: + j -= 1 + if j >= 0 and out[j] in FRONT: + ch = "e" + out.append(ch) + return "".join(out) + + +def grammar(): + o = ['', '', "", " "] + o.append(" early-lexicon-agglutinative") + o.append(' x') + o.append(" ") + for f in FEATS: + o.append(f' {f}+-') + o.append(" ") + o.append(" ") + o.append(' Main') + for s, d in SEGS.items(): + fv = "".join(f'' for f in FEATS) + o.append(f' {s}{fv}') + o.append(" ") + o.append(" ") + o.append(' Any') + o.append(' FrontV') + o.append(' Cons') + o.append(" ") + o.append(" ") + o.append( + ' vowel harmony' + '' + "" + '' + "" + '' + '' + "" + "" + ) + o.append(" ") + ids = " ".join(f"mr_{g}" for g, _ in RULES) + o.append(f' Main') + o.append("") + for g, allos in RULES: + subs = "" + for k, a in enumerate(allos): + subs += ( + f'' + '' + f'' + f"{a}" + ) + o.append( + f'{g}' + f"{subs}{g}{g}" + ) + o.append("") + for r in ROOTS: + o.append(f'{r}{r}{r}') + o.append("") + return "\n".join(o) + "\n" + + +def words(): + rnd = random.Random(562) + ws = [] + + def chain(n): + return rnd.sample(RULES, n) + + def surface(stem, rules): + s = stem + for _g, allos in rules: + s = s + rnd.choice(allos) + return harmony(s) + + for _ in range(20): + ws.append(surface(rnd.choice(ROOTS), chain(rnd.randint(1, 3)))) + unknown = set() + while len(unknown) < 40: + stem = rnd.choice(CONS) + rnd.choice("aeiou") + rnd.choice(CONS) + if stem not in ROOTS: + unknown.add(stem) + for stem in sorted(unknown): + ws.append(surface(stem, chain(rnd.randint(3, 6)))) + rnd.shuffle(ws) + return ws + + +with open(os.path.join(here, "grammar.xml"), "w", encoding="utf-8") as f: + f.write(grammar()) +with open(os.path.join(here, "words.txt"), "w", encoding="utf-8") as f: + f.write("\n".join(words()) + "\n") diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/grammar.xml b/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/grammar.xml new file mode 100644 index 000000000..b8d608fee --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/grammar.xml @@ -0,0 +1,95 @@ + + + + + early-lexicon-agglutinative + x + + syl+- + cons+- + son+- + nas+- + voice+- + cont+- + high+- + low+- + back+- + round+- + + + Main + a + e + i + o + u + b + d + g + k + l + m + n + p + r + s + t + y + + + Any + FrontV + Cons + + + vowel harmony + + Main + +PLlarPLPL +P1umP1P1 +P2unP2P2 +P3siiP3P3 +ACCniyiACCACC +DATkaaDATDAT +LOCdaLOCLOC +ABLdanABLABL +GENninGENGEN +INSTlaINSTINST +COMblaCOMCOM +EQUgaEQUEQU +NEGmaNEGNEG +PASTditiPASTPAST +EVIDmisEVIDEVID +FUTasakFUTFUT +CONDsaCONDCOND +IMPsinIMPIMP +ABILabilABILABIL +CAUStirCAUSCAUS +PASSilnPASSPASS +RECIPisRECIPRECIP +REFLinREFLREFL +DESIDgiDESIDDESID +AGENTkiAGENTAGENT + +katkatkat +polpolpol +murmurmur +toktoktok +basbasbas +gungungun +dardardar +loploplop +sutsutsut +noknoknok +kelkelkel +pirpirpir +sensensen +bilbilbil +desdesdes +gitgitgit +nernerner +tiptiptip +mekmekmek +lislislis + diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/words.txt b/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/words.txt new file mode 100644 index 000000000..5ffd5c01f --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/Research/StressFixtures/early-lexicon-agglutinative/words.txt @@ -0,0 +1,60 @@ +polsi +subissime +bille +katn +metkegeyiundanbla +lolisninseinkide +mekdeseum +dayblaabilsidenkiun +pokkiummisninebildi +nagunmisse +tigesekuminlede +negkemissinkiin +katgisin +didtiyimesi +dukgiiebilun +darga +polundanasak +bibiebildedenis +rugniilmissinble +mekge +yermisesekni +gundimis +kelsin +tatisesekinimissin +lapsinkemisti +piyikiebil +denkeebildeblege +yatabiltirilinge +gatagalarumsigi +yupdadiille +kelgiun +sekdiunblala +yimseleum +kelninmis +tipinleble +darun +kumninesekis +gidnininesekniki +likkikese +lagblaisgeninsin +pabnigelededennin +boksingeble +randasinkegi +bobkamislergime +demgiselelermis +seltirsimediin +kidingibleninse +toksin +riggebleleris +rasilmisis +sutblalar +gittirler +tipisunbla +pirkisium +darasakmis +nurlarinninuntiris +kegisinkileede +kosasinninumgagi +nemungiebilkidesin +lodundakiinsingi From 614f9a71ffa4c43a21b7a221f206ed098c319284 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Sat, 31 Oct 2026 22:05:39 -0400 Subject: [PATCH 2/3] Full parses now match with the edge prefilter on and off, sequentially and in parallel Co-Authored-By: Claude Sonnet 5.5 --- .../AnalysisEdgePrefilterTests.cs | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs index 4f8ee9c4f..d827e5e51 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AnalysisEdgePrefilterTests.cs @@ -2,6 +2,7 @@ using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; @@ -94,6 +95,25 @@ private static void AssertEquivalent(Word[] enabled, Word[] disabled) ); } + private static string StressFixturePath(string file) => + Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "Research", + "StressFixtures", + "early-lexicon-agglutinative", + file + ); + + private static string[] ParseSignatures(Morpher morpher, string word) => + morpher + .ParseWord(word) + .Select(MorpherTests.WordAnalysisSignature) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + [Test] public void MismatchingEdgeSkipsMatcher() { @@ -352,4 +372,95 @@ public void NaturalClassEdge_PrefilterMatchesFullMatcher(string word, bool match AssertEquivalent(enabled, disabled); Assert.That(enabled.Any(), Is.EqualTo(matches)); } + + [Test] + public void ParseWord_OptionalEdgeFromUnappliedDeletion_PrefilterMatchesFullMatcher() + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var highVowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons-") + .Symbol("voc+") + .Symbol("high+") + .Value; + var highFrontUnrndVowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons-") + .Symbol("voc+") + .Symbol("high+") + .Symbol("low-") + .Symbol("back-") + .Symbol("round-") + .Value; + var deletion = new RewriteRule + { + Name = "delete_i", + Lhs = Pattern.New().Annotation(highFrontUnrndVowel).Value, + }; + deletion.Subrules.Add( + new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(highVowel).Value } + ); + Allophonic.PhonologicalRules.Add(deletion); + var uSuffix = new AffixProcessRule + { + Id = "U1", + Name = "u_suffix", + Gloss = "U1", + }; + uSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "u") }, + } + ); + Morphophonemic.MorphologicalRules.Add(uSuffix); + + // Unapplying the deletion leaves an optional i after the final u, so the u suffix is reachable only + // by skipping an optional edge node. + foreach (int degree in new[] { 1, 0 }) + { + var on = new Morpher(TraceManager, Language, degree); + var off = new Morpher(TraceManager, Language, degree) { EdgePrefilterEnabled = false }; + foreach (string word in new[] { "puu", "pu", "pi", "bubuu" }) + { + Assert.That( + ParseSignatures(on, word), + Is.EqualTo(ParseSignatures(off, word)), + $"'{word}' degree {degree}" + ); + } + Assert.That(ParseSignatures(on, "puu"), Is.Not.Empty); + } + } + + [TestCase(1)] + [TestCase(0)] + public void ParseWord_StressFixture_PrefilterMatchesFullMatcher(int maxDegreeOfParallelism) + { + Language language = XmlLanguageLoader.Load(StressFixturePath("grammar.xml")); + string[] words = File.ReadAllLines(StressFixturePath("words.txt")) + .Where(line => line.Length > 0) + .Take(60) + .ToArray(); + int rejected = 0; + var on = new Morpher(TraceManager, language, maxDegreeOfParallelism) + { + EdgePrefilterRejected = () => Interlocked.Increment(ref rejected), + }; + var off = new Morpher(TraceManager, language, maxDegreeOfParallelism) { EdgePrefilterEnabled = false }; + + int parsed = 0; + foreach (string word in words) + { + string[] expected = ParseSignatures(off, word); + Assert.That(ParseSignatures(on, word), Is.EqualTo(expected), $"'{word}' parse differs with the prefilter"); + if (expected.Length > 0) + parsed++; + } + Assert.That(parsed, Is.GreaterThan(0)); + Assert.That(rejected, Is.GreaterThan(0)); + } } From 0af3bc0b24828205f017d0ebd4872274508859ee Mon Sep 17 00:00:00 2001 From: John Lambert Date: Sat, 31 Oct 2026 22:05:43 -0400 Subject: [PATCH 3/3] State that the edge prefilter rejection callback runs on analysis threads Co-Authored-By: Claude Sonnet 5.5 --- src/SIL.Machine.Morphology.HermitCrab/Morpher.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 6094ece12..4492facdf 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -19,6 +19,8 @@ namespace SIL.Machine.Morphology.HermitCrab public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator { internal bool EdgePrefilterEnabled { get; set; } = true; + + // Called from analysis threads, so a handler must be thread-safe. internal Action EdgePrefilterRejected { get; set; } private readonly Language _lang;