From 0bf9e6468a5b84be731cabc26fd8fd70e80d0f52 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 27 Oct 2026 21:02:39 -0400 Subject: [PATCH 1/4] Prune affix template partials whose required slot cannot find its material An analysis template partial now stops as soon as a required slot can no longer find the segments its affix inserts, instead of paying for every optional slot before the required one. The test is used only while every slot walked since is a pure copy-and-insert rule, so parses do not change. A count-only shadow mode and a null-by-default check observer let a harness verify soundness and cost on real grammars. Closes #561 Co-Authored-By: Claude Opus 5.5 --- .../AnalysisAffixTemplateRule.cs | 114 ++++- .../Morpher.cs | 20 + .../RequiredSlotMaterial.cs | 264 ++++++++++++ .../AnalysisAffixTemplateRuleTests.cs | 403 ++++++++++++++++++ 4 files changed, 790 insertions(+), 11 deletions(-) create mode 100644 src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index 9044a8488..d46b5971b 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -14,6 +14,7 @@ internal class AnalysisAffixTemplateRule : IRule private readonly Morpher _morpher; private readonly AffixTemplate _template; private readonly List> _rules; + private readonly RequiredSlotMaterial _lookahead; public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) { @@ -26,6 +27,7 @@ public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) FreezableEqualityComparer.Default )) ); + _lookahead = new RequiredSlotMaterial(template); } public IEnumerable Apply(Word input) @@ -61,24 +63,96 @@ public IEnumerable Apply(Word input) return Enumerable.Empty(); } + int[] fired = null; + if (RequiredSlotCannotUnapply(input, _rules.Count - 1, ref fired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, input, false, FailureReason.None); + return Enumerable.Empty(); + } + Word inWord = input.Clone(); inWord.Freeze(); var output = new HashSet(FreezableEqualityComparer.Default); if (_morpher.MaxDegreeOfParallelism == 1) - ApplySlots(inWord, _rules.Count - 1, output); + ApplySlots(inWord, _rules.Count - 1, output, fired); else - ParallelApplySlots(inWord, output); + ParallelApplySlots(inWord, output, fired); return output; } - private void ApplySlots(Word inWord, int index, HashSet output) + // Testing runs from the next slot to unapply and stops after an impure slot, which makes the test + // for any slot behind it unsound. Shadow mode prunes nothing and carries the fired slots in fired. + private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired) + { + RequiredSlotLookaheadMode mode = _morpher.RequiredSlotLookahead; + if (mode == RequiredSlotLookaheadMode.Off) + return false; + + long unifications = 0; + bool cannot = false; + for (int j = lastSlot; j >= 0; j--) + { + if ( + !_template.Slots[j].Optional + && !IsFired(fired, j) + && _lookahead.CannotUnapply(j, word.Shape, ref unifications) + ) + { + if (mode == RequiredSlotLookaheadMode.On) + { + cannot = true; + break; + } + fired = fired == null ? new[] { j } : fired.Append(j).ToArray(); + _morpher.RequiredSlotShadow?.Invoke("fired", _template, j); + } + if (!_lookahead.IsPure(j)) + break; + } + if (unifications != 0) + _morpher.RequiredSlotCheckObserver?.Invoke(unifications); + return cannot; + } + + private static bool IsFired(int[] fired, int slot) + { + return fired != null && Array.IndexOf(fired, slot) >= 0; + } + + private void ReportShadow(string kind, int[] fired, int slot) + { + if (fired == null || _morpher.RequiredSlotShadow == null) + return; + if (slot < 0) + { + foreach (int j in fired) + _morpher.RequiredSlotShadow(kind, _template, j); + } + else if (IsFired(fired, slot)) + { + _morpher.RequiredSlotShadow(kind, _template, slot); + } + } + + private void ApplySlots(Word inWord, int index, HashSet output, int[] fired) { for (int i = index; i >= 0; i--) { foreach (Word outWord in _rules[i].Apply(inWord)) - ApplySlots(outWord, i - 1, output); + { + ReportShadow("violated", fired, i); + int[] outFired = fired; + if (RequiredSlotCannotUnapply(outWord, i - 1, ref outFired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, outWord, false, FailureReason.None); + continue; + } + ApplySlots(outWord, i - 1, output, outFired); + } if (!_template.Slots[i].Optional) { @@ -90,16 +164,17 @@ private void ApplySlots(Word inWord, int index, HashSet output) if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None); + ReportShadow("completed-under-fired", fired, -1); output.Add(inWord); } - private void ParallelApplySlots(Word inWord, HashSet output) + private void ParallelApplySlots(Word inWord, HashSet output, int[] fired) { ParallelOptions parallelOptions = _morpher.CreateParallelOptions(); var outStack = new ConcurrentStack(); - var from = new ConcurrentStack>(); - from.Push(Tuple.Create(inWord, _rules.Count - 1)); - var to = new ConcurrentStack>(); + var from = new ConcurrentStack>(); + from.Push(Tuple.Create(inWord, _rules.Count - 1, fired)); + var to = new ConcurrentStack>(); while (!from.IsEmpty) { to.Clear(); @@ -111,9 +186,25 @@ private void ParallelApplySlots(Word inWord, HashSet output) bool add = true; for (int i = work.Item2; i >= 0; i--) { - Tuple[] workItems = _rules[i] + int next = i - 1; + Tuple[] workItems = _rules[i] .Apply(work.Item1) - .Select(res => Tuple.Create(res, i - 1)) + .Select(res => + { + ReportShadow("violated", work.Item3, i); + int[] resFired = work.Item3; + if (!RequiredSlotCannotUnapply(res, next, ref resFired)) + return Tuple.Create(res, next, resFired); + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate( + _template, + res, + false, + FailureReason.None + ); + return null; + }) + .Where(item => item != null) .ToArray(); if (workItems.Length > 0) to.PushRange(workItems); @@ -141,11 +232,12 @@ private void ParallelApplySlots(Word inWord, HashSet output) true, FailureReason.None ); + ReportShadow("completed-under-fired", work.Item3, -1); outStack.Push(work.Item1); } } ); - ConcurrentStack> temp = from; + ConcurrentStack> temp = from; from = to; to = temp; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index d0675509e..ca906f91e 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -83,6 +83,26 @@ public ITraceManager TraceManager public int DeletionReapplications { get; set; } + /// + /// Whether affix template walks drop partials in which a required slot can no longer unapply because + /// the material it inserts is absent. See . + /// + internal RequiredSlotLookaheadMode RequiredSlotLookahead { get; set; } = RequiredSlotLookaheadMode.On; + + /// + /// Called in with a kind and the template and slot it is about: + /// "fired" (the test is true for a required slot), "violated" (a rule of a fired slot then unapplied under + /// that partial) and "completed-under-fired" (the template walk completed with a fired slot on its path). + /// Any "violated" or "completed-under-fired" is a soundness failure. + /// + internal Action RequiredSlotShadow { get; set; } + + /// + /// Called after each required-slot check that unified, with the number of unifications it performed. + /// Null unless a diagnostic sets it; it may be called from several threads. + /// + internal Action RequiredSlotCheckObserver { get; set; } + public int MaxStemCount { get; set; } /// diff --git a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs new file mode 100644 index 000000000..27e60caa6 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs @@ -0,0 +1,264 @@ +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// How uses . + /// + internal enum RequiredSlotLookaheadMode + { + /// The look-ahead is not evaluated. + Off, + + /// The look-ahead is evaluated and reported to Morpher.RequiredSlotShadow, never pruning. + Shadow, + + /// The look-ahead prunes template partials. + On, + } + + /// + /// A necessary condition for a required affix template slot to unapply later in a template walk: some + /// allomorph of some rule in the slot must have every inserted segment unify with a segment of the shape. + /// + /// + /// The tables are built once from the template and depend only on the grammar. + /// + /// Soundness. The analysis pattern of an allomorph holds one top-level constraint per inserted segment + /// (, ), + /// and the rule can only unapply if each constraint unifies with a distinct segment node of the shape. + /// A rule is pure when every right-hand side action copies an input part or inserts material, and every + /// left-hand part is copied by a that must capture a segment, or has no + /// segment to restore. Then only copies input + /// nodes and never untruncates a segment. Unification can only narrow the feature structure of a copied + /// node, so a constraint that unifies with no node of the shape unifies with no node of the shape after + /// any number of pure unapplications. Inside one template walk only the rules of that template's slots + /// run, and the stratum's phonological rules ran before the walk. The test for slot j is therefore used + /// only when every rule in the slots still to be walked after j is pure. Optional nodes count as present + /// because the matcher can match them. Anything that cannot be summarised (a null affix, a constraint with + /// variables, an unknown rule type) leaves the slot unprunable. + /// + internal class RequiredSlotMaterial + { + private readonly SlotSummary[] _slots; + + public RequiredSlotMaterial(AffixTemplate template) + { + _slots = template.Slots.Select(Summarize).ToArray(); + } + + /// + /// Whether every rule of the slot is pure, so that unapplying the slot leaves the segments needed by + /// slots before it no easier to find. + /// + public bool IsPure(int slot) + { + return _slots[slot].IsPure; + } + + /// + /// True if no allomorph of the slot can unapply to a word with this shape, provided that every slot + /// walked between this word and the slot is pure. + /// + /// The slot to test. + /// The shape of the partial. + /// Incremented by the feature structure unifications performed. + public bool CannotUnapply(int slot, Shape shape, ref long unifications) + { + SlotSummary summary = _slots[slot]; + if (summary.MayAlwaysUnapply) + return false; + + // The constraints of a slot repeat across its allomorphs, so each distinct one is looked up once. + ulong known = 0; + ulong present = 0; + foreach (int[] requirements in summary.Allomorphs) + { + bool all = true; + foreach (int index in requirements) + { + bool found; + ulong bit = index < 64 ? 1UL << index : 0; + if ((known & bit) != 0) + { + found = (present & bit) != 0; + } + else + { + found = IsPresent(summary.Distinct[index], shape, ref unifications); + known |= bit; + if (found) + present |= bit; + } + if (!found) + { + all = false; + break; + } + } + if (all) + return false; + } + return true; + } + + private static bool IsPresent(FeatureStruct required, Shape shape, ref long unifications) + { + foreach (ShapeNode node in shape) + { + if (node.Type() != HCFeatureSystem.Segment) + continue; + + unifications++; + if (node.Annotation.FeatureStruct.IsUnifiable(required, false, null)) + return true; + } + return false; + } + + private static SlotSummary Summarize(AffixTemplateSlot slot) + { + var summary = new SlotSummary(); + var allomorphs = new List(); + foreach (MorphemicMorphologicalRule rule in slot.Rules) + { + IList ruleAllomorphs; + switch (rule) + { + case AffixProcessRule affixRule: + ruleAllomorphs = affixRule.Allomorphs; + break; + case RealizationalAffixProcessRule realizationalRule: + ruleAllomorphs = realizationalRule.Allomorphs; + break; + default: + summary.MayAlwaysUnapply = true; + summary.IsPure = false; + continue; + } + + foreach (AffixProcessAllomorph allomorph in ruleAllomorphs) + { + if (!IsPure(allomorph)) + summary.IsPure = false; + + FeatureStruct[] requirements = GetRequirements(allomorph); + if (requirements.Length == 0) + summary.MayAlwaysUnapply = true; + else + allomorphs.Add(requirements); + } + } + var distinct = new List(); + summary.Allomorphs = allomorphs + .Select(requirements => + requirements + .Select(fs => + { + int index = distinct.FindIndex(d => d.ValueEquals(fs)); + if (index < 0) + { + distinct.Add(fs); + index = distinct.Count - 1; + } + return index; + }) + .ToArray() + ) + .ToArray(); + summary.Distinct = distinct.ToArray(); + return summary; + } + + private static FeatureStruct[] GetRequirements(AffixProcessAllomorph allomorph) + { + var requirements = new List(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (action is InsertSegments insertSegments) + { + foreach (ShapeNode node in insertSegments.Segments.Shape) + { + if (node.Type() != HCFeatureSystem.Boundary) + AddRequirement(requirements, node.Annotation.FeatureStruct); + } + } + else if (action is InsertSimpleContext insertContext) + { + AddRequirement(requirements, insertContext.SimpleContext.FeatureStruct); + } + } + return requirements.ToArray(); + } + + // A constraint with variables is bound by earlier matches, and one that is not a segment is never + // matched against the segments the matcher considers, so neither can be tested here. + private static void AddRequirement(List requirements, FeatureStruct fs) + { + if (fs.HasVariables || (FeatureSymbol)fs.GetValue(HCFeatureSystem.Type) != HCFeatureSystem.Segment) + return; + + FeatureStruct copy = fs.Clone(); + copy.Freeze(); + requirements.Add(copy); + } + + private static bool IsPure(AffixProcessAllomorph allomorph) + { + var copied = new HashSet(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (action is CopyFromInput) + copied.Add(action.PartName); + else if (!(action is InsertSegments || action is InsertSimpleContext)) + return false; + } + + foreach (Pattern part in allomorph.Lhs) + { + if (!HasSegmentConstraint(part)) + continue; + if (!copied.Contains(part.Name) || MinSegments(part) == 0) + return false; + } + return true; + } + + private static bool HasSegmentConstraint(PatternNode node) + { + if (node is Constraint constraint) + return constraint.Type() == HCFeatureSystem.Segment; + return node.Children.Any(HasSegmentConstraint); + } + + // The fewest segments a successful match of the node must consume. A part matching none is captured + // as a null range, and GenerateShape then untruncates it, which adds segments that are not copies. + private static int MinSegments(PatternNode node) + { + switch (node) + { + case Constraint constraint: + return constraint.Type() == HCFeatureSystem.Segment ? 1 : 0; + case Quantifier quantifier: + return quantifier.MinOccur * quantifier.Children.Sum(MinSegments); + case Alternation alternation: + return alternation.Children.Count == 0 ? 0 : alternation.Children.Min(MinSegments); + default: + return node.Children.Sum(MinSegments); + } + } + + private class SlotSummary + { + public bool IsPure = true; + public bool MayAlwaysUnapply; + public int[][] Allomorphs; + public FeatureStruct[] Distinct; + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs index ed5cabc36..ee799323a 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs @@ -69,6 +69,409 @@ public void Apply_SlotLeftEmpty_LeavesTheSameStateAsNotEnteringTheTemplate() }); } + // Slots are listed in synthesis order and unapplied from the last to the first, so slot 0 is the suffix + // closest to the stem and a word ends with the material of its highest slot. + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "lookahead_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + // The analysis of a rule that deletes a final segment in synthesis: it adds the segment back, so the + // unapplication produces a shape that is not made of copies of the input. + private AffixProcessRule SegmentRestoringRule(string name, string restored) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Character(Table3, restored)).Value, + }, + Rhs = { new CopyFromInput("1") }, + } + ); + return rule; + } + + private AffixProcessRule SuffixRule(string name, string insert) => + Suffix(name, FeatureStruct.New().Value, FeatureStruct.New().Value, Table3, insert); + + private Word Input(string form, bool lastNodeOptional = false) + { + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + if (lastNodeOptional) + input.Shape.Last.Annotation.Optional = true; + return input; + } + + private sealed record WalkResult(List Parses, int RuleAttempts); + + private WalkResult Walk( + AffixTemplate template, + Word input, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1, + bool tracing = false + ) + { + int attempts = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + }; + TraceManager.IsTracing = tracing; + if (tracing) + TraceManager.AnalyzeWord(Language, input); + input.Freeze(); + IRule rule = template.CompileAnalysisRule(morpher); + List parses = rule.Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form, bool lastNodeOptional = false) + { + WalkResult off = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses"); + return (off, on); + } + + // stem + required -ti + optional -u/-f/-v + optional -z + private AffixTemplate RequiredInnerSuffixTemplate() => + TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("u", "u"), SuffixRule("f", "f"), SuffixRule("v", "v")), + Slot(true, SuffixRule("z", "z")) + ); + + [Test] + public void Apply_RequiredSlotLiteralAbsent_PrunesBeforeTryingTheOptionalSlots() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagufz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0), "no rule can supply -ti, so no slot rule is attempted"); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralPresent_DoesNotPrune() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtiuz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralLostAfterAnOptionalSlot_PrunesTheInnerPartial() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + // -z unapplies, which leaves "sagtuu"; no node can still be the i of -ti + (WalkResult off, WalkResult on) = WalkBoth(template, "sagttz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(on.RuleAttempts, Is.LessThan(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_ImpureSlotBetween_DoesNotPruneForTheInnerRequiredSlot() + { + // the middle slot restores the i that -ti needs, so "sagt" is a stem plus -ti once it has run + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SegmentRestoringRule("restore_i", "i")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagt"); + + Assert.That(on.Parses, Is.Not.Empty, "the parse that needs the restored segment must survive"); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + } + + [Test] + public void Apply_SeveralRequiredSlots_PrunesWhicheverLiteralIsAbsent() + { + // -u is required as well as -ti; the word has neither + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("z", "z")), + Slot(false, SuffixRule("u", "u")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagfz"); + + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + } + + [Test] + public void Apply_OptionalNodeThatUnifiesWithTheLiteral_CountsAsPresent() + { + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti"))); + + // "sagti" where the i stands for a deleted segment: the matcher may match or skip an optional node + (WalkResult off, WalkResult on) = WalkBoth(template, "sagti", lastNodeOptional: true); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.Parses, Is.EqualTo(off.Parses)); + }); + } + + [Test] + public void Apply_RequiredSlotWithNullAffix_IsNeverPruned() + { + AffixProcessRule withNull = SuffixRule("ti_or_null", "ti"); + withNull.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + AffixTemplate template = TemplateOf(Slot(false, withNull), Slot(true, SuffixRule("z", "z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + // An optional slot whose rule inserts "z" after the copied parts; the word never contains the t and i + // that the required inner slot inserts, so a pure rule lets the walk prune at entry, before the optional rule is tried, and an impure one does not. + private (WalkResult Off, WalkResult On) WalkWithOptionalSlot( + Pattern[] lhs, + MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = "z_slot", + Name = "z_slot", + Gloss = "z_slot", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + allomorph.Rhs.Add(new InsertSegments(Table3, "z")); + rule.Allomorphs.Add(allomorph); + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti")), Slot(true, rule)); + return WalkBoth(template, "sagz"); + } + + [Test] + public void Apply_CopiedPartThatMayMatchNothing_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).ZeroOrMore.Value }, + new MorphologicalOutputAction[] { new CopyFromInput("1") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_ModifyFromInputInASlot_MakesTheSlotImpure() + { + FeatureStruct voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + new MorphologicalOutputAction[] + { + new ModifyFromInput("1", voiced, System.Array.Empty()), + } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_CopiedAlternationOfSegments_KeepsTheSlotPure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [Test] + public void Apply_CopiedAlternationWithAnOptionalBranch_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Optional.Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_SlotOptionalityChangedAfterCompiling_IsReadLive() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var morpher = new Morpher(TraceManager, Language, 1) { RequiredSlotLookahead = RequiredSlotLookaheadMode.On }; + IRule rule = template.CompileAnalysisRule(morpher); + Word input = Input("sagu"); + input.Freeze(); + + Assert.That(rule.Apply(input), Is.Empty, "slot 0 is required and -ti is absent"); + + template.Slots[0].Optional = true; + Assert.That(rule.Apply(input), Is.Not.Empty, "an optional slot is not tested"); + + template.Slots[0].Optional = false; + Assert.That(rule.Apply(input), Is.Empty); + } + + [TestCase(0)] + [TestCase(4)] + public void Apply_Parallel_ReturnsTheSameParsesAsSequential(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif" }) + { + WalkResult sequential = Walk(template, Input(form), RequiredSlotLookaheadMode.On); + WalkResult parallel = Walk(template, Input(form), RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult parallelOff = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Assert.That(parallel.Parses, Is.EqualTo(sequential.Parses), form); + Assert.That(parallel.Parses, Is.EqualTo(parallelOff.Parses), form); + } + } + + [Test] + public void Apply_Tracing_ShowsAPrunedTemplateAsAFailedOutput() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + Word input = Input("sagufz"); + + Walk(template, input, RequiredSlotLookaheadMode.On, tracing: true); + TraceManager.IsTracing = false; + + var root = (Trace)input.CurrentTrace; + Trace[] children = root.Children.ToArray(); + Assert.Multiple(() => + { + Assert.That( + children.Select(t => t.Type), + Is.EqualTo(new[] { TraceType.TemplateAnalysisInput, TraceType.TemplateAnalysisOutput }) + ); + Assert.That(children[1].Output, Is.Null, "the partial is reported as not unapplied"); + }); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_Shadow_ReportsFiredPointsAndNeverViolatesOrChangesTheParses(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var events = new List(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif", "sagu" }) + { + WalkResult off = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Word input = Input(form); + input.Freeze(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = RequiredSlotLookaheadMode.Shadow, + RequiredSlotShadow = (kind, t, slot) => + { + lock (events) + events.Add(kind); + }, + }; + TraceManager.IsTracing = false; + string[] parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => + w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss)) + ) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToArray(); + Assert.That(parses, Is.EqualTo(off.Parses), form); + } + + Assert.Multiple(() => + { + Assert.That(events.Count(e => e == "fired"), Is.GreaterThan(0)); + Assert.That(events.Count(e => e == "violated"), Is.EqualTo(0)); + Assert.That(events.Count(e => e == "completed-under-fired"), Is.EqualTo(0)); + }); + } + private AffixTemplate VerbTemplate(FeatureStruct required) { var template = new AffixTemplate { Name = "verb_template", RequiredSyntacticFeatureStruct = required }; From 28cbee19992d95ee99a4f1d2c407c93d9c4a2de4 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 27 Oct 2026 21:53:26 -0400 Subject: [PATCH 2/4] Check required slot material in order and at the word edges before walking a template The required-slot look-ahead now needs each rule's inserted segments to match distinct segments in the order the rule inserts them, a slot that inserts only before or after its copied parts to find that material within the reach of the outer slots' own edge material, and the material of a required ordered slot to lie after the prefix and before the suffix of the required slots that unapply first. Slots that can reorder, repeat or remove from the middle switch the order and edge tests off for the slots behind them. Closes #567 Co-Authored-By: Claude Opus 5.5 --- .../AnalysisAffixTemplateRule.cs | 4 +- .../RequiredSlotMaterial.cs | 417 ++++++++++++++---- .../AnalysisAffixTemplateOrderTests.cs | 362 +++++++++++++++ 3 files changed, 691 insertions(+), 92 deletions(-) create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index d46b5971b..1f95113d0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -93,12 +93,13 @@ private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired) long unifications = 0; bool cannot = false; + SlotWalk walk = SlotWalk.Start(); for (int j = lastSlot; j >= 0; j--) { if ( !_template.Slots[j].Optional && !IsFired(fired, j) - && _lookahead.CannotUnapply(j, word.Shape, ref unifications) + && _lookahead.CannotUnapply(j, word.Shape, ref walk, ref unifications) ) { if (mode == RequiredSlotLookaheadMode.On) @@ -109,6 +110,7 @@ private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired) fired = fired == null ? new[] { j } : fired.Append(j).ToArray(); _morpher.RequiredSlotShadow?.Invoke("fired", _template, j); } + _lookahead.Advance(j, ref walk); if (!_lookahead.IsPure(j)) break; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs index 27e60caa6..f7e9cfdc3 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using SIL.Machine.Annotations; @@ -22,26 +23,73 @@ internal enum RequiredSlotLookaheadMode On, } + /// + /// What a walk over the slots of one template has established about the shape, in the ordinals of the shape + /// the walk started from. Create it with . + /// + internal struct SlotWalk + { + /// Stands for "any number" in and . + public const int Unbounded = int.MaxValue / 4; + + /// Every node the slots still to unapply can match lies after this ordinal. + public int Lo; + + /// Every node the slots still to unapply can match lies before this ordinal. + public int Hi; + + /// The most nodes that the slots walked so far can have removed from the left edge. + public int ReachLeft; + + /// The most nodes that the slots walked so far can have removed from the right edge. + public int ReachRight; + + /// False once a slot walked so far can reorder the nodes it copies. + public bool Ordered; + + public static SlotWalk Start() + { + return new SlotWalk + { + Lo = -1, + Hi = int.MaxValue, + Ordered = true, + }; + } + } + /// /// A necessary condition for a required affix template slot to unapply later in a template walk: some - /// allomorph of some rule in the slot must have every inserted segment unify with a segment of the shape. + /// allomorph of some rule in the slot must have its inserted segments, in the order it inserts them, unify + /// with distinct segment nodes of the shape in that order, and the nodes must be where the slots that unapply + /// first can leave them. /// /// /// The tables are built once from the template and depend only on the grammar. /// - /// Soundness. The analysis pattern of an allomorph holds one top-level constraint per inserted segment - /// (, ), - /// and the rule can only unapply if each constraint unifies with a distinct segment node of the shape. - /// A rule is pure when every right-hand side action copies an input part or inserts material, and every - /// left-hand part is copied by a that must capture a segment, or has no - /// segment to restore. Then only copies input - /// nodes and never untruncates a segment. Unification can only narrow the feature structure of a copied - /// node, so a constraint that unifies with no node of the shape unifies with no node of the shape after - /// any number of pure unapplications. Inside one template walk only the rules of that template's slots - /// run, and the stratum's phonological rules ran before the walk. The test for slot j is therefore used - /// only when every rule in the slots still to be walked after j is pure. Optional nodes count as present - /// because the matcher can match them. Anything that cannot be summarised (a null affix, a constraint with - /// variables, an unknown rule type) leaves the slot unprunable. + /// Presence. The analysis pattern of an allomorph holds one top-level constraint per inserted segment, in the + /// order of the right-hand side (, + /// ), and the rule can only unapply if the constraints + /// unify with segment nodes of the shape in that order, each node used once. A rule is pure when every + /// right-hand side action copies an input part or inserts material, and every left-hand part is copied by a + /// that must capture a segment, or has no segment to restore. Then + /// only copies input nodes and never untruncates a + /// segment. Unification can only narrow the feature structure of a copied node, so a constraint that unifies + /// with no node of the shape unifies with no node of the shape after any number of pure unapplications. Inside + /// one template walk only the rules of that template's slots run, and the stratum's phonological rules ran + /// before the walk. The test for slot j is therefore used only when every rule in the slots still to be walked + /// after j is pure. Optional nodes count as present because the matcher can match them. Anything that cannot + /// be summarised (a null affix, a constraint with variables that is then matched by any segment, an unknown + /// rule type) can only weaken the test. + /// + /// Order. A pure rule is ordered when it copies each part at most once and in left-hand order, so the shape + /// it unapplies to is a subsequence of the shape it was given. An ordered rule whose inserts all come before + /// its first copied part (after its last) is anchored on that side: the analysis matcher is anchored at both + /// ends of the shape, so the first inserted constraint must match the first segment left once the outer slots + /// have removed their own prefixes. A rule can remove from an edge at most as many nodes as it inserts there + /// (), and the nodes of a required ordered slot lie after the prefix and before + /// the suffix it inserts (, ). Both only hold while every + /// slot walked so far is ordered, and the reach only while none can remove from the middle. /// internal class RequiredSlotMaterial { @@ -61,70 +109,184 @@ public bool IsPure(int slot) return _slots[slot].IsPure; } + /// + /// Records that the slot is the next one to unapply, so that the slots before it are tested against what + /// it can leave of the shape. + /// + public void Advance(int slot, ref SlotWalk walk) + { + SlotSummary summary = _slots[slot]; + if (!walk.Ordered || !summary.IsOrdered) + { + walk.Ordered = false; + walk.Lo = -1; + walk.Hi = int.MaxValue; + walk.ReachLeft = SlotWalk.Unbounded; + walk.ReachRight = SlotWalk.Unbounded; + return; + } + walk.ReachLeft = Math.Min(SlotWalk.Unbounded, walk.ReachLeft + summary.ReachLeft); + walk.ReachRight = Math.Min(SlotWalk.Unbounded, walk.ReachRight + summary.ReachRight); + } + /// /// True if no allomorph of the slot can unapply to a word with this shape, provided that every slot - /// walked between this word and the slot is pure. + /// walked between this word and the slot is pure. When the slot can unapply and is ordered, narrows the + /// window of the walk for the slots before it. /// /// The slot to test. /// The shape of the partial. + /// What the slots walked since the shape was taken have established. /// Incremented by the feature structure unifications performed. - public bool CannotUnapply(int slot, Shape shape, ref long unifications) + public bool CannotUnapply(int slot, Shape shape, ref SlotWalk walk, ref long unifications) { SlotSummary summary = _slots[slot]; if (summary.MayAlwaysUnapply) return false; - // The constraints of a slot repeat across its allomorphs, so each distinct one is looked up once. - ulong known = 0; - ulong present = 0; - foreach (int[] requirements in summary.Allomorphs) + // The window is only narrowed for a slot that has slots after it in the walk, and that needs every + // passing allomorph, not the first. + bool narrow = walk.Ordered && summary.IsOrdered && slot > 0; + int newLo = int.MaxValue; + int newHi = -1; + bool any = false; + foreach (AllomorphSummary allomorph in summary.Allomorphs) { - bool all = true; - foreach (int index in requirements) + int prefixEnd = -1; + int suffixStart = int.MaxValue; + bool found = walk.Ordered + ? Embeds(allomorph, shape, ref walk, ref unifications, out prefixEnd, out suffixStart) + : AllPresent(allomorph, shape, ref unifications); + if (!found) + continue; + + if (!narrow) + return false; + any = true; + newLo = Math.Min(newLo, allomorph.Pre > 0 ? prefixEnd : walk.Lo); + newHi = Math.Max(newHi, allomorph.Post > 0 ? suffixStart : walk.Hi); + } + if (!any) + return true; + + walk.Lo = newLo; + walk.Hi = newHi; + return false; + } + + // The ordinals are those of the shape, margins excluded. A node is counted against the reach only when it + // cannot be skipped, because the matcher may skip an optional node. + private static bool Embeds( + AllomorphSummary allomorph, + Shape shape, + ref SlotWalk walk, + ref long unifications, + out int prefixEnd, + out int suffixStart + ) + { + prefixEnd = -1; + suffixStart = int.MaxValue; + FeatureStruct[] sequence = allomorph.Sequence; + int front = sequence.Length - allomorph.Post; + int forwardEnd = -1; + + int next = 0; + int before = 0; + int ordinal = -1; + foreach (ShapeNode node in shape) + { + if (next == front) + break; + ordinal++; + if (ordinal >= walk.Hi) + break; + + bool segment = node.Type() == HCFeatureSystem.Segment; + if (segment && ordinal > walk.Lo) { - bool found; - ulong bit = index < 64 ? 1UL << index : 0; - if ((known & bit) != 0) + if (next < allomorph.Pre && before > Math.Min(SlotWalk.Unbounded, walk.ReachLeft + next)) + return false; + + FeatureStruct required = sequence[next]; + if (required == null || IsUnifiable(node, required, ref unifications)) { - found = (present & bit) != 0; + if (next == allomorph.Pre - 1) + prefixEnd = ordinal; + forwardEnd = ordinal; + next++; } - else + } + if (segment && !node.Annotation.Optional) + before++; + } + if (next < front) + return false; + if (allomorph.Post == 0) + return true; + + int taken = 0; + int after = 0; + ordinal = shape.Count; + for (ShapeNode node = shape.Last; node != null && taken < allomorph.Post; node = node.Prev) + { + ordinal--; + if (ordinal <= walk.Lo || ordinal <= forwardEnd) + break; + + bool segment = node.Type() == HCFeatureSystem.Segment; + if (segment && ordinal < walk.Hi) + { + if (after > Math.Min(SlotWalk.Unbounded, walk.ReachRight + taken)) + return false; + + FeatureStruct required = sequence[sequence.Length - 1 - taken]; + if (required == null || IsUnifiable(node, required, ref unifications)) { - found = IsPresent(summary.Distinct[index], shape, ref unifications); - known |= bit; - if (found) - present |= bit; + suffixStart = ordinal; + taken++; } - if (!found) + } + if (segment && !node.Annotation.Optional) + after++; + } + return taken == allomorph.Post; + } + + // Once a slot that can reorder nodes has been walked the order of the shape says nothing about the order of + // the shape the slot unapplies to, so each constraint is only looked for on its own. + private static bool AllPresent(AllomorphSummary allomorph, Shape shape, ref long unifications) + { + foreach (FeatureStruct required in allomorph.Sequence) + { + if (required == null) + continue; + + bool found = false; + foreach (ShapeNode node in shape) + { + if (node.Type() == HCFeatureSystem.Segment && IsUnifiable(node, required, ref unifications)) { - all = false; + found = true; break; } } - if (all) + if (!found) return false; } return true; } - private static bool IsPresent(FeatureStruct required, Shape shape, ref long unifications) + private static bool IsUnifiable(ShapeNode node, FeatureStruct required, ref long unifications) { - foreach (ShapeNode node in shape) - { - if (node.Type() != HCFeatureSystem.Segment) - continue; - - unifications++; - if (node.Annotation.FeatureStruct.IsUnifiable(required, false, null)) - return true; - } - return false; + unifications++; + return node.Annotation.FeatureStruct.IsUnifiable(required, false, null); } private static SlotSummary Summarize(AffixTemplateSlot slot) { var summary = new SlotSummary(); - var allomorphs = new List(); + var allomorphs = new List(); foreach (MorphemicMorphologicalRule rule in slot.Rules) { IList ruleAllomorphs; @@ -139,73 +301,117 @@ private static SlotSummary Summarize(AffixTemplateSlot slot) default: summary.MayAlwaysUnapply = true; summary.IsPure = false; + summary.IsOrdered = false; + summary.ReachLeft = SlotWalk.Unbounded; + summary.ReachRight = SlotWalk.Unbounded; continue; } foreach (AffixProcessAllomorph allomorph in ruleAllomorphs) { - if (!IsPure(allomorph)) + bool pure = IsPure(allomorph); + if (!pure) summary.IsPure = false; + if (!pure || !IsOrdered(allomorph)) + summary.IsOrdered = false; - FeatureStruct[] requirements = GetRequirements(allomorph); - if (requirements.Length == 0) + AllomorphSummary allomorphSummary = Summarize(allomorph); + summary.ReachLeft = Math.Max(summary.ReachLeft, allomorphSummary.ReachLeft); + summary.ReachRight = Math.Max(summary.ReachRight, allomorphSummary.ReachRight); + if (allomorphSummary.Sequence.Length == 0) summary.MayAlwaysUnapply = true; else - allomorphs.Add(requirements); + allomorphs.Add(allomorphSummary); } } - var distinct = new List(); - summary.Allomorphs = allomorphs - .Select(requirements => - requirements - .Select(fs => - { - int index = distinct.FindIndex(d => d.ValueEquals(fs)); - if (index < 0) - { - distinct.Add(fs); - index = distinct.Count - 1; - } - return index; - }) - .ToArray() - ) - .ToArray(); - summary.Distinct = distinct.ToArray(); + summary.Allomorphs = allomorphs.ToArray(); return summary; } - private static FeatureStruct[] GetRequirements(AffixProcessAllomorph allomorph) + // Pre is the inserts before the first non-insertion, Post those after the last; any between is an infix. + // A non-segment constraint matches nothing the matcher sees, so skipping it only weakens the test. + private static AllomorphSummary Summarize(AffixProcessAllomorph allomorph) { - var requirements = new List(); - foreach (MorphologicalOutputAction action in allomorph.Rhs) + var sequence = new List(); + IList rhs = allomorph.Rhs; + int first = -1; + int last = -1; + for (int i = 0; i < rhs.Count; i++) { - if (action is InsertSegments insertSegments) + if (IsInsertion(rhs[i])) + continue; + if (first < 0) + first = i; + last = i; + } + + int pre = 0; + int post = 0; + bool infix = false; + for (int i = 0; i < rhs.Count; i++) + { + if (!IsInsertion(rhs[i])) + continue; + + int before = sequence.Count; + foreach (FeatureStruct fs in InsertedConstraints(rhs[i])) { - foreach (ShapeNode node in insertSegments.Segments.Shape) + if ((FeatureSymbol)fs.GetValue(HCFeatureSystem.Type) != HCFeatureSystem.Segment) + continue; + + // A constraint with variables is bound by earlier matches, so any segment stands for it. + FeatureStruct copy = null; + if (!fs.HasVariables) { - if (node.Type() != HCFeatureSystem.Boundary) - AddRequirement(requirements, node.Annotation.FeatureStruct); + copy = fs.Clone(); + copy.Freeze(); } + sequence.Add(copy); } - else if (action is InsertSimpleContext insertContext) - { - AddRequirement(requirements, insertContext.SimpleContext.FeatureStruct); - } + int added = sequence.Count - before; + if (first < 0 || (i > first && i < last)) + infix = true; + else if (i < first) + pre += added; + else + post += added; + } + + var summary = new AllomorphSummary { Sequence = sequence.ToArray() }; + if (infix) + { + summary.ReachLeft = SlotWalk.Unbounded; + summary.ReachRight = SlotWalk.Unbounded; } - return requirements.ToArray(); + else + { + summary.Pre = pre; + summary.Post = post; + summary.ReachLeft = pre; + summary.ReachRight = post; + } + return summary; } - // A constraint with variables is bound by earlier matches, and one that is not a segment is never - // matched against the segments the matcher considers, so neither can be tested here. - private static void AddRequirement(List requirements, FeatureStruct fs) + private static bool IsInsertion(MorphologicalOutputAction action) { - if (fs.HasVariables || (FeatureSymbol)fs.GetValue(HCFeatureSystem.Type) != HCFeatureSystem.Segment) - return; + return action is InsertSegments || action is InsertSimpleContext; + } - FeatureStruct copy = fs.Clone(); - copy.Freeze(); - requirements.Add(copy); + private static IEnumerable InsertedConstraints(MorphologicalOutputAction action) + { + if (action is InsertSegments insertSegments) + { + foreach (ShapeNode node in insertSegments.Segments.Shape) + { + if (node.Type() != HCFeatureSystem.Boundary) + yield return node.Annotation.FeatureStruct; + } + } + else if (action is InsertSimpleContext insertContext) + { + yield return insertContext.SimpleContext.FeatureStruct; + } } private static bool IsPure(AffixProcessAllomorph allomorph) @@ -215,7 +421,7 @@ private static bool IsPure(AffixProcessAllomorph allomorph) { if (action is CopyFromInput) copied.Add(action.PartName); - else if (!(action is InsertSegments || action is InsertSimpleContext)) + else if (!IsInsertion(action)) return false; } @@ -229,6 +435,24 @@ private static bool IsPure(AffixProcessAllomorph allomorph) return true; } + // GenerateShape writes parts in left-hand order, so copying them in another order, or one twice, can + // reorder or repeat nodes. + private static bool IsOrdered(AffixProcessAllomorph allomorph) + { + int previous = -1; + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (!(action is CopyFromInput)) + continue; + + int index = allomorph.Lhs.ToList().FindIndex(part => part.Name == action.PartName); + if (index <= previous) + return false; + previous = index; + } + return true; + } + private static bool HasSegmentConstraint(PatternNode node) { if (node is Constraint constraint) @@ -253,12 +477,23 @@ private static int MinSegments(PatternNode node) } } + private class AllomorphSummary + { + public FeatureStruct[] Sequence; + public int Pre; + public int Post; + public int ReachLeft; + public int ReachRight; + } + private class SlotSummary { public bool IsPure = true; + public bool IsOrdered = true; public bool MayAlwaysUnapply; - public int[][] Allomorphs; - public FeatureStruct[] Distinct; + public int ReachLeft; + public int ReachRight; + public AllomorphSummary[] Allomorphs; } } } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs new file mode 100644 index 000000000..93b223e20 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs @@ -0,0 +1,362 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +// Order and edge tests of the required-slot look-ahead; slots are in synthesis order and unapplied last to first. +[TestFixture] +public class AnalysisAffixTemplateOrderTests : HermitCrabTestBase +{ + private static readonly FeatureStruct Any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + private sealed record WalkResult(List Parses, int RuleAttempts, int Violations); + + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "order_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + private static Pattern Part(string name) => + Pattern.New(name).Annotation(Any).OneOrMore.Value; + + private static Pattern OneSegment(string name) => + Pattern.New(name).Annotation(Any).Value; + + private static AffixProcessRule RuleOf( + string name, + Pattern[] lhs, + params MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + rule.Allomorphs.Add(allomorph); + return rule; + } + + private AffixProcessRule Suffix(string insert) => + RuleOf("suffix_" + insert, new[] { Part("1") }, new CopyFromInput("1"), new InsertSegments(Table3, insert)); + + private AffixProcessRule Prefix(string insert) => + RuleOf("prefix_" + insert, new[] { Part("1") }, new InsertSegments(Table3, insert), new CopyFromInput("1")); + + private AffixProcessRule Circumfix(string before, string after) => + RuleOf( + "circumfix_" + before + after, + new[] { Part("1") }, + new InsertSegments(Table3, before), + new CopyFromInput("1"), + new InsertSegments(Table3, after) + ); + + // inserts the material after the first segment of the stem + private AffixProcessRule Infix(string insert) => + RuleOf( + "infix_" + insert, + new[] { OneSegment("1"), Part("2") }, + new CopyFromInput("1"), + new InsertSegments(Table3, insert), + new CopyFromInput("2") + ); + + // writes the last segment of the stem twice + private static AffixProcessRule RepeatLast() => + RuleOf( + "repeat_last", + new[] { Part("1"), OneSegment("2") }, + new CopyFromInput("1"), + new CopyFromInput("2"), + new CopyFromInput("2") + ); + + // swaps the last two segments of the stem, so it copies its parts out of order + private static AffixProcessRule Metathesis() => + RuleOf( + "metathesis", + new[] { Part("1"), OneSegment("2"), OneSegment("3") }, + new CopyFromInput("1"), + new CopyFromInput("3"), + new CopyFromInput("2") + ); + + private WalkResult Walk( + AffixTemplate template, + string form, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1 + ) + { + int attempts = 0; + int violations = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + RequiredSlotShadow = (kind, t, slot) => + { + if (kind != "fired") + System.Threading.Interlocked.Increment(ref violations); + }, + }; + TraceManager.IsTracing = false; + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + input.Freeze(); + List parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts, violations); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses of " + form); + return (off, on); + } + + [Test] + public void Apply_RequiredSuffixesInTheWrongOrder_PrunesAtEntry() + { + // both literals are present in "saguti", but the outer suffix -u has to end the word + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(false, Suffix("u"))); + + (WalkResult off, WalkResult wrong) = WalkBoth(template, "saguti"); + (WalkResult offRight, WalkResult right) = WalkBoth(template, "sagtiu"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(wrong.RuleAttempts, Is.EqualTo(0)); + Assert.That(right.Parses, Is.Not.Empty); + Assert.That(right.RuleAttempts, Is.EqualTo(offRight.RuleAttempts)); + }); + } + + [Test] + public void Apply_PrefixSlotWhoseMaterialIsNotAtTheLeftEdge_PrunesAtEntry() + { + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(false, Prefix("ka"))); + + (WalkResult off, WalkResult wrong) = WalkBoth(template, "sagkati"); + (WalkResult offRight, WalkResult right) = WalkBoth(template, "kasagti"); + + Assert.Multiple(() => + { + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(wrong.RuleAttempts, Is.EqualTo(0)); + Assert.That(right.Parses, Is.Not.Empty); + Assert.That(right.RuleAttempts, Is.EqualTo(offRight.RuleAttempts)); + }); + } + + [Test] + public void Apply_CircumfixWhoseSuffixPartIsNotAtTheRightEdge_PrunesAtEntry() + { + AffixTemplate template = TemplateOf(Slot(false, Circumfix("ka", "ti"))); + + (WalkResult off, WalkResult wrong) = WalkBoth(template, "kasagtz"); + (WalkResult offRight, WalkResult right) = WalkBoth(template, "kasagti"); + + Assert.Multiple(() => + { + Assert.That(wrong.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(right.Parses, Is.Not.Empty); + Assert.That(right.RuleAttempts, Is.EqualTo(offRight.RuleAttempts)); + }); + } + + [Test] + public void Apply_CircumfixPartsThatWouldShareANode_PrunesAtEntry() + { + // ka- and -ak could both use the a of "kak" + AffixTemplate template = TemplateOf(Slot(false, Circumfix("ka", "ak"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "kak"); + + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + } + + // an infix does not anchor, so only the window left by the outer required prefix can rule out its material + private AffixTemplate InfixInsidePrefixTemplate() => + TemplateOf(Slot(false, Infix("ak")), Slot(false, Prefix("ka"))); + + [Test] + public void Apply_InnerMaterialOnlyInsideTheOuterPrefix_PrunesAtEntry() + { + // the a of "ka" is the only a before the k that -ak needs, so the infix has no material of its own + (WalkResult off, WalkResult on) = WalkBoth(InfixInsidePrefixTemplate(), "kakg"); + + Assert.Multiple(() => + { + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + }); + } + + [Test] + public void Apply_InnerMaterialAfterTheOuterPrefix_IsNotPruned() + { + // "sag" with the infix ak is "sakag", and ka- in front gives "kasakag" + (WalkResult off, WalkResult on) = WalkBoth(InfixInsidePrefixTemplate(), "kasakag"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_OptionalOuterPrefix_CountsTowardsTheLeftEdgeBudget() + { + AffixTemplate template = TemplateOf(Slot(false, Prefix("ka")), Slot(true, Prefix("ba"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "bakasag"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_OuterSlotWithNullAffix_IsNeverPrunedAndTakesNoEdgeMaterial() + { + AffixProcessRule uOrNull = Suffix("u"); + uOrNull.Allomorphs.Add(new AffixProcessAllomorph { Lhs = { Part("1") }, Rhs = { new CopyFromInput("1") } }); + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(false, uOrNull)); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagti"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_OptionalOuterSlot_CountsTowardsTheEdgeBudget() + { + // the optional -z may stand between -ti and the end of the word + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, Suffix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtiz"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_InfixSlotBetween_DoesNotAnchorTheInnerSlot() + { + // unapplying the infix z of "kzasag" gives "kasag", which starts with ka- + AffixTemplate template = TemplateOf(Slot(false, Prefix("ka")), Slot(true, Infix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "kzasag"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_SlotThatCopiesOutOfOrderBetween_ChecksOnlyPresence() + { + // unapplying the metathesis of "sagit" gives "sagti", whose -ti is in the opposite order + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, Metathesis())); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagit"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_PartCopiedTwiceBetween_DoesNotAnchorTheInnerSlot() + { + // unapplying the repeated i of "sagtii" gives "sagti", which ends in -ti + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, RepeatLast())); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtii"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_ConstraintWithVariables_IsMatchedByAnySegment() + { + var context = new SimpleContext(new NaturalClass(Any), new[] { Variable("vd", "a") }); + AffixProcessRule suffix = RuleOf( + "variable_suffix", + new[] { Part("1") }, + new CopyFromInput("1"), + new InsertSimpleContext(context) + ); + AffixTemplate template = TemplateOf(Slot(false, suffix)); + + foreach (string form in new[] { "sagd", "sagt", "sag" }) + WalkBoth(template, form); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_OnEveryTemplateAndWord_KeepsTheParsesAndNeverViolatesTheShadow(int maxDegreeOfParallelism) + { + var templates = new Dictionary + { + [TemplateOf(Slot(false, Suffix("ti")), Slot(false, Suffix("u")))] = new[] { "saguti", "sagtiu", "sagti" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(false, Prefix("ka")))] = new[] { "kasagti", "sagkati" }, + [TemplateOf(Slot(false, Circumfix("ka", "ti")), Slot(true, Suffix("z")), Slot(false, Prefix("ba")))] = new[] + { + "bakasagtiz", + "kabasagti", + "bakasagtzi", + }, + [TemplateOf(Slot(false, Prefix("ka")), Slot(true, Infix("z")))] = new[] { "kzasag", "kasag" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, RepeatLast()))] = new[] { "sagtii", "sagti" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, Metathesis()))] = new[] { "sagit", "sagti" }, + }; + + foreach ((AffixTemplate template, string[] forms) in templates) + { + foreach (string form in forms) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult shadow = Walk(template, form, RequiredSlotLookaheadMode.Shadow, maxDegreeOfParallelism); + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Violations, Is.EqualTo(0), form); + }); + } + } + } +} From faa24998044cfbb97cba79a67eb862f95c4212bb Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 28 Oct 2026 10:37:49 -0400 Subject: [PATCH 3/4] The required-slot look-ahead keeps testing behind a slot whose unmatched parts are unbounded runs, which restore no segments. Co-Authored-By: Claude Opus 5.5 --- .../RequiredSlotMaterial.cs | 22 +- .../AnalysisAffixTemplateFusionTests.cs | 238 ++++++++++++++++++ .../AnalysisAffixTemplateRuleTests.cs | 4 +- 3 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs index f7e9cfdc3..2d0e24ec8 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs @@ -72,7 +72,8 @@ public static SlotWalk Start() /// ), and the rule can only unapply if the constraints /// unify with segment nodes of the shape in that order, each node used once. A rule is pure when every /// right-hand side action copies an input part or inserts material, and every left-hand part is copied by a - /// that must capture a segment, or has no segment to restore. Then + /// that must capture a segment, or restores no segment when it captures nothing + /// ( writes nothing back for an unbounded quantifier). Then /// only copies input nodes and never untruncates a /// segment. Unification can only narrow the feature structure of a copied node, so a constraint that unifies /// with no node of the shape unifies with no node of the shape after any number of pure unapplications. Inside @@ -429,6 +430,8 @@ private static bool IsPure(AffixProcessAllomorph allomorph) { if (!HasSegmentConstraint(part)) continue; + if (RestoredSegments(part) == 0) + continue; if (!copied.Contains(part.Name) || MinSegments(part) == 0) return false; } @@ -460,6 +463,23 @@ private static bool HasSegmentConstraint(PatternNode node) return node.Children.Any(HasSegmentConstraint); } + // The segment nodes that AnalysisMorphologicalTransform.Untruncate writes for a part that was not + // captured. An unbounded quantifier writes none, as in Mbugwe's vowel-fusion slots. + private static int RestoredSegments(PatternNode node) + { + int total = 0; + foreach (PatternNode child in node.Children) + { + if (child is Constraint constraint && constraint.Type() == HCFeatureSystem.Segment) + total++; + else if (child is Quantifier quantifier) + total += Math.Max(0, quantifier.MaxOccur) * RestoredSegments(quantifier); + else + total += RestoredSegments(child); + } + return total; + } + // The fewest segments a successful match of the node must consume. A part matching none is captured // as a null range, and GenerateShape then untruncates it, which adds segments that are not copies. private static int MinSegments(PatternNode node) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs new file mode 100644 index 000000000..4c1c7c2a2 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs @@ -0,0 +1,238 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +// A vowel-fusion slot copies a part that may match nothing; the look-ahead may test behind it only if it +// restores no segment. Slots are in synthesis order and unapplied last to first. +[TestFixture] +public class AnalysisAffixTemplateFusionTests : HermitCrabTestBase +{ + private static readonly FeatureStruct Any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + private sealed record WalkResult(List Parses, int RuleAttempts, int Violations); + + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "fusion_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + private static AffixProcessRule RuleOf( + string name, + Pattern[] lhs, + params MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + rule.Allomorphs.Add(allomorph); + return rule; + } + + private AffixProcessRule Suffix(string insert) => + RuleOf( + "suffix_" + insert, + new[] { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + new CopyFromInput("1"), + new InsertSegments(Table3, insert) + ); + + // Like Mbugwe's object markers: a prefix whose stem vowel is copied twice, followed by an unbounded run + // that may be empty. + private AffixProcessRule FusionPrefix(string name, string before, string between) => + RuleOf( + name, + new[] + { + Pattern.New("1").Annotation(Any).Value, + Pattern.New("2").Annotation(Any).ZeroOrMore.Value, + }, + new InsertSegments(Table3, before), + new CopyFromInput("1"), + new InsertSegments(Table3, between), + new CopyFromInput("1"), + new CopyFromInput("2") + ); + + // The run is capped at one segment, so an unmatched run is written back as an optional segment. + private AffixProcessRule BoundedRunSuffix(string insert) => + RuleOf( + "bounded_" + insert, + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Any).Optional.Value, + }, + new CopyFromInput("1"), + new InsertSegments(Table3, insert) + ); + + private WalkResult Walk( + AffixTemplate template, + string form, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1 + ) + { + int attempts = 0; + int violations = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + RequiredSlotShadow = (kind, t, slot) => + { + if (kind != "fired") + System.Threading.Interlocked.Increment(ref violations); + }, + }; + TraceManager.IsTracing = false; + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + input.Freeze(); + List parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts, violations); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses of " + form); + return (off, on); + } + + [Test] + public void Apply_FusionSlotWithAnUnboundedRun_StillTestsTheRequiredSlotBehindIt() + { + // -ti is required inside the fusion prefix and "sagu" has no t + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, FusionPrefix("fuse", "k", "a"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagu"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0), "the fusion slot restores nothing, so -ti cannot appear"); + }); + } + + [Test] + public void Apply_FusionSlotWithAnUnboundedRun_KeepsAWordThatHasTheMaterial() + { + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, FusionPrefix("fuse", "k", "a"))); + + foreach (string form in new[] { "kasagti", "ksagti", "sagti", "kasagtiz" }) + { + (WalkResult off, WalkResult on) = WalkBoth(template, form); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts), form); + } + } + + [Test] + public void Apply_SlotWithABoundedRunThatMayBeEmpty_RestoresASegmentTheInnerSlotCanUse() + { + // the run is written back as an optional segment, which can stand for the i of -ti + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, BoundedRunSuffix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtz"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty, "the parse that needs the restored segment must survive"); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_SlotWithABoundedRunThatMayBeEmpty_StaysImpureWhenTheMaterialIsAbsent() + { + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, BoundedRunSuffix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the slot may restore up to one segment, so no pruning"); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_OnEveryFusionTemplateAndWord_KeepsTheParsesAndNeverViolatesTheShadow(int maxDegreeOfParallelism) + { + var templates = new Dictionary + { + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, FusionPrefix("fuse", "k", "a")))] = new[] + { + "kasagti", + "ksagti", + "sagu", + "sagti", + }, + [ + TemplateOf( + Slot(false, Suffix("ti")), + Slot(false, FusionPrefix("fuse", "k", "a")), + Slot(true, Suffix("z")) + ) + ] = new[] { "kasagtiz", "kasagtzi", "kasagz" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, BoundedRunSuffix("z")))] = new[] + { + "sagtz", + "sagz", + "sagtiz", + }, + [ + TemplateOf( + Slot(false, Suffix("u")), + Slot(false, FusionPrefix("fuse", "k", "a")), + Slot(false, Suffix("ti")) + ) + ] = new[] { "kasagtiu", "kasaguti", "sagtiu" }, + }; + + foreach ((AffixTemplate template, string[] forms) in templates) + { + foreach (string form in forms) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult shadow = Walk(template, form, RequiredSlotLookaheadMode.Shadow, maxDegreeOfParallelism); + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Violations, Is.EqualTo(0), form); + }); + } + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs index ee799323a..494235ddc 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs @@ -307,10 +307,10 @@ MorphologicalOutputAction[] rhs } [Test] - public void Apply_CopiedPartThatMayMatchNothing_MakesTheSlotImpure() + public void Apply_CopiedPartCappedAtOneThatMayMatchNothing_MakesTheSlotImpure() { (WalkResult off, WalkResult on) = WalkWithOptionalSlot( - new[] { Pattern.New("1").Annotation(Any).ZeroOrMore.Value }, + new[] { Pattern.New("1").Annotation(Any).Optional.Value }, new MorphologicalOutputAction[] { new CopyFromInput("1") } ); From 91ee3dac5f02bb0b22815799b84aeeb048bcb547 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 30 Oct 2026 12:12:34 -0400 Subject: [PATCH 4/4] Make required-slot pruning switchable; treat untyped and boundary-only rules conservatively - Morpher.PruneRequiredSlots turns the prune off (default on); set it before parsing. - Tracing runs the slot rules without pruning, so failure reasons stay. - An allomorph with an untyped constraint is impure and unprunable, so a grammar that compiled before cannot throw. - A boundary-only insertion no longer counts as an infix. - Drop the redundant HasSegmentConstraint guard. - Tests for the optional-node rule, the suffix-side window, empty and deleted unbounded runs, and ordered copies. --- .../AnalysisAffixTemplateRule.cs | 2 +- .../Morpher.cs | 10 +++ .../RequiredSlotMaterial.cs | 38 ++++++++--- .../AnalysisAffixTemplateFusionTests.cs | 40 ++++++++++++ .../AnalysisAffixTemplateOrderTests.cs | 65 +++++++++++++++++++ .../AnalysisAffixTemplateRuleTests.cs | 64 ++++++++++++++---- 6 files changed, 196 insertions(+), 23 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index 1f95113d0..f93dff9c5 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -88,7 +88,7 @@ public IEnumerable Apply(Word input) private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired) { RequiredSlotLookaheadMode mode = _morpher.RequiredSlotLookahead; - if (mode == RequiredSlotLookaheadMode.Off) + if (mode == RequiredSlotLookaheadMode.Off || _morpher.TraceManager.IsTracing) return false; long unifications = 0; diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index ca906f91e..6eaaf1417 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -89,6 +89,16 @@ public ITraceManager TraceManager /// internal RequiredSlotLookaheadMode RequiredSlotLookahead { get; set; } = RequiredSlotLookaheadMode.On; + /// + /// Enable required-slot pruning during affix template analysis. Enabled by default. Set it before + /// parsing; changing it during a parse alters how much is pruned, not the analyses returned. + /// + public bool PruneRequiredSlots + { + get { return RequiredSlotLookahead != RequiredSlotLookaheadMode.Off; } + set { RequiredSlotLookahead = value ? RequiredSlotLookaheadMode.On : RequiredSlotLookaheadMode.Off; } + } + /// /// Called in with a kind and the template and slot it is about: /// "fired" (the test is true for a required slot), "violated" (a rule of a fired slot then unapplied under diff --git a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs index 2d0e24ec8..615a6678d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs @@ -65,7 +65,7 @@ public static SlotWalk Start() /// first can leave them. /// /// - /// The tables are built once from the template and depend only on the grammar. + /// The tables are built once from the template's slots, rules, and allomorphs. /// /// Presence. The analysis pattern of an allomorph holds one top-level constraint per inserted segment, in the /// order of the right-hand side (, @@ -83,6 +83,9 @@ public static SlotWalk Start() /// be summarised (a null affix, a constraint with variables that is then matched by any segment, an unknown /// rule type) can only weaken the test. /// + /// The edge checks rely on the matchers in and + /// being anchored and filtering to segments. + /// /// Order. A pure rule is ordered when it copies each part at most once and in left-hand order, so the shape /// it unapplies to is a subsequence of the shape it was given. An ordered rule whose inserts all come before /// its first copied part (after its last) is anchored on that side: the analysis matcher is anchored at both @@ -175,8 +178,8 @@ public bool CannotUnapply(int slot, Shape shape, ref SlotWalk walk, ref long uni return false; } - // The ordinals are those of the shape, margins excluded. A node is counted against the reach only when it - // cannot be skipped, because the matcher may skip an optional node. + // AnalysisAffixProcessRule and AnalysisRealizationalAffixProcessRule use an edge-anchored, + // segment-only matcher. Optional nodes do not count against reach because the matcher may skip them. private static bool Embeds( AllomorphSummary allomorph, Shape shape, @@ -310,6 +313,19 @@ private static SlotSummary Summarize(AffixTemplateSlot slot) foreach (AffixProcessAllomorph allomorph in ruleAllomorphs) { + if ( + allomorph.Lhs.Any(HasUntypedConstraint) + || allomorph.Rhs.Where(IsInsertion).SelectMany(InsertedConstraints).Any(HasNoType) + ) + { + summary.MayAlwaysUnapply = true; + summary.IsPure = false; + summary.IsOrdered = false; + summary.ReachLeft = SlotWalk.Unbounded; + summary.ReachRight = SlotWalk.Unbounded; + continue; + } + bool pure = IsPure(allomorph); if (!pure) summary.IsPure = false; @@ -370,7 +386,7 @@ private static AllomorphSummary Summarize(AffixProcessAllomorph allomorph) sequence.Add(copy); } int added = sequence.Count - before; - if (first < 0 || (i > first && i < last)) + if (added > 0 && (first < 0 || (i > first && i < last))) infix = true; else if (i < first) pre += added; @@ -428,8 +444,6 @@ private static bool IsPure(AffixProcessAllomorph allomorph) foreach (Pattern part in allomorph.Lhs) { - if (!HasSegmentConstraint(part)) - continue; if (RestoredSegments(part) == 0) continue; if (!copied.Contains(part.Name) || MinSegments(part) == 0) @@ -456,11 +470,15 @@ private static bool IsOrdered(AffixProcessAllomorph allomorph) return true; } - private static bool HasSegmentConstraint(PatternNode node) + private static bool HasNoType(FeatureStruct fs) + { + return !fs.TryGetValue(HCFeatureSystem.Type, out SymbolicFeatureValue _); + } + + private static bool HasUntypedConstraint(PatternNode node) { - if (node is Constraint constraint) - return constraint.Type() == HCFeatureSystem.Segment; - return node.Children.Any(HasSegmentConstraint); + return (node is Constraint constraint && HasNoType(constraint.FeatureStruct)) + || node.Children.Any(HasUntypedConstraint); } // The segment nodes that AnalysisMorphologicalTransform.Untruncate writes for a part that was not diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs index 4c1c7c2a2..928b416af 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs @@ -184,6 +184,46 @@ public void Apply_SlotWithABoundedRunThatMayBeEmpty_StaysImpureWhenTheMaterialIs Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the slot may restore up to one segment, so no pruning"); } + [Test] + public void Apply_EmptyUnboundedRun_RestoresNoSegmentForTheRequiredSlot() + { + AffixProcessRule fusion = FusionPrefix("fuse", "k", "a"); + AffixTemplate template = TemplateOf(Slot(false, Suffix("i")), Slot(true, fusion)); + var morpher = new Morpher(TraceManager, Language, 1); + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment("ktat")); + input.Freeze(); + var outputs = fusion.CompileAnalysisRule(morpher).Apply(input).ToArray(); + Assert.That(outputs, Has.Length.EqualTo(1)); + Assert.That(outputs[0].Shape.Count(n => n.Type() == HCFeatureSystem.Segment), Is.EqualTo(1)); + Assert.That(outputs[0].Shape.Any(n => n.Annotation.Optional), Is.False); + (WalkResult off, WalkResult on) = WalkBoth(template, "ktat"); + Assert.That(off.Parses, Is.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [TestCase(true)] + [TestCase(false)] + public void Apply_UnboundedRunWithOrderedCopiesOrDeletion_RemainsPure(bool copyRun) + { + var actions = new List { new CopyFromInput("1") }; + if (copyRun) + actions.Add(new CopyFromInput("2")); + actions.Add(new InsertSegments(Table3, "z")); + AffixProcessRule outer = RuleOf( + "outer", + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Any).ZeroOrMore.Value, + }, + actions.ToArray() + ); + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, outer)); + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + [TestCase(1)] [TestCase(0)] public void Apply_OnEveryFusionTemplateAndWord_KeepsTheParsesAndNeverViolatesTheShadow(int maxDegreeOfParallelism) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs index 93b223e20..df2ea5655 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs @@ -225,6 +225,71 @@ public void Apply_InnerMaterialOnlyInsideTheOuterPrefix_PrunesAtEntry() }); } + [Test] + public void Apply_InnerMaterialOnlyInsideTheOuterSuffix_PrunesAtEntry() + { + AffixTemplate template = TemplateOf(Slot(false, Infix("ak")), Slot(false, Suffix("ka"))); + (WalkResult off, WalkResult on) = WalkBoth(template, "gaka"); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [Test] + public void Apply_BoundaryOnlyInfix_DoesNotMakeReachUnbounded() + { + var outer = RuleOf( + "boundary", + new[] { OneSegment("1"), Part("2") }, + new CopyFromInput("1"), + new InsertSegments(Table3, "+"), + new CopyFromInput("2") + ); + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, outer)); + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtig"); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_DeletedOptionalSegmentBeforePrefix_DoesNotCountAgainstReach(int parallelism) + { + var deletion = RuleOf( + "delete", + new[] { Pattern.New("1").Annotation(Any).Optional.Value, Part("2") }, + new CopyFromInput("2") + ); + var morpher = new Morpher(TraceManager, Language, parallelism); + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment("kasag")); + input.Freeze(); + Word restored = deletion.CompileAnalysisRule(morpher).Apply(input).Single(); + Assert.That(restored.Shape.First(n => n.Type() == HCFeatureSystem.Segment).Annotation.Optional, Is.True); + var template = TemplateOf(Slot(false, Prefix("ka"))); + var parses = new List(); + foreach ( + RequiredSlotLookaheadMode mode in new[] + { + RequiredSlotLookaheadMode.Off, + RequiredSlotLookaheadMode.On, + RequiredSlotLookaheadMode.Shadow, + } + ) + { + morpher.RequiredSlotLookahead = mode; + parses.Add( + template + .CompileAnalysisRule(morpher) + .Apply(restored) + .Select(w => w.ToString()) + .OrderBy(x => x, System.StringComparer.Ordinal) + .ToArray() + ); + } + Assert.That(parses[0], Is.Not.Empty); + Assert.That(parses[1], Is.EqualTo(parses[0])); + Assert.That(parses[2], Is.EqualTo(parses[0])); + } + [Test] public void Apply_InnerMaterialAfterTheOuterPrefix_IsNotPruned() { diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs index 494235ddc..d6f47e441 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs @@ -280,8 +280,7 @@ public void Apply_RequiredSlotWithNullAffix_IsNeverPruned() }); } - // An optional slot whose rule inserts "z" after the copied parts; the word never contains the t and i - // that the required inner slot inserts, so a pure rule lets the walk prune at entry, before the optional rule is tried, and an impure one does not. + // The input lacks the inner slot's "ti"; a pure outer rule permits pruning before it is tried. private (WalkResult Off, WalkResult On) WalkWithOptionalSlot( Pattern[] lhs, MorphologicalOutputAction[] rhs @@ -411,24 +410,65 @@ public void Apply_Parallel_ReturnsTheSameParsesAsSequential(int maxDegreeOfParal } [Test] - public void Apply_Tracing_ShowsAPrunedTemplateAsAFailedOutput() + public void Apply_Tracing_RunsTheRulesForDetailedFailureInformation() { AffixTemplate template = RequiredInnerSuffixTemplate(); Word input = Input("sagufz"); - Walk(template, input, RequiredSlotLookaheadMode.On, tracing: true); + WalkResult result = Walk(template, input, RequiredSlotLookaheadMode.On, tracing: true); TraceManager.IsTracing = false; + Assert.That(result.RuleAttempts, Is.GreaterThan(0)); var root = (Trace)input.CurrentTrace; - Trace[] children = root.Children.ToArray(); - Assert.Multiple(() => + Assert.That(root.Children.Count, Is.GreaterThan(2)); + } + + [Test] + public void Apply_PublicPruningSwitch_ControlsRuleAttempts() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + int attempts = 0; + var morpher = new Morpher(TraceManager, Language, 1) { - Assert.That( - children.Select(t => t.Type), - Is.EqualTo(new[] { TraceType.TemplateAnalysisInput, TraceType.TemplateAnalysisOutput }) - ); - Assert.That(children[1].Output, Is.Null, "the partial is reported as not unapplied"); - }); + RuleSelector = rule => + { + if (rule is AffixProcessRule) + attempts++; + return true; + }, + }; + Word input = Input("sagufz"); + input.Freeze(); + var rule = template.CompileAnalysisRule(morpher); + Assert.That(morpher.PruneRequiredSlots, Is.True); + Assert.That(rule.Apply(input), Is.Empty); + Assert.That(attempts, Is.EqualTo(0)); + morpher.PruneRequiredSlots = false; + Assert.That(rule.Apply(input), Is.Empty); + Assert.That(attempts, Is.GreaterThan(0)); + } + + [TestCase(true)] + [TestCase(false)] + public void Compile_UntypedConstraints_DisablesTheSummaryConservatively(bool untypedLhs) + { + var fs = FeatureStruct.New().Value; + fs.Freeze(); + var rule = SuffixRule("untyped", "z"); + if (untypedLhs) + { + rule.Allomorphs[0].Lhs.Add(Pattern.New("2").Annotation(fs).Value); + } + else + { + rule.Allomorphs[0] + .Rhs.Add(new InsertSimpleContext(new NaturalClass(fs), System.Array.Empty())); + } + var summary = new RequiredSlotMaterial(TemplateOf(Slot(false, rule))); + var walk = SlotWalk.Start(); + long unifications = 0; + Assert.That(summary.IsPure(0), Is.False); + Assert.That(summary.CannotUnapply(0, Input("sag").Shape, ref walk, ref unifications), Is.False); } [TestCase(1)]