diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index 9044a8488..f93dff9c5 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -14,6 +14,7 @@ internal class AnalysisAffixTemplateRule : IRule private readonly Morpher _morpher; private readonly AffixTemplate _template; private readonly List> _rules; + private readonly RequiredSlotMaterial _lookahead; public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) { @@ -26,6 +27,7 @@ public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) FreezableEqualityComparer.Default )) ); + _lookahead = new RequiredSlotMaterial(template); } public IEnumerable Apply(Word input) @@ -61,24 +63,98 @@ public IEnumerable Apply(Word input) return Enumerable.Empty(); } + int[] fired = null; + if (RequiredSlotCannotUnapply(input, _rules.Count - 1, ref fired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, input, false, FailureReason.None); + return Enumerable.Empty(); + } + Word inWord = input.Clone(); inWord.Freeze(); var output = new HashSet(FreezableEqualityComparer.Default); if (_morpher.MaxDegreeOfParallelism == 1) - ApplySlots(inWord, _rules.Count - 1, output); + ApplySlots(inWord, _rules.Count - 1, output, fired); else - ParallelApplySlots(inWord, output); + ParallelApplySlots(inWord, output, fired); return output; } - private void ApplySlots(Word inWord, int index, HashSet output) + // Testing runs from the next slot to unapply and stops after an impure slot, which makes the test + // for any slot behind it unsound. Shadow mode prunes nothing and carries the fired slots in fired. + private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired) + { + RequiredSlotLookaheadMode mode = _morpher.RequiredSlotLookahead; + if (mode == RequiredSlotLookaheadMode.Off || _morpher.TraceManager.IsTracing) + return false; + + long unifications = 0; + bool cannot = false; + SlotWalk walk = SlotWalk.Start(); + for (int j = lastSlot; j >= 0; j--) + { + if ( + !_template.Slots[j].Optional + && !IsFired(fired, j) + && _lookahead.CannotUnapply(j, word.Shape, ref walk, ref unifications) + ) + { + if (mode == RequiredSlotLookaheadMode.On) + { + cannot = true; + break; + } + fired = fired == null ? new[] { j } : fired.Append(j).ToArray(); + _morpher.RequiredSlotShadow?.Invoke("fired", _template, j); + } + _lookahead.Advance(j, ref walk); + if (!_lookahead.IsPure(j)) + break; + } + if (unifications != 0) + _morpher.RequiredSlotCheckObserver?.Invoke(unifications); + return cannot; + } + + private static bool IsFired(int[] fired, int slot) + { + return fired != null && Array.IndexOf(fired, slot) >= 0; + } + + private void ReportShadow(string kind, int[] fired, int slot) + { + if (fired == null || _morpher.RequiredSlotShadow == null) + return; + if (slot < 0) + { + foreach (int j in fired) + _morpher.RequiredSlotShadow(kind, _template, j); + } + else if (IsFired(fired, slot)) + { + _morpher.RequiredSlotShadow(kind, _template, slot); + } + } + + private void ApplySlots(Word inWord, int index, HashSet output, int[] fired) { for (int i = index; i >= 0; i--) { foreach (Word outWord in _rules[i].Apply(inWord)) - ApplySlots(outWord, i - 1, output); + { + ReportShadow("violated", fired, i); + int[] outFired = fired; + if (RequiredSlotCannotUnapply(outWord, i - 1, ref outFired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, outWord, false, FailureReason.None); + continue; + } + ApplySlots(outWord, i - 1, output, outFired); + } if (!_template.Slots[i].Optional) { @@ -90,16 +166,17 @@ private void ApplySlots(Word inWord, int index, HashSet output) if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None); + ReportShadow("completed-under-fired", fired, -1); output.Add(inWord); } - private void ParallelApplySlots(Word inWord, HashSet output) + private void ParallelApplySlots(Word inWord, HashSet output, int[] fired) { ParallelOptions parallelOptions = _morpher.CreateParallelOptions(); var outStack = new ConcurrentStack(); - var from = new ConcurrentStack>(); - from.Push(Tuple.Create(inWord, _rules.Count - 1)); - var to = new ConcurrentStack>(); + var from = new ConcurrentStack>(); + from.Push(Tuple.Create(inWord, _rules.Count - 1, fired)); + var to = new ConcurrentStack>(); while (!from.IsEmpty) { to.Clear(); @@ -111,9 +188,25 @@ private void ParallelApplySlots(Word inWord, HashSet output) bool add = true; for (int i = work.Item2; i >= 0; i--) { - Tuple[] workItems = _rules[i] + int next = i - 1; + Tuple[] workItems = _rules[i] .Apply(work.Item1) - .Select(res => Tuple.Create(res, i - 1)) + .Select(res => + { + ReportShadow("violated", work.Item3, i); + int[] resFired = work.Item3; + if (!RequiredSlotCannotUnapply(res, next, ref resFired)) + return Tuple.Create(res, next, resFired); + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate( + _template, + res, + false, + FailureReason.None + ); + return null; + }) + .Where(item => item != null) .ToArray(); if (workItems.Length > 0) to.PushRange(workItems); @@ -141,11 +234,12 @@ private void ParallelApplySlots(Word inWord, HashSet output) true, FailureReason.None ); + ReportShadow("completed-under-fired", work.Item3, -1); outStack.Push(work.Item1); } } ); - ConcurrentStack> temp = from; + ConcurrentStack> temp = from; from = to; to = temp; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index d0675509e..6eaaf1417 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -83,6 +83,36 @@ public ITraceManager TraceManager public int DeletionReapplications { get; set; } + /// + /// Whether affix template walks drop partials in which a required slot can no longer unapply because + /// the material it inserts is absent. See . + /// + internal RequiredSlotLookaheadMode RequiredSlotLookahead { get; set; } = RequiredSlotLookaheadMode.On; + + /// + /// Enable required-slot pruning during affix template analysis. Enabled by default. Set it before + /// parsing; changing it during a parse alters how much is pruned, not the analyses returned. + /// + public bool PruneRequiredSlots + { + get { return RequiredSlotLookahead != RequiredSlotLookaheadMode.Off; } + set { RequiredSlotLookahead = value ? RequiredSlotLookaheadMode.On : RequiredSlotLookaheadMode.Off; } + } + + /// + /// Called in with a kind and the template and slot it is about: + /// "fired" (the test is true for a required slot), "violated" (a rule of a fired slot then unapplied under + /// that partial) and "completed-under-fired" (the template walk completed with a fired slot on its path). + /// Any "violated" or "completed-under-fired" is a soundness failure. + /// + internal Action RequiredSlotShadow { get; set; } + + /// + /// Called after each required-slot check that unified, with the number of unifications it performed. + /// Null unless a diagnostic sets it; it may be called from several threads. + /// + internal Action RequiredSlotCheckObserver { get; set; } + public int MaxStemCount { get; set; } /// diff --git a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs new file mode 100644 index 000000000..615a6678d --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs @@ -0,0 +1,537 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// How uses . + /// + internal enum RequiredSlotLookaheadMode + { + /// The look-ahead is not evaluated. + Off, + + /// The look-ahead is evaluated and reported to Morpher.RequiredSlotShadow, never pruning. + Shadow, + + /// The look-ahead prunes template partials. + On, + } + + /// + /// What a walk over the slots of one template has established about the shape, in the ordinals of the shape + /// the walk started from. Create it with . + /// + internal struct SlotWalk + { + /// Stands for "any number" in and . + public const int Unbounded = int.MaxValue / 4; + + /// Every node the slots still to unapply can match lies after this ordinal. + public int Lo; + + /// Every node the slots still to unapply can match lies before this ordinal. + public int Hi; + + /// The most nodes that the slots walked so far can have removed from the left edge. + public int ReachLeft; + + /// The most nodes that the slots walked so far can have removed from the right edge. + public int ReachRight; + + /// False once a slot walked so far can reorder the nodes it copies. + public bool Ordered; + + public static SlotWalk Start() + { + return new SlotWalk + { + Lo = -1, + Hi = int.MaxValue, + Ordered = true, + }; + } + } + + /// + /// A necessary condition for a required affix template slot to unapply later in a template walk: some + /// allomorph of some rule in the slot must have its inserted segments, in the order it inserts them, unify + /// with distinct segment nodes of the shape in that order, and the nodes must be where the slots that unapply + /// first can leave them. + /// + /// + /// The tables are built once from the template's slots, rules, and allomorphs. + /// + /// Presence. The analysis pattern of an allomorph holds one top-level constraint per inserted segment, in the + /// order of the right-hand side (, + /// ), and the rule can only unapply if the constraints + /// unify with segment nodes of the shape in that order, each node used once. A rule is pure when every + /// right-hand side action copies an input part or inserts material, and every left-hand part is copied by a + /// that must capture a segment, or restores no segment when it captures nothing + /// ( writes nothing back for an unbounded quantifier). Then + /// only copies input nodes and never untruncates a + /// segment. Unification can only narrow the feature structure of a copied node, so a constraint that unifies + /// with no node of the shape unifies with no node of the shape after any number of pure unapplications. Inside + /// one template walk only the rules of that template's slots run, and the stratum's phonological rules ran + /// before the walk. The test for slot j is therefore used only when every rule in the slots still to be walked + /// after j is pure. Optional nodes count as present because the matcher can match them. Anything that cannot + /// be summarised (a null affix, a constraint with variables that is then matched by any segment, an unknown + /// rule type) can only weaken the test. + /// + /// The edge checks rely on the matchers in and + /// being anchored and filtering to segments. + /// + /// Order. A pure rule is ordered when it copies each part at most once and in left-hand order, so the shape + /// it unapplies to is a subsequence of the shape it was given. An ordered rule whose inserts all come before + /// its first copied part (after its last) is anchored on that side: the analysis matcher is anchored at both + /// ends of the shape, so the first inserted constraint must match the first segment left once the outer slots + /// have removed their own prefixes. A rule can remove from an edge at most as many nodes as it inserts there + /// (), and the nodes of a required ordered slot lie after the prefix and before + /// the suffix it inserts (, ). Both only hold while every + /// slot walked so far is ordered, and the reach only while none can remove from the middle. + /// + internal class RequiredSlotMaterial + { + private readonly SlotSummary[] _slots; + + public RequiredSlotMaterial(AffixTemplate template) + { + _slots = template.Slots.Select(Summarize).ToArray(); + } + + /// + /// Whether every rule of the slot is pure, so that unapplying the slot leaves the segments needed by + /// slots before it no easier to find. + /// + public bool IsPure(int slot) + { + return _slots[slot].IsPure; + } + + /// + /// Records that the slot is the next one to unapply, so that the slots before it are tested against what + /// it can leave of the shape. + /// + public void Advance(int slot, ref SlotWalk walk) + { + SlotSummary summary = _slots[slot]; + if (!walk.Ordered || !summary.IsOrdered) + { + walk.Ordered = false; + walk.Lo = -1; + walk.Hi = int.MaxValue; + walk.ReachLeft = SlotWalk.Unbounded; + walk.ReachRight = SlotWalk.Unbounded; + return; + } + walk.ReachLeft = Math.Min(SlotWalk.Unbounded, walk.ReachLeft + summary.ReachLeft); + walk.ReachRight = Math.Min(SlotWalk.Unbounded, walk.ReachRight + summary.ReachRight); + } + + /// + /// True if no allomorph of the slot can unapply to a word with this shape, provided that every slot + /// walked between this word and the slot is pure. When the slot can unapply and is ordered, narrows the + /// window of the walk for the slots before it. + /// + /// The slot to test. + /// The shape of the partial. + /// What the slots walked since the shape was taken have established. + /// Incremented by the feature structure unifications performed. + public bool CannotUnapply(int slot, Shape shape, ref SlotWalk walk, ref long unifications) + { + SlotSummary summary = _slots[slot]; + if (summary.MayAlwaysUnapply) + return false; + + // The window is only narrowed for a slot that has slots after it in the walk, and that needs every + // passing allomorph, not the first. + bool narrow = walk.Ordered && summary.IsOrdered && slot > 0; + int newLo = int.MaxValue; + int newHi = -1; + bool any = false; + foreach (AllomorphSummary allomorph in summary.Allomorphs) + { + int prefixEnd = -1; + int suffixStart = int.MaxValue; + bool found = walk.Ordered + ? Embeds(allomorph, shape, ref walk, ref unifications, out prefixEnd, out suffixStart) + : AllPresent(allomorph, shape, ref unifications); + if (!found) + continue; + + if (!narrow) + return false; + any = true; + newLo = Math.Min(newLo, allomorph.Pre > 0 ? prefixEnd : walk.Lo); + newHi = Math.Max(newHi, allomorph.Post > 0 ? suffixStart : walk.Hi); + } + if (!any) + return true; + + walk.Lo = newLo; + walk.Hi = newHi; + return false; + } + + // AnalysisAffixProcessRule and AnalysisRealizationalAffixProcessRule use an edge-anchored, + // segment-only matcher. Optional nodes do not count against reach because the matcher may skip them. + private static bool Embeds( + AllomorphSummary allomorph, + Shape shape, + ref SlotWalk walk, + ref long unifications, + out int prefixEnd, + out int suffixStart + ) + { + prefixEnd = -1; + suffixStart = int.MaxValue; + FeatureStruct[] sequence = allomorph.Sequence; + int front = sequence.Length - allomorph.Post; + int forwardEnd = -1; + + int next = 0; + int before = 0; + int ordinal = -1; + foreach (ShapeNode node in shape) + { + if (next == front) + break; + ordinal++; + if (ordinal >= walk.Hi) + break; + + bool segment = node.Type() == HCFeatureSystem.Segment; + if (segment && ordinal > walk.Lo) + { + if (next < allomorph.Pre && before > Math.Min(SlotWalk.Unbounded, walk.ReachLeft + next)) + return false; + + FeatureStruct required = sequence[next]; + if (required == null || IsUnifiable(node, required, ref unifications)) + { + if (next == allomorph.Pre - 1) + prefixEnd = ordinal; + forwardEnd = ordinal; + next++; + } + } + if (segment && !node.Annotation.Optional) + before++; + } + if (next < front) + return false; + if (allomorph.Post == 0) + return true; + + int taken = 0; + int after = 0; + ordinal = shape.Count; + for (ShapeNode node = shape.Last; node != null && taken < allomorph.Post; node = node.Prev) + { + ordinal--; + if (ordinal <= walk.Lo || ordinal <= forwardEnd) + break; + + bool segment = node.Type() == HCFeatureSystem.Segment; + if (segment && ordinal < walk.Hi) + { + if (after > Math.Min(SlotWalk.Unbounded, walk.ReachRight + taken)) + return false; + + FeatureStruct required = sequence[sequence.Length - 1 - taken]; + if (required == null || IsUnifiable(node, required, ref unifications)) + { + suffixStart = ordinal; + taken++; + } + } + if (segment && !node.Annotation.Optional) + after++; + } + return taken == allomorph.Post; + } + + // Once a slot that can reorder nodes has been walked the order of the shape says nothing about the order of + // the shape the slot unapplies to, so each constraint is only looked for on its own. + private static bool AllPresent(AllomorphSummary allomorph, Shape shape, ref long unifications) + { + foreach (FeatureStruct required in allomorph.Sequence) + { + if (required == null) + continue; + + bool found = false; + foreach (ShapeNode node in shape) + { + if (node.Type() == HCFeatureSystem.Segment && IsUnifiable(node, required, ref unifications)) + { + found = true; + break; + } + } + if (!found) + return false; + } + return true; + } + + private static bool IsUnifiable(ShapeNode node, FeatureStruct required, ref long unifications) + { + unifications++; + return node.Annotation.FeatureStruct.IsUnifiable(required, false, null); + } + + private static SlotSummary Summarize(AffixTemplateSlot slot) + { + var summary = new SlotSummary(); + var allomorphs = new List(); + foreach (MorphemicMorphologicalRule rule in slot.Rules) + { + IList ruleAllomorphs; + switch (rule) + { + case AffixProcessRule affixRule: + ruleAllomorphs = affixRule.Allomorphs; + break; + case RealizationalAffixProcessRule realizationalRule: + ruleAllomorphs = realizationalRule.Allomorphs; + break; + default: + summary.MayAlwaysUnapply = true; + summary.IsPure = false; + summary.IsOrdered = false; + summary.ReachLeft = SlotWalk.Unbounded; + summary.ReachRight = SlotWalk.Unbounded; + continue; + } + + foreach (AffixProcessAllomorph allomorph in ruleAllomorphs) + { + if ( + allomorph.Lhs.Any(HasUntypedConstraint) + || allomorph.Rhs.Where(IsInsertion).SelectMany(InsertedConstraints).Any(HasNoType) + ) + { + summary.MayAlwaysUnapply = true; + summary.IsPure = false; + summary.IsOrdered = false; + summary.ReachLeft = SlotWalk.Unbounded; + summary.ReachRight = SlotWalk.Unbounded; + continue; + } + + bool pure = IsPure(allomorph); + if (!pure) + summary.IsPure = false; + if (!pure || !IsOrdered(allomorph)) + summary.IsOrdered = false; + + AllomorphSummary allomorphSummary = Summarize(allomorph); + summary.ReachLeft = Math.Max(summary.ReachLeft, allomorphSummary.ReachLeft); + summary.ReachRight = Math.Max(summary.ReachRight, allomorphSummary.ReachRight); + if (allomorphSummary.Sequence.Length == 0) + summary.MayAlwaysUnapply = true; + else + allomorphs.Add(allomorphSummary); + } + } + summary.Allomorphs = allomorphs.ToArray(); + return summary; + } + + // Pre is the inserts before the first non-insertion, Post those after the last; any between is an infix. + // A non-segment constraint matches nothing the matcher sees, so skipping it only weakens the test. + private static AllomorphSummary Summarize(AffixProcessAllomorph allomorph) + { + var sequence = new List(); + IList rhs = allomorph.Rhs; + int first = -1; + int last = -1; + for (int i = 0; i < rhs.Count; i++) + { + if (IsInsertion(rhs[i])) + continue; + if (first < 0) + first = i; + last = i; + } + + int pre = 0; + int post = 0; + bool infix = false; + for (int i = 0; i < rhs.Count; i++) + { + if (!IsInsertion(rhs[i])) + continue; + + int before = sequence.Count; + foreach (FeatureStruct fs in InsertedConstraints(rhs[i])) + { + if ((FeatureSymbol)fs.GetValue(HCFeatureSystem.Type) != HCFeatureSystem.Segment) + continue; + + // A constraint with variables is bound by earlier matches, so any segment stands for it. + FeatureStruct copy = null; + if (!fs.HasVariables) + { + copy = fs.Clone(); + copy.Freeze(); + } + sequence.Add(copy); + } + int added = sequence.Count - before; + if (added > 0 && (first < 0 || (i > first && i < last))) + infix = true; + else if (i < first) + pre += added; + else + post += added; + } + + var summary = new AllomorphSummary { Sequence = sequence.ToArray() }; + if (infix) + { + summary.ReachLeft = SlotWalk.Unbounded; + summary.ReachRight = SlotWalk.Unbounded; + } + else + { + summary.Pre = pre; + summary.Post = post; + summary.ReachLeft = pre; + summary.ReachRight = post; + } + return summary; + } + + private static bool IsInsertion(MorphologicalOutputAction action) + { + return action is InsertSegments || action is InsertSimpleContext; + } + + private static IEnumerable InsertedConstraints(MorphologicalOutputAction action) + { + if (action is InsertSegments insertSegments) + { + foreach (ShapeNode node in insertSegments.Segments.Shape) + { + if (node.Type() != HCFeatureSystem.Boundary) + yield return node.Annotation.FeatureStruct; + } + } + else if (action is InsertSimpleContext insertContext) + { + yield return insertContext.SimpleContext.FeatureStruct; + } + } + + private static bool IsPure(AffixProcessAllomorph allomorph) + { + var copied = new HashSet(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (action is CopyFromInput) + copied.Add(action.PartName); + else if (!IsInsertion(action)) + return false; + } + + foreach (Pattern part in allomorph.Lhs) + { + if (RestoredSegments(part) == 0) + continue; + if (!copied.Contains(part.Name) || MinSegments(part) == 0) + return false; + } + return true; + } + + // GenerateShape writes parts in left-hand order, so copying them in another order, or one twice, can + // reorder or repeat nodes. + private static bool IsOrdered(AffixProcessAllomorph allomorph) + { + int previous = -1; + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (!(action is CopyFromInput)) + continue; + + int index = allomorph.Lhs.ToList().FindIndex(part => part.Name == action.PartName); + if (index <= previous) + return false; + previous = index; + } + return true; + } + + private static bool HasNoType(FeatureStruct fs) + { + return !fs.TryGetValue(HCFeatureSystem.Type, out SymbolicFeatureValue _); + } + + private static bool HasUntypedConstraint(PatternNode node) + { + return (node is Constraint constraint && HasNoType(constraint.FeatureStruct)) + || node.Children.Any(HasUntypedConstraint); + } + + // The segment nodes that AnalysisMorphologicalTransform.Untruncate writes for a part that was not + // captured. An unbounded quantifier writes none, as in Mbugwe's vowel-fusion slots. + private static int RestoredSegments(PatternNode node) + { + int total = 0; + foreach (PatternNode child in node.Children) + { + if (child is Constraint constraint && constraint.Type() == HCFeatureSystem.Segment) + total++; + else if (child is Quantifier quantifier) + total += Math.Max(0, quantifier.MaxOccur) * RestoredSegments(quantifier); + else + total += RestoredSegments(child); + } + return total; + } + + // The fewest segments a successful match of the node must consume. A part matching none is captured + // as a null range, and GenerateShape then untruncates it, which adds segments that are not copies. + private static int MinSegments(PatternNode node) + { + switch (node) + { + case Constraint constraint: + return constraint.Type() == HCFeatureSystem.Segment ? 1 : 0; + case Quantifier quantifier: + return quantifier.MinOccur * quantifier.Children.Sum(MinSegments); + case Alternation alternation: + return alternation.Children.Count == 0 ? 0 : alternation.Children.Min(MinSegments); + default: + return node.Children.Sum(MinSegments); + } + } + + private class AllomorphSummary + { + public FeatureStruct[] Sequence; + public int Pre; + public int Post; + public int ReachLeft; + public int ReachRight; + } + + private class SlotSummary + { + public bool IsPure = true; + public bool IsOrdered = true; + public bool MayAlwaysUnapply; + public int ReachLeft; + public int ReachRight; + public AllomorphSummary[] Allomorphs; + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs new file mode 100644 index 000000000..928b416af --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs @@ -0,0 +1,278 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +// A vowel-fusion slot copies a part that may match nothing; the look-ahead may test behind it only if it +// restores no segment. Slots are in synthesis order and unapplied last to first. +[TestFixture] +public class AnalysisAffixTemplateFusionTests : HermitCrabTestBase +{ + private static readonly FeatureStruct Any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + private sealed record WalkResult(List Parses, int RuleAttempts, int Violations); + + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "fusion_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + private static AffixProcessRule RuleOf( + string name, + Pattern[] lhs, + params MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + rule.Allomorphs.Add(allomorph); + return rule; + } + + private AffixProcessRule Suffix(string insert) => + RuleOf( + "suffix_" + insert, + new[] { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + new CopyFromInput("1"), + new InsertSegments(Table3, insert) + ); + + // Like Mbugwe's object markers: a prefix whose stem vowel is copied twice, followed by an unbounded run + // that may be empty. + private AffixProcessRule FusionPrefix(string name, string before, string between) => + RuleOf( + name, + new[] + { + Pattern.New("1").Annotation(Any).Value, + Pattern.New("2").Annotation(Any).ZeroOrMore.Value, + }, + new InsertSegments(Table3, before), + new CopyFromInput("1"), + new InsertSegments(Table3, between), + new CopyFromInput("1"), + new CopyFromInput("2") + ); + + // The run is capped at one segment, so an unmatched run is written back as an optional segment. + private AffixProcessRule BoundedRunSuffix(string insert) => + RuleOf( + "bounded_" + insert, + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Any).Optional.Value, + }, + new CopyFromInput("1"), + new InsertSegments(Table3, insert) + ); + + private WalkResult Walk( + AffixTemplate template, + string form, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1 + ) + { + int attempts = 0; + int violations = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + RequiredSlotShadow = (kind, t, slot) => + { + if (kind != "fired") + System.Threading.Interlocked.Increment(ref violations); + }, + }; + TraceManager.IsTracing = false; + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + input.Freeze(); + List parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts, violations); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses of " + form); + return (off, on); + } + + [Test] + public void Apply_FusionSlotWithAnUnboundedRun_StillTestsTheRequiredSlotBehindIt() + { + // -ti is required inside the fusion prefix and "sagu" has no t + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, FusionPrefix("fuse", "k", "a"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagu"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0), "the fusion slot restores nothing, so -ti cannot appear"); + }); + } + + [Test] + public void Apply_FusionSlotWithAnUnboundedRun_KeepsAWordThatHasTheMaterial() + { + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, FusionPrefix("fuse", "k", "a"))); + + foreach (string form in new[] { "kasagti", "ksagti", "sagti", "kasagtiz" }) + { + (WalkResult off, WalkResult on) = WalkBoth(template, form); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts), form); + } + } + + [Test] + public void Apply_SlotWithABoundedRunThatMayBeEmpty_RestoresASegmentTheInnerSlotCanUse() + { + // the run is written back as an optional segment, which can stand for the i of -ti + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, BoundedRunSuffix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtz"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty, "the parse that needs the restored segment must survive"); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_SlotWithABoundedRunThatMayBeEmpty_StaysImpureWhenTheMaterialIsAbsent() + { + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, BoundedRunSuffix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the slot may restore up to one segment, so no pruning"); + } + + [Test] + public void Apply_EmptyUnboundedRun_RestoresNoSegmentForTheRequiredSlot() + { + AffixProcessRule fusion = FusionPrefix("fuse", "k", "a"); + AffixTemplate template = TemplateOf(Slot(false, Suffix("i")), Slot(true, fusion)); + var morpher = new Morpher(TraceManager, Language, 1); + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment("ktat")); + input.Freeze(); + var outputs = fusion.CompileAnalysisRule(morpher).Apply(input).ToArray(); + Assert.That(outputs, Has.Length.EqualTo(1)); + Assert.That(outputs[0].Shape.Count(n => n.Type() == HCFeatureSystem.Segment), Is.EqualTo(1)); + Assert.That(outputs[0].Shape.Any(n => n.Annotation.Optional), Is.False); + (WalkResult off, WalkResult on) = WalkBoth(template, "ktat"); + Assert.That(off.Parses, Is.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [TestCase(true)] + [TestCase(false)] + public void Apply_UnboundedRunWithOrderedCopiesOrDeletion_RemainsPure(bool copyRun) + { + var actions = new List { new CopyFromInput("1") }; + if (copyRun) + actions.Add(new CopyFromInput("2")); + actions.Add(new InsertSegments(Table3, "z")); + AffixProcessRule outer = RuleOf( + "outer", + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Any).ZeroOrMore.Value, + }, + actions.ToArray() + ); + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, outer)); + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_OnEveryFusionTemplateAndWord_KeepsTheParsesAndNeverViolatesTheShadow(int maxDegreeOfParallelism) + { + var templates = new Dictionary + { + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, FusionPrefix("fuse", "k", "a")))] = new[] + { + "kasagti", + "ksagti", + "sagu", + "sagti", + }, + [ + TemplateOf( + Slot(false, Suffix("ti")), + Slot(false, FusionPrefix("fuse", "k", "a")), + Slot(true, Suffix("z")) + ) + ] = new[] { "kasagtiz", "kasagtzi", "kasagz" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, BoundedRunSuffix("z")))] = new[] + { + "sagtz", + "sagz", + "sagtiz", + }, + [ + TemplateOf( + Slot(false, Suffix("u")), + Slot(false, FusionPrefix("fuse", "k", "a")), + Slot(false, Suffix("ti")) + ) + ] = new[] { "kasagtiu", "kasaguti", "sagtiu" }, + }; + + foreach ((AffixTemplate template, string[] forms) in templates) + { + foreach (string form in forms) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult shadow = Walk(template, form, RequiredSlotLookaheadMode.Shadow, maxDegreeOfParallelism); + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Violations, Is.EqualTo(0), form); + }); + } + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs new file mode 100644 index 000000000..df2ea5655 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateOrderTests.cs @@ -0,0 +1,427 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +// Order and edge tests of the required-slot look-ahead; slots are in synthesis order and unapplied last to first. +[TestFixture] +public class AnalysisAffixTemplateOrderTests : HermitCrabTestBase +{ + private static readonly FeatureStruct Any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + private sealed record WalkResult(List Parses, int RuleAttempts, int Violations); + + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "order_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + private static Pattern Part(string name) => + Pattern.New(name).Annotation(Any).OneOrMore.Value; + + private static Pattern OneSegment(string name) => + Pattern.New(name).Annotation(Any).Value; + + private static AffixProcessRule RuleOf( + string name, + Pattern[] lhs, + params MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + rule.Allomorphs.Add(allomorph); + return rule; + } + + private AffixProcessRule Suffix(string insert) => + RuleOf("suffix_" + insert, new[] { Part("1") }, new CopyFromInput("1"), new InsertSegments(Table3, insert)); + + private AffixProcessRule Prefix(string insert) => + RuleOf("prefix_" + insert, new[] { Part("1") }, new InsertSegments(Table3, insert), new CopyFromInput("1")); + + private AffixProcessRule Circumfix(string before, string after) => + RuleOf( + "circumfix_" + before + after, + new[] { Part("1") }, + new InsertSegments(Table3, before), + new CopyFromInput("1"), + new InsertSegments(Table3, after) + ); + + // inserts the material after the first segment of the stem + private AffixProcessRule Infix(string insert) => + RuleOf( + "infix_" + insert, + new[] { OneSegment("1"), Part("2") }, + new CopyFromInput("1"), + new InsertSegments(Table3, insert), + new CopyFromInput("2") + ); + + // writes the last segment of the stem twice + private static AffixProcessRule RepeatLast() => + RuleOf( + "repeat_last", + new[] { Part("1"), OneSegment("2") }, + new CopyFromInput("1"), + new CopyFromInput("2"), + new CopyFromInput("2") + ); + + // swaps the last two segments of the stem, so it copies its parts out of order + private static AffixProcessRule Metathesis() => + RuleOf( + "metathesis", + new[] { Part("1"), OneSegment("2"), OneSegment("3") }, + new CopyFromInput("1"), + new CopyFromInput("3"), + new CopyFromInput("2") + ); + + private WalkResult Walk( + AffixTemplate template, + string form, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1 + ) + { + int attempts = 0; + int violations = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + RequiredSlotShadow = (kind, t, slot) => + { + if (kind != "fired") + System.Threading.Interlocked.Increment(ref violations); + }, + }; + TraceManager.IsTracing = false; + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + input.Freeze(); + List parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts, violations); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses of " + form); + return (off, on); + } + + [Test] + public void Apply_RequiredSuffixesInTheWrongOrder_PrunesAtEntry() + { + // both literals are present in "saguti", but the outer suffix -u has to end the word + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(false, Suffix("u"))); + + (WalkResult off, WalkResult wrong) = WalkBoth(template, "saguti"); + (WalkResult offRight, WalkResult right) = WalkBoth(template, "sagtiu"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(wrong.RuleAttempts, Is.EqualTo(0)); + Assert.That(right.Parses, Is.Not.Empty); + Assert.That(right.RuleAttempts, Is.EqualTo(offRight.RuleAttempts)); + }); + } + + [Test] + public void Apply_PrefixSlotWhoseMaterialIsNotAtTheLeftEdge_PrunesAtEntry() + { + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(false, Prefix("ka"))); + + (WalkResult off, WalkResult wrong) = WalkBoth(template, "sagkati"); + (WalkResult offRight, WalkResult right) = WalkBoth(template, "kasagti"); + + Assert.Multiple(() => + { + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(wrong.RuleAttempts, Is.EqualTo(0)); + Assert.That(right.Parses, Is.Not.Empty); + Assert.That(right.RuleAttempts, Is.EqualTo(offRight.RuleAttempts)); + }); + } + + [Test] + public void Apply_CircumfixWhoseSuffixPartIsNotAtTheRightEdge_PrunesAtEntry() + { + AffixTemplate template = TemplateOf(Slot(false, Circumfix("ka", "ti"))); + + (WalkResult off, WalkResult wrong) = WalkBoth(template, "kasagtz"); + (WalkResult offRight, WalkResult right) = WalkBoth(template, "kasagti"); + + Assert.Multiple(() => + { + Assert.That(wrong.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(right.Parses, Is.Not.Empty); + Assert.That(right.RuleAttempts, Is.EqualTo(offRight.RuleAttempts)); + }); + } + + [Test] + public void Apply_CircumfixPartsThatWouldShareANode_PrunesAtEntry() + { + // ka- and -ak could both use the a of "kak" + AffixTemplate template = TemplateOf(Slot(false, Circumfix("ka", "ak"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "kak"); + + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + } + + // an infix does not anchor, so only the window left by the outer required prefix can rule out its material + private AffixTemplate InfixInsidePrefixTemplate() => + TemplateOf(Slot(false, Infix("ak")), Slot(false, Prefix("ka"))); + + [Test] + public void Apply_InnerMaterialOnlyInsideTheOuterPrefix_PrunesAtEntry() + { + // the a of "ka" is the only a before the k that -ak needs, so the infix has no material of its own + (WalkResult off, WalkResult on) = WalkBoth(InfixInsidePrefixTemplate(), "kakg"); + + Assert.Multiple(() => + { + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + }); + } + + [Test] + public void Apply_InnerMaterialOnlyInsideTheOuterSuffix_PrunesAtEntry() + { + AffixTemplate template = TemplateOf(Slot(false, Infix("ak")), Slot(false, Suffix("ka"))); + (WalkResult off, WalkResult on) = WalkBoth(template, "gaka"); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [Test] + public void Apply_BoundaryOnlyInfix_DoesNotMakeReachUnbounded() + { + var outer = RuleOf( + "boundary", + new[] { OneSegment("1"), Part("2") }, + new CopyFromInput("1"), + new InsertSegments(Table3, "+"), + new CopyFromInput("2") + ); + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, outer)); + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtig"); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_DeletedOptionalSegmentBeforePrefix_DoesNotCountAgainstReach(int parallelism) + { + var deletion = RuleOf( + "delete", + new[] { Pattern.New("1").Annotation(Any).Optional.Value, Part("2") }, + new CopyFromInput("2") + ); + var morpher = new Morpher(TraceManager, Language, parallelism); + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment("kasag")); + input.Freeze(); + Word restored = deletion.CompileAnalysisRule(morpher).Apply(input).Single(); + Assert.That(restored.Shape.First(n => n.Type() == HCFeatureSystem.Segment).Annotation.Optional, Is.True); + var template = TemplateOf(Slot(false, Prefix("ka"))); + var parses = new List(); + foreach ( + RequiredSlotLookaheadMode mode in new[] + { + RequiredSlotLookaheadMode.Off, + RequiredSlotLookaheadMode.On, + RequiredSlotLookaheadMode.Shadow, + } + ) + { + morpher.RequiredSlotLookahead = mode; + parses.Add( + template + .CompileAnalysisRule(morpher) + .Apply(restored) + .Select(w => w.ToString()) + .OrderBy(x => x, System.StringComparer.Ordinal) + .ToArray() + ); + } + Assert.That(parses[0], Is.Not.Empty); + Assert.That(parses[1], Is.EqualTo(parses[0])); + Assert.That(parses[2], Is.EqualTo(parses[0])); + } + + [Test] + public void Apply_InnerMaterialAfterTheOuterPrefix_IsNotPruned() + { + // "sag" with the infix ak is "sakag", and ka- in front gives "kasakag" + (WalkResult off, WalkResult on) = WalkBoth(InfixInsidePrefixTemplate(), "kasakag"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_OptionalOuterPrefix_CountsTowardsTheLeftEdgeBudget() + { + AffixTemplate template = TemplateOf(Slot(false, Prefix("ka")), Slot(true, Prefix("ba"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "bakasag"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_OuterSlotWithNullAffix_IsNeverPrunedAndTakesNoEdgeMaterial() + { + AffixProcessRule uOrNull = Suffix("u"); + uOrNull.Allomorphs.Add(new AffixProcessAllomorph { Lhs = { Part("1") }, Rhs = { new CopyFromInput("1") } }); + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(false, uOrNull)); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagti"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_OptionalOuterSlot_CountsTowardsTheEdgeBudget() + { + // the optional -z may stand between -ti and the end of the word + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, Suffix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtiz"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_InfixSlotBetween_DoesNotAnchorTheInnerSlot() + { + // unapplying the infix z of "kzasag" gives "kasag", which starts with ka- + AffixTemplate template = TemplateOf(Slot(false, Prefix("ka")), Slot(true, Infix("z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "kzasag"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_SlotThatCopiesOutOfOrderBetween_ChecksOnlyPresence() + { + // unapplying the metathesis of "sagit" gives "sagti", whose -ti is in the opposite order + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, Metathesis())); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagit"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_PartCopiedTwiceBetween_DoesNotAnchorTheInnerSlot() + { + // unapplying the repeated i of "sagtii" gives "sagti", which ends in -ti + AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, RepeatLast())); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtii"); + + Assert.That(on.Parses, Is.Not.Empty); + } + + [Test] + public void Apply_ConstraintWithVariables_IsMatchedByAnySegment() + { + var context = new SimpleContext(new NaturalClass(Any), new[] { Variable("vd", "a") }); + AffixProcessRule suffix = RuleOf( + "variable_suffix", + new[] { Part("1") }, + new CopyFromInput("1"), + new InsertSimpleContext(context) + ); + AffixTemplate template = TemplateOf(Slot(false, suffix)); + + foreach (string form in new[] { "sagd", "sagt", "sag" }) + WalkBoth(template, form); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_OnEveryTemplateAndWord_KeepsTheParsesAndNeverViolatesTheShadow(int maxDegreeOfParallelism) + { + var templates = new Dictionary + { + [TemplateOf(Slot(false, Suffix("ti")), Slot(false, Suffix("u")))] = new[] { "saguti", "sagtiu", "sagti" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(false, Prefix("ka")))] = new[] { "kasagti", "sagkati" }, + [TemplateOf(Slot(false, Circumfix("ka", "ti")), Slot(true, Suffix("z")), Slot(false, Prefix("ba")))] = new[] + { + "bakasagtiz", + "kabasagti", + "bakasagtzi", + }, + [TemplateOf(Slot(false, Prefix("ka")), Slot(true, Infix("z")))] = new[] { "kzasag", "kasag" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, RepeatLast()))] = new[] { "sagtii", "sagti" }, + [TemplateOf(Slot(false, Suffix("ti")), Slot(true, Metathesis()))] = new[] { "sagit", "sagti" }, + }; + + foreach ((AffixTemplate template, string[] forms) in templates) + { + foreach (string form in forms) + { + WalkResult off = Walk(template, form, RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + WalkResult on = Walk(template, form, RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult shadow = Walk(template, form, RequiredSlotLookaheadMode.Shadow, maxDegreeOfParallelism); + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Parses, Is.EqualTo(off.Parses), form); + Assert.That(shadow.Violations, Is.EqualTo(0), form); + }); + } + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs index ed5cabc36..d6f47e441 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs @@ -69,6 +69,449 @@ public void Apply_SlotLeftEmpty_LeavesTheSameStateAsNotEnteringTheTemplate() }); } + // Slots are listed in synthesis order and unapplied from the last to the first, so slot 0 is the suffix + // closest to the stem and a word ends with the material of its highest slot. + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "lookahead_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + // The analysis of a rule that deletes a final segment in synthesis: it adds the segment back, so the + // unapplication produces a shape that is not made of copies of the input. + private AffixProcessRule SegmentRestoringRule(string name, string restored) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Character(Table3, restored)).Value, + }, + Rhs = { new CopyFromInput("1") }, + } + ); + return rule; + } + + private AffixProcessRule SuffixRule(string name, string insert) => + Suffix(name, FeatureStruct.New().Value, FeatureStruct.New().Value, Table3, insert); + + private Word Input(string form, bool lastNodeOptional = false) + { + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + if (lastNodeOptional) + input.Shape.Last.Annotation.Optional = true; + return input; + } + + private sealed record WalkResult(List Parses, int RuleAttempts); + + private WalkResult Walk( + AffixTemplate template, + Word input, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1, + bool tracing = false + ) + { + int attempts = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + }; + TraceManager.IsTracing = tracing; + if (tracing) + TraceManager.AnalyzeWord(Language, input); + input.Freeze(); + IRule rule = template.CompileAnalysisRule(morpher); + List parses = rule.Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form, bool lastNodeOptional = false) + { + WalkResult off = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses"); + return (off, on); + } + + // stem + required -ti + optional -u/-f/-v + optional -z + private AffixTemplate RequiredInnerSuffixTemplate() => + TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("u", "u"), SuffixRule("f", "f"), SuffixRule("v", "v")), + Slot(true, SuffixRule("z", "z")) + ); + + [Test] + public void Apply_RequiredSlotLiteralAbsent_PrunesBeforeTryingTheOptionalSlots() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagufz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0), "no rule can supply -ti, so no slot rule is attempted"); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralPresent_DoesNotPrune() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtiuz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralLostAfterAnOptionalSlot_PrunesTheInnerPartial() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + // -z unapplies, which leaves "sagtuu"; no node can still be the i of -ti + (WalkResult off, WalkResult on) = WalkBoth(template, "sagttz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(on.RuleAttempts, Is.LessThan(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_ImpureSlotBetween_DoesNotPruneForTheInnerRequiredSlot() + { + // the middle slot restores the i that -ti needs, so "sagt" is a stem plus -ti once it has run + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SegmentRestoringRule("restore_i", "i")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagt"); + + Assert.That(on.Parses, Is.Not.Empty, "the parse that needs the restored segment must survive"); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + } + + [Test] + public void Apply_SeveralRequiredSlots_PrunesWhicheverLiteralIsAbsent() + { + // -u is required as well as -ti; the word has neither + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("z", "z")), + Slot(false, SuffixRule("u", "u")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagfz"); + + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + } + + [Test] + public void Apply_OptionalNodeThatUnifiesWithTheLiteral_CountsAsPresent() + { + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti"))); + + // "sagti" where the i stands for a deleted segment: the matcher may match or skip an optional node + (WalkResult off, WalkResult on) = WalkBoth(template, "sagti", lastNodeOptional: true); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.Parses, Is.EqualTo(off.Parses)); + }); + } + + [Test] + public void Apply_RequiredSlotWithNullAffix_IsNeverPruned() + { + AffixProcessRule withNull = SuffixRule("ti_or_null", "ti"); + withNull.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + AffixTemplate template = TemplateOf(Slot(false, withNull), Slot(true, SuffixRule("z", "z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + // The input lacks the inner slot's "ti"; a pure outer rule permits pruning before it is tried. + private (WalkResult Off, WalkResult On) WalkWithOptionalSlot( + Pattern[] lhs, + MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = "z_slot", + Name = "z_slot", + Gloss = "z_slot", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + allomorph.Rhs.Add(new InsertSegments(Table3, "z")); + rule.Allomorphs.Add(allomorph); + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti")), Slot(true, rule)); + return WalkBoth(template, "sagz"); + } + + [Test] + public void Apply_CopiedPartCappedAtOneThatMayMatchNothing_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).Optional.Value }, + new MorphologicalOutputAction[] { new CopyFromInput("1") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_ModifyFromInputInASlot_MakesTheSlotImpure() + { + FeatureStruct voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + new MorphologicalOutputAction[] + { + new ModifyFromInput("1", voiced, System.Array.Empty()), + } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_CopiedAlternationOfSegments_KeepsTheSlotPure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [Test] + public void Apply_CopiedAlternationWithAnOptionalBranch_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Optional.Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_SlotOptionalityChangedAfterCompiling_IsReadLive() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var morpher = new Morpher(TraceManager, Language, 1) { RequiredSlotLookahead = RequiredSlotLookaheadMode.On }; + IRule rule = template.CompileAnalysisRule(morpher); + Word input = Input("sagu"); + input.Freeze(); + + Assert.That(rule.Apply(input), Is.Empty, "slot 0 is required and -ti is absent"); + + template.Slots[0].Optional = true; + Assert.That(rule.Apply(input), Is.Not.Empty, "an optional slot is not tested"); + + template.Slots[0].Optional = false; + Assert.That(rule.Apply(input), Is.Empty); + } + + [TestCase(0)] + [TestCase(4)] + public void Apply_Parallel_ReturnsTheSameParsesAsSequential(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif" }) + { + WalkResult sequential = Walk(template, Input(form), RequiredSlotLookaheadMode.On); + WalkResult parallel = Walk(template, Input(form), RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult parallelOff = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Assert.That(parallel.Parses, Is.EqualTo(sequential.Parses), form); + Assert.That(parallel.Parses, Is.EqualTo(parallelOff.Parses), form); + } + } + + [Test] + public void Apply_Tracing_RunsTheRulesForDetailedFailureInformation() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + Word input = Input("sagufz"); + + WalkResult result = Walk(template, input, RequiredSlotLookaheadMode.On, tracing: true); + TraceManager.IsTracing = false; + + Assert.That(result.RuleAttempts, Is.GreaterThan(0)); + var root = (Trace)input.CurrentTrace; + Assert.That(root.Children.Count, Is.GreaterThan(2)); + } + + [Test] + public void Apply_PublicPruningSwitch_ControlsRuleAttempts() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + int attempts = 0; + var morpher = new Morpher(TraceManager, Language, 1) + { + RuleSelector = rule => + { + if (rule is AffixProcessRule) + attempts++; + return true; + }, + }; + Word input = Input("sagufz"); + input.Freeze(); + var rule = template.CompileAnalysisRule(morpher); + Assert.That(morpher.PruneRequiredSlots, Is.True); + Assert.That(rule.Apply(input), Is.Empty); + Assert.That(attempts, Is.EqualTo(0)); + morpher.PruneRequiredSlots = false; + Assert.That(rule.Apply(input), Is.Empty); + Assert.That(attempts, Is.GreaterThan(0)); + } + + [TestCase(true)] + [TestCase(false)] + public void Compile_UntypedConstraints_DisablesTheSummaryConservatively(bool untypedLhs) + { + var fs = FeatureStruct.New().Value; + fs.Freeze(); + var rule = SuffixRule("untyped", "z"); + if (untypedLhs) + { + rule.Allomorphs[0].Lhs.Add(Pattern.New("2").Annotation(fs).Value); + } + else + { + rule.Allomorphs[0] + .Rhs.Add(new InsertSimpleContext(new NaturalClass(fs), System.Array.Empty())); + } + var summary = new RequiredSlotMaterial(TemplateOf(Slot(false, rule))); + var walk = SlotWalk.Start(); + long unifications = 0; + Assert.That(summary.IsPure(0), Is.False); + Assert.That(summary.CannotUnapply(0, Input("sag").Shape, ref walk, ref unifications), Is.False); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_Shadow_ReportsFiredPointsAndNeverViolatesOrChangesTheParses(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var events = new List(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif", "sagu" }) + { + WalkResult off = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Word input = Input(form); + input.Freeze(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = RequiredSlotLookaheadMode.Shadow, + RequiredSlotShadow = (kind, t, slot) => + { + lock (events) + events.Add(kind); + }, + }; + TraceManager.IsTracing = false; + string[] parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => + w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss)) + ) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToArray(); + Assert.That(parses, Is.EqualTo(off.Parses), form); + } + + Assert.Multiple(() => + { + Assert.That(events.Count(e => e == "fired"), Is.GreaterThan(0)); + Assert.That(events.Count(e => e == "violated"), Is.EqualTo(0)); + Assert.That(events.Count(e => e == "completed-under-fired"), Is.EqualTo(0)); + }); + } + private AffixTemplate VerbTemplate(FeatureStruct required) { var template = new AffixTemplate { Name = "verb_template", RequiredSyntacticFeatureStruct = required };