Make Paragraph#substitute work on w:t nodes (approach by @Andy9822 in #187)

lib/docx/containers/paragraph.rb

@@ -70,9 +70,13 @@ def each_text_run
         # such as "{{fi", "rst_na", "me}}"). The per-run TextRun#substitute cannot
         # match those, but this can, because it joins the runs first.
         #
-        # The matched region is collapsed into the first run it touches, so that
-        # run's formatting is kept while the other spanned runs are emptied; runs
-        # outside the match are left untouched.
+        # The substitution works on the paragraph's w:t nodes, so it also handles
+        # runs holding several w:t (LibreOffice writes "text<w:br/>text" in one
+        # run) and placeholders that start inside a hyperlink. The replacement
+        # goes into the w:t holding the first matched character, so that run's
+        # formatting is used; the rest of the match is cut from the following
+        # w:t, and line breaks and tabs inside the match are dropped. Text,
+        # breaks and tabs outside the match stay where they are.
         #
         # +pattern+ may be a String or a Regexp; +replacement+ follows String#sub
         # semantics, so capture-group backreferences (e.g. '\1') work with a Regexp.
@@ -81,36 +85,41 @@ def each_text_run
         #   paragraph.substitute('{{first_name}}', 'Jane')   # => "Hello Jane!"
         #   paragraph.substitute(/\{\{(\w+)\}\}/, 'value of \1')
         #
-        # See https://github.com/ruby-docx/docx/issues/147
+        # See https://github.com/ruby-docx/docx/issues/147 and #187
         def substitute(pattern, replacement)
           search_from = 0
           loop do
-            runs = text_runs
-            break if runs.empty?
+            texts = @node.xpath(TEXT_XPATH).map { |t_node| Elements::Text.new(t_node) }
 
             offsets = []
             cursor = 0
-            runs.each do |run|
+            texts.each do |text|
               offsets << cursor
-              cursor += run.text.length
+              cursor += text.content.length
             end
-            full_text = runs.map(&:text).join
+            full_text = texts.map(&:content).join
 
             match = full_text.match(pattern, search_from)
             break unless match
             break if match.end(0) == match.begin(0) # ignore empty matches
 
             match_start = match.begin(0)
             match_end   = match.end(0) # exclusive
-            first = offsets.rindex { |offset| offset <= match_start }
-            last  = offsets.rindex { |offset| offset < match_end }
-
-            combined = runs[first..last].map(&:text).join
-            local_start = match_start - offsets[first]
-            local_end   = match_end - offsets[first]
-            replaced = combined[local_start...local_end].sub(pattern, replacement)
-            runs[first].text = combined[0...local_start] + replaced + combined[local_end..-1]
-            ((first + 1)..last).each { |index| runs[index].text = '' }
+            # the w:t holding the first and the last matched character
+            first = texts.each_index.find { |i| offsets[i] + texts[i].content.length > match_start }
+            last  = texts.each_index.find { |i| offsets[i] + texts[i].content.length >= match_end }
+
+            replaced = match[0].sub(pattern, replacement)
+            head = texts[first].content[0...(match_start - offsets[first])]
+            tail = texts[last].content[(match_end - offsets[last])..]
+            if first == last
+              texts[first].content = head + replaced + tail
+            else
+              texts[first].content = head + replaced
+              texts[(first + 1)...last].each { |text| text.content = '' }
+              texts[last].content = tail
+              remove_separators_between(texts[first].node, texts[last].node)
+            end
 
             # advance past the inserted replacement so it is not re-matched
             search_from = match_start + replaced.length
@@ -165,6 +174,22 @@ def style=(identifier)
 
         private
 
+        # The w:t read by #text (those of the runs and hyperlink runs), in
+        # document order.
+        TEXT_XPATH = 'w:r/w:t|w:hyperlink/w:r/w:t'
+        # Line breaks and tabs that sit between those w:t.
+        SEPARATOR_XPATH = %w[br tab cr].map { |tag| "w:r/w:#{tag}|w:hyperlink/w:r/w:#{tag}" }.join('|')
+        private_constant :TEXT_XPATH, :SEPARATOR_XPATH
+
+        # Remove the line breaks and tabs lying between two w:t of a match.
+        def remove_separators_between(first_t, last_t)
+          separators = @node.xpath(SEPARATOR_XPATH).to_a
+          nodes = @node.xpath("#{TEXT_XPATH}|#{SEPARATOR_XPATH}").to_a
+          from = nodes.index(first_t)
+          to = nodes.index(last_t)
+          nodes[(from + 1)...to].each { |node| node.remove if separators.include?(node) }
+        end
+
         def style_property
           properties&.at_xpath('w:pStyle') || properties&.add_child('<w:pStyle/>').first
         end

spec/docx/containers/paragraph_spec.rb

@@ -140,6 +140,84 @@ def paragraph(xml, document_properties = {}, document = nil)
     it 'returns self' do
       expect(split.substitute('{{first_name}}', 'World')).to equal(split)
     end
+
+    it 'keeps the text after the match in its own run and formatting' do
+      p = paragraph('<w:p><w:r><w:t>{{na</w:t></w:r><w:r><w:rPr><w:i/></w:rPr><w:t>me}} after</w:t></w:r></w:p>')
+
+      p.substitute('{{name}}', 'Jane')
+
+      expect(p.text_runs.map(&:text)).to eq(['Jane', ' after'])
+      expect(p.text_runs[1]).to be_italicized
+    end
+
+    # Regression tests for #187: files saved by LibreOffice or textutil keep
+    # several w:t in one run (around w:br / w:tab), and a hyperlink's runs are
+    # read as one TextRun. The substitution must work on the w:t nodes.
+    context 'with several w:t in a run or a hyperlink (#187)' do
+      # The paragraph content in document order, breaks and tabs included.
+      def layout(p)
+        p.node.xpath('.//w:t|.//w:br|.//w:tab').map { |n| n.name == 't' ? n.text : "<#{n.name}>" }
+      end
+
+      it 'replaces a placeholder sitting before a line break in the same run' do
+        p = paragraph('<w:p><w:r><w:t>Name: {{name}}</w:t><w:br/><w:t>Next line</w:t></w:r></w:p>')
+
+        p.substitute('{{name}}', 'Jane')
+
+        expect(layout(p)).to eq(['Name: Jane', '<br>', 'Next line'])
+      end
+
+      it 'replaces a split placeholder whose last run also holds a line break' do
+        p = paragraph(<<~XML)
+          <w:p><w:r><w:t xml:space="preserve">Name: {{na</w:t></w:r><w:r><w:t>me}}</w:t><w:br/><w:t>Next line</w:t></w:r></w:p>
+        XML
+
+        p.substitute('{{name}}', 'Jane')
+
+        expect(p.text).to eq('Name: JaneNext line')
+        expect(layout(p)).to eq(['Name: Jane', '', '<br>', 'Next line'])
+      end
+
+      it 'drops a tab that lies inside the placeholder' do
+        p = paragraph(<<~XML)
+          <w:p><w:r><w:t xml:space="preserve">A {{n</w:t></w:r><w:r><w:t>a</w:t><w:tab/><w:t>m</w:t></w:r><w:r><w:t xml:space="preserve">e}} B</w:t></w:r></w:p>
+        XML
+
+        p.substitute('{{name}}', 'Jane')
+
+        expect(p.text).to eq('A Jane B')
+        expect(layout(p)).not_to include('<tab>')
+      end
+
+      it 'keeps breaks and tabs outside the placeholder' do
+        p = paragraph('<w:p><w:r><w:t>a</w:t><w:tab/><w:t>{{na</w:t></w:r><w:r><w:t>me}}</w:t><w:br/><w:t>b</w:t></w:r></w:p>')
+
+        p.substitute('{{name}}', 'Jane')
+
+        expect(layout(p)).to eq(['a', '<tab>', 'Jane', '', '<br>', 'b'])
+      end
+
+      it 'replaces a placeholder that starts inside a hyperlink' do
+        p = paragraph(<<~XML)
+          <w:p><w:r><w:t xml:space="preserve">See </w:t></w:r><w:hyperlink w:anchor="x"><w:r><w:t xml:space="preserve">site </w:t></w:r><w:r><w:t>{{na</w:t></w:r></w:hyperlink><w:r><w:t xml:space="preserve">me}} end</w:t></w:r></w:p>
+        XML
+
+        p.substitute('{{name}}', 'Jane')
+
+        expect(p.text).to eq('See site Jane end')
+      end
+
+      it 'keeps the space after the placeholder through a save' do
+        p = paragraph('<w:p><w:r><w:t>{{na</w:t></w:r><w:r><w:t xml:space="preserve">me}} </w:t></w:r><w:r><w:t>end</w:t></w:r></w:p>')
+
+        p.substitute('{{name}}', 'Jane')
+
+        expect(p.text).to eq('Jane end')
+        p.node.xpath('.//w:t').select { |t| t.text.match?(/\A\s|\s\z/) }.each do |t|
+          expect(t.to_xml).to include('xml:space="preserve"')
+        end
+      end
+    end
   end
 
   describe 'alignment' do