fix(generator): escape multi-line braces and backtick unknown doc tags in yard formatting

gapic-generator/lib/gapic/formatting_utils.rb

@@ -21,16 +21,23 @@ module Gapic
   # Various string formatting utils
   #
   module FormattingUtils
-    @brace_detector = /\A(?<pre>[^`]*(?:`[^`]*`[^`]*)*[^`\\])?\{(?<inside>[^\s][^}]*)\}(?<post>.*)\z/m
     @xref_detector = /\A(?<pre>[^`]*(?:`[^`]*`[^`]*)*)?\[(?<text>[\w. `-]+)\]\[(?<addr>[\w.]+)\](?<post>.*)\z/m
     @list_element_detector = /\A\s*(?:\*|\+|-|[0-9a-zA-Z]+\.)\s/
     @omit_lines = ["@InputOnly\n", "@OutputOnly\n"]
+    # Built-in YARD meta-data tags as documented in:
+    # https://rubydoc.info/gems/yard/file/docs/Tags.md#Tag_List
+    @known_yard_tags = [
+      "abstract", "api", "attr", "attr_reader", "attr_writer", "author", "deprecated", "example",
+      "note", "option", "overload", "param", "private", "raise", "return", "see", "since", "todo",
+      "version", "yield", "yieldparam", "yieldreturn"
+    ].freeze
 
     class << self
       ##
       # Given an enumerable of lines, performs yardoc formatting, including:
       # * Interpreting cross-references identified as described in AIP 192
       # * Escaping literal braces that look like yardoc type links
+      # * Backticking unknown doc tags so they are not parsed as YARD tags
       #
       # Tries to be smart about exempting preformatted text blocks.
       #
@@ -45,23 +52,27 @@ class << self
       #
       def format_doc_lines api, lines, disable_xrefs: false, transport: nil
         transport ||= api&.default_transport || :grpc
-        # To detect preformatted blocks, this tracks the "expected" base indent
-        # according to Markdown. Specifically, this is the effective indent of
-        # previous block, which is normally 0 except if we're in a list item.
-        # Then, if a block is indented at least 4 spaces past that expected
-        # indent (and as long as it remains so), those lines are considered
-        # preformatted.
+        # Tracks fenced blocks, multiline inline code spans, and indented code blocks.
+        in_fence = false
+        in_code_span = false
         in_block = nil
         base_indent = 0
         (lines - @omit_lines).map do |line|
-          indent = line_indent line
-          if indent.nil?
+          if line =~ /^\s*(?:```|~~~)/
+            in_fence = !in_fence
+            in_code_span = false
             in_block = nil
-          else
-            in_block, base_indent = update_indent_state in_block, base_indent, line, indent
-            if in_block == false
-              line = escape_line_braces line
-              line = format_line_xrefs api, line, disable_xrefs, transport
+          elsif !in_fence
+            indent = line_indent line
+            if indent.nil?
+              in_block = nil
+              in_code_span = false
+            else
+              in_block, base_indent = update_indent_state in_block, base_indent, line, indent
+              if in_block == false
+                line, in_code_span = format_line_content line, in_code_span
+                line = format_line_xrefs api, line, disable_xrefs, transport
+              end
             end
           end
           line
@@ -106,11 +117,40 @@ def line_indent line
         m[1].length
       end
 
-      def escape_line_braces line
-        while (m = @brace_detector.match line)
-          line = "#{m[:pre]}\\\\{#{m[:inside]}}#{m[:post]}"
+      def format_line_content line, in_code_span
+        parts = line.split("`", -1)
+        formatted_parts = parts.each_with_index.map do |part, idx|
+          if in_code_span
+            in_code_span = false if idx < parts.length - 1
+            part
+          else
+            is_followed_by_backtick = idx < parts.length - 1
+            in_code_span = true if is_followed_by_backtick
+            formatted = escape_prose_braces part, is_followed_by_backtick: is_followed_by_backtick
+            sanitize_prose_tags formatted
+          end
+        end
+        [formatted_parts.join("`"), in_code_span]
+      end
+
+      def escape_prose_braces text, is_followed_by_backtick: false
+        # Matches unescaped `{` outside backtick spans followed by non-whitespace.
+        # If `{` is at the end of a non-code chunk (is_followed_by_backtick: true), it is followed
+        # immediately by a backticked code span (starting with a non-whitespace backtick),
+        # so \z (end of string) is also matched.
+        pattern = is_followed_by_backtick ? /(?<!\\)\{(?=[^\s]|\z)/ : /(?<!\\)\{(?=[^\s])/
+        text.gsub(pattern) { "\\\\{" }
+      end
+
+      def sanitize_prose_tags text
+        # Matches doc tags starting with `@` at the start of a line or preceded by whitespace.
+        # Avoids matching `@` within email addresses (e.g. user@example.com) or quotes.
+        # Any tag not in the YARD recognized list (or starting with `!`) is wrapped in backticks
+        # so YARD renders it as literal text rather than an unrecognized tag directive.
+        text.gsub(/(?<=\A|\s)@([a-zA-Z_]\w*)/) do |match|
+          tag = Regexp.last_match 1
+          @known_yard_tags.include?(tag) || tag.start_with?("!") ? match : "`#{match}`"
         end
-        line
       end
 
       def format_line_xrefs api, line, disable_xrefs, transport

gapic-generator/test/gapic/formatting_utils_test.rb

@@ -45,7 +45,7 @@ def test_escape_braces_backtick_brace_line
 
   def test_escape_braces_unmatched_brace_line
     result = Gapic::FormattingUtils.format_doc_lines nil, ["hello {ruby world\n"]
-    assert_equal ["hello {ruby world\n"], result
+    assert_equal ["hello \\\\{ruby world\n"], result
   end
 
   def test_escape_braces_escaped_brace_line
@@ -528,4 +528,199 @@ def test_format_number_negative_large_float
     str = Gapic::FormattingUtils.format_number(-1_234_567.89)
     assert_equal "-1_234_567.89", str
   end
+
+  def test_escape_braces_multiline_unmatched
+    result = Gapic::FormattingUtils.format_doc_lines nil, [
+      "Formatted as an array of inclusive ranges {min: min-value, max:\n",
+      "max-value}. For example, [{min: 123, max: 123}, {min: 64512, max: 65534}]\n"
+    ]
+    assert_equal [
+      "Formatted as an array of inclusive ranges \\\\{min: min-value, max:\n",
+      "max-value}. For example, [\\\\{min: 123, max: 123}, \\\\{min: 64512, max: 65534}]\n"
+    ], result
+  end
+
+  def test_escape_braces_multiline_unmatched_json
+    result = Gapic::FormattingUtils.format_doc_lines nil, [
+      "port number. Named ports can also contain multiple ports. " \
+      "For example:[{name: \"app1\", port: 8080}, {name:\n",
+      "\"app1\", port: 8081}, {name: \"app2\", port:\n",
+      "8082}]\n"
+    ]
+    assert_equal [
+      "port number. Named ports can also contain multiple ports. " \
+      "For example:[\\\\{name: \"app1\", port: 8080}, \\\\{name:\n",
+      "\"app1\", port: 8081}, \\\\{name: \"app2\", port:\n",
+      "8082}]\n"
+    ], result
+  end
+
+  def test_sanitize_unknown_tags
+    result = Gapic::FormattingUtils.format_doc_lines nil, [
+      "@pattern: \\d+(?:-\\d+)?\n",
+      "@required compute.instancegroups.addInstances\n",
+      "RFC1035 @pattern [a-z](?:[-a-z0-9]\\{0,61}[a-z0-9])?\n"
+    ]
+    assert_equal [
+      "`@pattern`: \\d+(?:-\\d+)?\n",
+      "`@required` compute.instancegroups.addInstances\n",
+      "RFC1035 `@pattern` [a-z](?:[-a-z0-9]\\{0,61}[a-z0-9])?\n"
+    ], result
+  end
+
+  def test_dont_sanitize_known_yard_tags
+    result = Gapic::FormattingUtils.format_doc_lines nil, [
+      "@param foo [String]\n",
+      "@return [Integer]\n",
+      "@deprecated Do not use\n",
+      "@see http://example.com\n",
+      "@attr [String] name description\n",
+      "@attr_reader [String] name description\n",
+      "@attr_writer [String] name description\n",
+      "@!attribute [rw] foo\n"
+    ]
+    assert_equal [
+      "@param foo [String]\n",
+      "@return [Integer]\n",
+      "@deprecated Do not use\n",
+      "@see http://example.com\n",
+      "@attr [String] name description\n",
+      "@attr_reader [String] name description\n",
+      "@attr_writer [String] name description\n",
+      "@!attribute [rw] foo\n"
+    ], result
+  end
+
+  def test_dont_sanitize_email_addresses
+    result = Gapic::FormattingUtils.format_doc_lines nil, [
+      "Contact support@example.com for help\n"
+    ]
+    assert_equal [
+      "Contact support@example.com for help\n"
+    ], result
+  end
+
+  def test_dont_sanitize_already_backticked_tags
+    result = Gapic::FormattingUtils.format_doc_lines nil, [
+      "Use `@pattern` to specify format\n"
+    ]
+    assert_equal [
+      "Use `@pattern` to specify format\n"
+    ], result
+  end
+
+  def test_escape_braces_followed_by_backtick
+    result = Gapic::FormattingUtils.format_doc_lines nil, [
+      "must be one of {`training`, `validation`, `test`}, and it defines\n"
+    ]
+    assert_equal [
+      "must be one of \\\\{`training`, `validation`, `test`}, and it defines\n"
+    ], result
+  end
+
+  def test_fenced_code_block_preserves_braces_and_tags
+    lines = [
+      "For example, the following JSON creates a divider:\n",
+      "\n",
+      "```\n",
+      "\"divider\": {}\n",
+      "Hello @FooBot how are you!\n",
+      "```\n",
+      "\n",
+      "Choose from {100, 200, 300}.\n"
+    ]
+    result = Gapic::FormattingUtils.format_doc_lines nil, lines
+    assert_equal [
+      "For example, the following JSON creates a divider:\n",
+      "\n",
+      "```\n",
+      "\"divider\": {}\n",
+      "Hello @FooBot how are you!\n",
+      "```\n",
+      "\n",
+      "Choose from \\\\{100, 200, 300}.\n"
+    ], result
+  end
+
+  def test_fenced_code_block_with_language_tag
+    lines = [
+      "Example with language:\n",
+      "```json\n",
+      "{\"name\": \"app\", \"ports\": [{8080}]}\n",
+      "```\n"
+    ]
+    result = Gapic::FormattingUtils.format_doc_lines nil, lines
+    assert_equal lines, result
+  end
+
+  def test_fenced_code_block_with_tilde
+    lines = [
+      "Example with tilde:\n",
+      "~~~\n",
+      "{\"name\": \"app\", \"ports\": [{8080}]}\n",
+      "~~~\n"
+    ]
+    result = Gapic::FormattingUtils.format_doc_lines nil, lines
+    assert_equal lines, result
+  end
+
+  def test_multiline_inline_code_span_preserves_braces
+    lines = [
+      "Value format:\n",
+      "`projects/{project}/locations/{location}/featurestores/\n",
+      "{featurestore}/entityTypes/{entityType}`. For example,\n",
+      "choose from {100, 200}.\n"
+    ]
+    result = Gapic::FormattingUtils.format_doc_lines nil, lines
+    assert_equal [
+      "Value format:\n",
+      "`projects/{project}/locations/{location}/featurestores/\n",
+      "{featurestore}/entityTypes/{entityType}`. For example,\n",
+      "choose from \\\\{100, 200}.\n"
+    ], result
+  end
+
+  def test_multiline_inline_code_span_preserves_tags
+    lines = [
+      "Here is an example:\n",
+      "`Hello @FooBot\n",
+      "@BarBot` in code\n",
+      "@FooBot outside code\n"
+    ]
+    result = Gapic::FormattingUtils.format_doc_lines nil, lines
+    assert_equal [
+      "Here is an example:\n",
+      "`Hello @FooBot\n",
+      "@BarBot` in code\n",
+      "`@FooBot` outside code\n"
+    ], result
+  end
+
+  def test_multiline_inline_code_span_three_lines
+    lines = [
+      "Start `code line 1 {foo}\n",
+      "code line 2 {bar}\n",
+      "code line 3 {baz}` end {qux}\n"
+    ]
+    result = Gapic::FormattingUtils.format_doc_lines nil, lines
+    assert_equal [
+      "Start `code line 1 {foo}\n",
+      "code line 2 {bar}\n",
+      "code line 3 {baz}` end \\\\{qux}\n"
+    ], result
+  end
+
+  def test_multiline_code_span_resets_on_blank_line
+    lines = [
+      "Unclosed `code span\n",
+      "\n",
+      "New paragraph with {100, 200}\n"
+    ]
+    result = Gapic::FormattingUtils.format_doc_lines nil, lines
+    assert_equal [
+      "Unclosed `code span\n",
+      "\n",
+      "New paragraph with \\\\{100, 200}\n"
+    ], result
+  end
 end