Optimize CEL PrattParser and Lexer performance

Improve parsing throughput and reduce memory allocations across CEL expressions.

Measured with CelParserBenchmark (parseOnly, built -c opt), comparing three
parsers back to back in one session: ANTLR, the Pratt parser before this change,
and the Pratt parser after it.

Objects allocated per parse:

| Case | ANTLR | Pratt before | Pratt after | Pratt vs ANTLR | Delta this CL |
| :--- | ---: | ---: | ---: | ---: | ---: |
| SMOKE_TEST | 357 | 131 | 123 | 2.9x smaller | -6.1% |
| CHAINED_ORS | 968 | 374 | 350 | 2.8x smaller | -6.4% |
| LIST_COMPREHENSION | 512 | 218 | 166 | 3.1x smaller | -23.9% |
| MESSAGE_CREATION | 1,253 | 502 | 427 | 2.9x smaller | -14.9% |
| LONG_LIST | 81,794 | 19,310 | 19,271 | 4.2x smaller | -0.2% |

Bytes allocated per parse:

| Case | ANTLR | Pratt before | Pratt after | Pratt vs ANTLR | Delta this CL |
| :--- | ---: | ---: | ---: | ---: | ---: |
| SMOKE_TEST | 12,256 | 3,776 | 3,608 | 3.4x smaller | -4.4% |
| CHAINED_ORS | 32,160 | 10,288 | 9,864 | 3.3x smaller | -4.1% |
| LIST_COMPREHENSION | 17,320 | 6,120 | 4,736 | 3.7x smaller | -22.6% |
| MESSAGE_CREATION | 43,128 | 13,888 | 12,240 | 3.5x smaller | -11.9% |
| LONG_LIST | 2,907,488 | 553,160 | 568,080 | 5.1x smaller | +2.7% |

Wall clock, mean of 3 caliper trial medians:

| Case | ANTLR | Pratt before | Pratt after | Pratt vs ANTLR | Delta this CL |
| :--- | ---: | ---: | ---: | ---: | ---: |
| SMOKE_TEST | 4,940 ns | 733 ns | 692 ns | 7.1x faster | -5.7% |
| CHAINED_ORS | 14,641 ns | 2,090 ns | 2,041 ns | 7.2x faster | -2.3% |
| LIST_COMPREHENSION | 7,514 ns | 1,640 ns | 1,218 ns | 6.2x faster | -25.7% |
| MESSAGE_CREATION | 20,979 ns | 3,962 ns | 3,487 ns | 6.0x faster | -12.0% |
| LONG_LIST | 1,616,631 ns | 144,980 ns | 142,583 ns | 11.3x faster | -1.7% |

LONG_LIST is the one case that allocates slightly more than before. It is an
extreme outlier (1,000 list elements, ~20k objects per parse) and the +2.7% comes
from letting the positions map grow from its default capacity instead of
presizing it; presizing cost more on every other case, so the tradeoff is worth
it. The map is removed entirely later in this series.

PiperOrigin-RevId: 980131070

Author: dmitriplotnikov

parser/src/main/java/dev/cel/parser/BUILD.bazel

@@ -118,6 +118,7 @@ java_library(
         "//common:source_location",
         "//common/ast",
         "//common/internal",
+        "@maven//:com_google_errorprone_error_prone_annotations",
         "@maven//:com_google_guava_guava",
         "@maven//:org_jspecify_jspecify",
     ],

parser/src/main/java/dev/cel/parser/Lexer.java

@@ -29,8 +29,6 @@ final class Lexer {
   enum TokenType {
     ERROR("error"),
     END("end"),
-    WHITESPACE("whitespace"),
-    COMMENT("comment"),
 
     // Keywords
     NULL("null"),
@@ -98,11 +96,17 @@ static final class Token {
     final TokenType type;
     final int start;
     final int end;
+    final @Nullable String text;
 
     Token(TokenType type, int start, int end) {
+      this(type, start, end, null);
+    }
+
+    Token(TokenType type, int start, int end, @Nullable String text) {
       this.type = type;
       this.start = start;
       this.end = end;
+      this.text = text;
     }
 
     @Override
@@ -149,35 +153,28 @@ static final class LexerError {
           .buildOrThrow();
 
   private final CelCodePointArray content;
+  private final int size;
   private int position;
   private LexerError error;
 
   Lexer(CelCodePointArray content) {
     this.content = content;
+    this.size = content.size();
     this.position = 0;
     this.error = null;
   }
 
   Token lex() {
+    consumeWhitespaceAndComments();
     int start = position;
-    if (position >= content.size()) {
+    if (position >= size) {
       return makeToken(TokenType.END, start, start);
     }
     int c = content.get(position);
     switch (c) {
-      case '\f':
-      case '\n':
-      case ' ':
-      case '\r':
-      case 0x0B: // \v (vertical tab)
-      case '\t':
-        {
-          consumeWhitespace();
-          return makeToken(TokenType.WHITESPACE, start, position);
-        }
       case '.':
         {
-          if (position + 1 < content.size() && isDigit(content.get(position + 1))) {
+          if (position + 1 < size && isDigit(content.get(position + 1))) {
             return consumeNumericLiteral();
           }
           advance(1);
@@ -283,10 +280,6 @@ Token lex() {
       case '/':
         {
           advance(1);
-          if (consume('/')) {
-            consumeLine();
-            return makeToken(TokenType.COMMENT, start, position);
-          }
           return makeToken(TokenType.SLASH, start, position);
         }
       case '&':
@@ -381,6 +374,10 @@ private Token makeToken(TokenType type, int start, int end) {
     return new Token(type, start, end);
   }
 
+  private Token makeToken(TokenType type, int start, int end, @Nullable String text) {
+    return new Token(type, start, end, text);
+  }
+
   private Token setError(int start, int end, String message) {
     this.error = new LexerError(start, end, message);
     return new Token(TokenType.ERROR, start, end);
@@ -391,7 +388,7 @@ private void advance(int n) {
   }
 
   private boolean match(int c) {
-    return position < content.size() && content.get(position) == c;
+    return position < size && content.get(position) == c;
   }
 
   private boolean consume(int c) {
@@ -403,7 +400,7 @@ private boolean consume(int c) {
   }
 
   private boolean consumeIf(IntPredicate predicate) {
-    if (position < content.size()) {
+    if (position < size) {
       int cp = content.get(position);
       if (predicate.test(cp)) {
         advance(1);
@@ -414,7 +411,7 @@ private boolean consumeIf(IntPredicate predicate) {
   }
 
   private void consumeLine() {
-    while (position < content.size()) {
+    while (position < size) {
       if (content.get(position) == '\n') {
         advance(1);
         return;
@@ -423,8 +420,8 @@ private void consumeLine() {
     }
   }
 
-  private void consumeWhitespace() {
-    while (position < content.size()) {
+  private void consumeWhitespaceAndComments() {
+    while (position < size) {
       int c = content.get(position);
       switch (c) {
         case '\f':
@@ -433,38 +430,35 @@ private void consumeWhitespace() {
         case '\r':
         case 11: // \v
         case '\t':
-          advance(1);
+          position++;
           break;
+        case '/':
+          if (position + 1 < size && content.get(position + 1) == '/') {
+            consumeLine();
+            break;
+          } else {
+            return;
+          }
         default:
           return;
       }
     }
   }
 
   private boolean consumeDigits() {
-    boolean advanced = false;
-    while (position < content.size()) {
-      int c = content.get(position);
-      if (!isDigit(c)) {
-        break;
-      }
-      advance(1);
-      advanced = true;
+    int start = position;
+    while (position < size && isDigit(content.get(position))) {
+      position++;
     }
-    return advanced;
+    return position > start;
   }
 
   private boolean consumeHexDigits() {
-    boolean advanced = false;
-    while (position < content.size()) {
-      int c = content.get(position);
-      if (!isHexDigit(c)) {
-        break;
-      }
-      advance(1);
-      advanced = true;
+    int start = position;
+    while (position < size && isHexDigit(content.get(position))) {
+      position++;
     }
-    return advanced;
+    return position > start;
   }
 
   private TokenType consumeIntegralSuffix() {
@@ -486,7 +480,7 @@ private Token consumeQuotedIdent() {
   private boolean consumeUntilAfter(int c, boolean isRaw) {
     int pos = position;
     boolean escaped = false;
-    while (pos < content.size()) {
+    while (pos < size) {
       int cc = content.get(pos);
       if (cc == '\n' || cc == '\r') {
         position = pos;
@@ -503,20 +497,20 @@ private boolean consumeUntilAfter(int c, boolean isRaw) {
       }
       pos++;
     }
-    position = content.size();
+    position = size;
     return false;
   }
 
   private boolean consumeUntilAfterTripleQuote(int quote, boolean isRaw) {
     int pos = position;
     boolean escaped = false;
-    while (pos < content.size()) {
+    while (pos < size) {
       int cc = content.get(pos);
       if (!isRaw && cc == '\\') {
         escaped = !escaped;
       } else {
         if ((isRaw || !escaped)
-            && pos + 2 < content.size()
+            && pos + 2 < size
             && cc == quote
             && content.get(pos + 1) == quote
             && content.get(pos + 2) == quote) {
@@ -527,16 +521,14 @@ private boolean consumeUntilAfterTripleQuote(int quote, boolean isRaw) {
       }
       pos++;
     }
-    position = content.size();
+    position = size;
     return false;
   }
 
   private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolean isRaw) {
     advance(1);
     boolean isTripleQuote =
-        position + 1 < content.size()
-            && content.get(position) == quote
-            && content.get(position + 1) == quote;
+        position + 1 < size && content.get(position) == quote && content.get(position + 1) == quote;
     if (isTripleQuote) {
       advance(2);
       if (!consumeUntilAfterTripleQuote(quote, isRaw)) {
@@ -556,7 +548,7 @@ private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolea
 
   private @Nullable Token consumePrefixedStringLiteral() {
     int start = position;
-    if (position >= content.size()) {
+    if (position >= size) {
       return null;
     }
     int c = content.get(position);
@@ -566,15 +558,15 @@ private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolea
       return null;
     }
     int lookahead = 1;
-    if (position + 1 < content.size()) {
+    if (position + 1 < size) {
       int c2 = content.get(position + 1);
       if (isBytes ? (c2 == 'r' || c2 == 'R') : (c2 == 'b' || c2 == 'B')) {
         isBytes = true;
         isRaw = true;
         lookahead = 2;
       }
     }
-    if (position + lookahead < content.size()) {
+    if (position + lookahead < size) {
       int quote = content.get(position + lookahead);
       if (quote == '"' || quote == '\'') {
         advance(lookahead);
@@ -612,9 +604,9 @@ private Token consumeNumericLiteral() {
         return makeToken(tokenType, start, position);
       }
       consumeDigits();
-      if (position < content.size()
+      if (position < size
           && content.get(position) == '.'
-          && position + 1 < content.size()
+          && position + 1 < size
           && isDigit(content.get(position + 1))) {
         floatingPoint = true;
         advance(1);
@@ -639,19 +631,15 @@ && isDigit(content.get(position + 1))) {
 
   private Token consumeIdent() {
     int start = position;
-    while (position < content.size()) {
-      int c = content.get(position);
-      if (!isIdentTrailing(c)) {
-        break;
-      }
-      advance(1);
+    while (position < size && isIdentTrailing(content.get(position))) {
+      position++;
     }
     int end = position;
     String word = content.slice(start, end).toString();
     TokenType keywordType = KEYWORDS.get(word);
     if (keywordType != null) {
       return makeToken(keywordType, start, end);
     }
-    return makeToken(TokenType.IDENT, start, end);
+    return makeToken(TokenType.IDENT, start, end, word);
   }
 }