From ef5bd65ee6d76afa56ba7fdc7dc337bb3cd09e43 Mon Sep 17 00:00:00 2001
From: Zijian Yi <zijianyi@google.com>
Date: Tue, 4 Aug 2026 09:03:32 -0700
Subject: [PATCH] [gemma_cpp] add tokenizer benchmark test

PiperOrigin-RevId: 959053992
---
 BUILD.bazel                               |  15 +++
 tokenizer/bpe_tokenizer_benchmark_test.cc | 116 ++++++++++++++++++++++
 2 files changed, 131 insertions(+)
 create mode 100644 tokenizer/bpe_tokenizer_benchmark_test.cc

diff --git a/BUILD.bazel b/BUILD.bazel
index cb93ac75..47730efc 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -244,6 +244,21 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "bpe_tokenizer_benchmark_test",
+    srcs = ["tokenizer/bpe_tokenizer_benchmark_test.cc"],
+    data = [
+        "testdata/frankenstein.txt",
+        "tokenizer/testdata/tokenizer.model",
+        ":packed_tokenizer",
+    ],
+    deps = [
+        ":tokenizer",
+        "//testing/base/public:gunit_main",  # buildcleaner: keep
+        "//io",
+    ],
+)
+
 cc_library(
     name = "tensor_info",
     srcs = [
diff --git a/tokenizer/bpe_tokenizer_benchmark_test.cc b/tokenizer/bpe_tokenizer_benchmark_test.cc
new file mode 100644
index 00000000..012a71d4
--- /dev/null
+++ b/tokenizer/bpe_tokenizer_benchmark_test.cc
@@ -0,0 +1,116 @@
+// Copyright 2024 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+#include "testing/base/public/benchmark.h"
+#include "gemma/tokenizer.h"
+#include "tokenizer/bpe_tokenizer.h"
+#include "io/io.h"
+
+namespace gcpp {
+namespace {
+
+constexpr const char* kTokenizerPacked =
+    "tokenizer/testdata/tokenizer.packed";
+constexpr const char* kTokenizerModel =
+    "tokenizer/testdata/tokenizer.model";
+constexpr const char* kCorpus =
+    "testdata/frankenstein.txt";
+
+std::string ReadFileOrAbort(const std::string& path) {
+  return ReadFileToString(Path(path));
+}
+
+void BM_LoadSentencePiece(benchmark::State& state) {
+  std::string blob = ReadFileOrAbort(kTokenizerModel);
+  state.SetLabel("BlobSize: " + std::to_string(blob.size()) + "B");
+  for (auto s : state) {
+    GemmaTokenizer sp(blob);
+    benchmark::DoNotOptimize(sp);
+  }
+}
+BENCHMARK(BM_LoadSentencePiece);
+
+void BM_LoadCompactBpe(benchmark::State& state) {
+  std::string blob = ReadFileOrAbort(kTokenizerPacked);
+  state.SetLabel("BlobSize: " + std::to_string(blob.size()) + "B");
+  for (auto s : state) {
+    GemmaTokenizer bpe(CreateBpeTokenizer(blob));
+    benchmark::DoNotOptimize(bpe);
+  }
+}
+BENCHMARK(BM_LoadCompactBpe);
+
+void BM_EncodeSentencePiece(benchmark::State& state) {
+  std::string corpus = ReadFileOrAbort(kCorpus);
+  GemmaTokenizer sp(ReadFileOrAbort(kTokenizerModel));
+  for (auto s : state) {
+    std::vector<int> ids;
+    sp.Encode(corpus, &ids);
+    benchmark::DoNotOptimize(ids);
+  }
+  state.SetBytesProcessed(state.iterations() * corpus.size());
+}
+BENCHMARK(BM_EncodeSentencePiece);
+
+void BM_EncodeCompactBpe(benchmark::State& state) {
+  std::string corpus = ReadFileOrAbort(kCorpus);
+  GemmaTokenizer bpe(CreateBpeTokenizer(ReadFileOrAbort(kTokenizerPacked)));
+  for (auto s : state) {
+    std::vector<int> ids;
+    bpe.Encode(corpus, &ids);
+    benchmark::DoNotOptimize(ids);
+  }
+  state.SetBytesProcessed(state.iterations() * corpus.size());
+}
+BENCHMARK(BM_EncodeCompactBpe);
+
+void BM_DecodeSentencePiece(benchmark::State& state) {
+  std::string corpus = ReadFileOrAbort(kCorpus);
+  GemmaTokenizer sp(ReadFileOrAbort(kTokenizerModel));
+  std::vector<int> ids;
+  sp.Encode(corpus, &ids);
+  size_t detokenized_size = 0;
+  for (auto s : state) {
+    std::string detokenized;
+    sp.Decode(ids, &detokenized);
+    benchmark::DoNotOptimize(detokenized);
+    detokenized_size = detokenized.size();
+  }
+  state.SetBytesProcessed(state.iterations() * detokenized_size);
+}
+BENCHMARK(BM_DecodeSentencePiece);
+
+void BM_DecodeCompactBpe(benchmark::State& state) {
+  std::string corpus = ReadFileOrAbort(kCorpus);
+  GemmaTokenizer bpe(CreateBpeTokenizer(ReadFileOrAbort(kTokenizerPacked)));
+  std::vector<int> ids;
+  bpe.Encode(corpus, &ids);
+  size_t detokenized_size = 0;
+  for (auto s : state) {
+    std::string detokenized;
+    bpe.Decode(ids, &detokenized);
+    benchmark::DoNotOptimize(detokenized);
+    detokenized_size = detokenized.size();
+  }
+  state.SetBytesProcessed(state.iterations() * detokenized_size);
+}
+BENCHMARK(BM_DecodeCompactBpe);
+
+}  // namespace
+}  // namespace gcpp