[feature](inverted-index) Add Japanese (Kuromoji) morphological analyzer

.clang-format-ignore

@@ -9,4 +9,5 @@ be/src/util/sse2neon.h
 be/src/util/mustache/mustache.h
 be/src/util/mustache/mustache.cc
 be/src/util/utf8_check.cpp
+be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h
 cloud/src/common/defer.h

.gitignore

@@ -150,6 +150,9 @@ docker/runtime/be/resource/apache-doris/
 compile_commands.json
 .github
 
+# generated kuromoji dictionary binaries
+/be/dict/kuromoji/*.bin
+
 .worktrees/
 .worktree_initialized
 

.licenserc.yaml

@@ -112,6 +112,7 @@ header:
     - "be/src/util/sse2neo.h"
     - "be/src/util/sse2neon.h"
     - "be/src/util/utf8_check.cpp"
+    - "be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h"
     - "be/src/pch/*"
     - "be/test/data"
     - "be/test/expected_result"

NOTICE.txt

@@ -73,6 +73,9 @@ This software includes third party software subject to the following copyrights:
 - Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams
 - Jackson-core - https://github.com/FasterXML/jackson-core
 - Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary
+- Darts-clone (double-array trie) - Copyright 2008-2014 Susumu Yata - https://github.com/s-yata/darts-clone (BSD 2-clause; see dist/licenses/LICENSE-darts-clone.txt)
+- mecab-ipadic (IPADIC) Japanese morphological dictionary - Copyright 2000-2003 Nara Institute of Science and Technology (NAIST) - licensed under NAIST-2003 (BSD-style); the kuromoji analyzer bundles the UTF-8 form from https://github.com/lindera/mecab-ipadic (content of mecab-ipadic-2.7.0-20070801). See dist/licenses/LICENSE-ipadic.txt.
+- Apache Lucene - https://github.com/apache/lucene (Apache-2.0): the kuromoji Japanese analyzer under be/src/storage/index/inverted/analyzer/kuromoji is an independent C++ implementation modeled on Lucene's kuromoji analyzer (JapaneseTokenizer), including its search-mode compound-decomposition cost model.
 
 The licenses for these third party components are included in LICENSE.txt
 

be/CMakeLists.txt

@@ -352,6 +352,26 @@ install(DIRECTORY
     ${BASE_DIR}/dict/pinyin
     DESTINATION ${OUTPUT_DIR}/dict)
 
+# Japanese kuromoji dictionary. A real build always requires it the
+# binary files. However, Unit-test builds (MAKE_TEST=ON) skip it entirely as they don't
+# package for real use.
+if (NOT MAKE_TEST)
+    install(CODE "
+        foreach(_kmj_f
+                \"${BASE_DIR}/dict/kuromoji/system.bin\"
+                \"${BASE_DIR}/dict/kuromoji/matrix.bin\"
+                \"${BASE_DIR}/dict/kuromoji/chardef.bin\"
+                \"${BASE_DIR}/dict/kuromoji/unkdict.bin\")
+            if(NOT EXISTS \"\${_kmj_f}\")
+                message(FATAL_ERROR \"kuromoji dictionary file \${_kmj_f} is missing; build the 'kuromoji_dict' target (stage mecab-ipadic via thirdparty) before packaging.\")
+            endif()
+        endforeach()
+    ")
+    install(DIRECTORY
+        ${BASE_DIR}/dict/kuromoji
+        DESTINATION ${OUTPUT_DIR}/dict)
+endif()
+
 # Check if functions are supported in this platform. All flags will generated
 # in gensrc/build/common/env_config.h.
 # You can check funcion here which depends on platform. Don't forget add this
@@ -1071,6 +1091,49 @@ if (BUILD_META_TOOL OR BUILD_INDEX_TOOL)
     add_subdirectory(${SRC_DIR}/tools)
 endif()
 
+if (NOT MAKE_TEST)
+    # Offline generator: compiles the UTF-8 mecab-ipadic source into binary files.
+    add_executable(kuromoji_build_dict EXCLUDE_FROM_ALL ${SRC_DIR}/tools/kuromoji_build_dict.cpp)
+    target_include_directories(kuromoji_build_dict PRIVATE ${PROJECT_SOURCE_DIR}/..)
+    pch_reuse(kuromoji_build_dict)
+    set_target_properties(kuromoji_build_dict PROPERTIES ENABLE_EXPORTS 1)
+    if (COMPILER_CLANG)
+        target_compile_options(kuromoji_build_dict PRIVATE
+            -Wno-implicit-int-conversion
+            -Wno-shorten-64-to-32)
+    endif()
+    target_link_libraries(kuromoji_build_dict ${DORIS_LINK_LIBS})
+
+    set(KUROMOJI_IPADIC_SRC "${THIRDPARTY_DIR}/share/mecab-ipadic-2.7.0-20250920"
+        CACHE PATH "UTF-8 mecab-ipadic source directory used to generate the kuromoji dictionary")
+    set(KUROMOJI_DICT_OUT "${BASE_DIR}/dict/kuromoji")
+    file(GLOB KUROMOJI_IPADIC_SRC_FILES CONFIGURE_DEPENDS
+         "${KUROMOJI_IPADIC_SRC}/*.csv"
+         "${KUROMOJI_IPADIC_SRC}/*.def")
+    get_filename_component(KUROMOJI_LIBJVM_DIR "${LIB_JVM}" DIRECTORY)
+    set(KUROMOJI_DYLD_PATH "${KUROMOJI_LIBJVM_DIR}")
+    set(KUROMOJI_LD_PATH "${KUROMOJI_LIBJVM_DIR}")
+    if (NOT "$ENV{DYLD_LIBRARY_PATH}" STREQUAL "")
+        set(KUROMOJI_DYLD_PATH "${KUROMOJI_LIBJVM_DIR}:$ENV{DYLD_LIBRARY_PATH}")
+    endif()
+    if (NOT "$ENV{LD_LIBRARY_PATH}" STREQUAL "")
+        set(KUROMOJI_LD_PATH "${KUROMOJI_LIBJVM_DIR}:$ENV{LD_LIBRARY_PATH}")
+    endif()
+    add_custom_command(
+        OUTPUT "${KUROMOJI_DICT_OUT}/system.bin" "${KUROMOJI_DICT_OUT}/matrix.bin"
+               "${KUROMOJI_DICT_OUT}/chardef.bin" "${KUROMOJI_DICT_OUT}/unkdict.bin"
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${KUROMOJI_DICT_OUT}"
+        COMMAND ${CMAKE_COMMAND} -E env
+                "DYLD_LIBRARY_PATH=${KUROMOJI_DYLD_PATH}"
+                "LD_LIBRARY_PATH=${KUROMOJI_LD_PATH}"
+                $<TARGET_FILE:kuromoji_build_dict> "${KUROMOJI_IPADIC_SRC}" "${KUROMOJI_DICT_OUT}"
+        DEPENDS kuromoji_build_dict ${KUROMOJI_IPADIC_SRC_FILES}
+        COMMENT "Generating kuromoji IPADIC dictionary from ${KUROMOJI_IPADIC_SRC}"
+        VERBATIM)
+    # ALL so a real build generates the dictionary before the install step runs.
+    add_custom_target(kuromoji_dict ALL DEPENDS "${KUROMOJI_DICT_OUT}/system.bin")
+endif()
+
 option(BUILD_FILE_CACHE_MICROBENCH_TOOL "Build file cache mirobench Tool" OFF)
 if (BUILD_FILE_CACHE_MICROBENCH_TOOL)
     add_subdirectory(${SRC_DIR}/io/tools)

be/dict/kuromoji/README.md

@@ -0,0 +1,60 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Kuromoji (Japanese) dictionary
+
+This directory holds the compiled IPADIC dictionary consumed at runtime by the
+`kuromoji` inverted-index analyzer (`KuromojiAnalyzer` → `KuromojiDictionary`):
+
+- `system.bin`  — surface→word Darts trie + word entries + feature blob
+- `matrix.bin`  — connection-cost matrix (1316×1316)
+- `chardef.bin` — character-category map + per-category flags
+- `unkdict.bin` — unknown-word entries per category
+
+These `*.bin` files are **generated** (not committed; see `.gitignore`). The
+runtime resolves them at `${inverted_index_dict_path}/kuromoji`
+(default `${DORIS_HOME}/dict/kuromoji`); `be/CMakeLists.txt` installs this
+directory into the BE package.
+
+## How it's (re)generated
+
+Source: the UTF-8 IPADIC from <https://github.com/lindera/mecab-ipadic>
+(tag `2.7.0-20250920`) — the original `mecab-ipadic-2.7.0-20070801` lexicon
+converted to UTF-8 (license: NAIST-2003, see `dist/licenses/LICENSE-ipadic.txt`).
+
+A normal BE build (`sh build.sh`) generates these `*.bin` automatically: the
+`kuromoji_dict` target is part of `ALL` and the `install` rule then ships this
+directory. The target is defined only for real (`MAKE_TEST=OFF`) builds, not for
+the unit-test tree.
+
+To regenerate manually:
+
+```bash
+# 1. thirdparty fetches + stages the UTF-8 IPADIC source into
+#    ${DORIS_THIRDPARTY}/installed/share/mecab-ipadic-2.7.0-20250920
+sh thirdparty/build-thirdparty.sh mecab_ipadic
+
+# 2. run the target in a real (non-test) build tree, e.g. the one sh build.sh
+#    creates under be/build_<BUILD_TYPE> (build_Release by default)
+ninja -C be/build_Release kuromoji_dict
+```
+
+Override the source dir with `-DKUROMOJI_IPADIC_SRC=<path>` at CMake configure
+time. (The tool can also be run directly:
+`kuromoji_build_dict <utf8_ipadic_src_dir> be/dict/kuromoji`.)

be/src/common/config.cpp

@@ -1416,6 +1416,8 @@ DEFINE_mInt64(snii_forced_spill_min_arena_bytes, "67108864");
 DEFINE_mInt32(snii_spill_max_run_files_per_buffer, "64");
 // dict path for chinese analyzer
 DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict");
+// The kuromoji (Japanese) analyzer
+DEFINE_mBool(enable_kuromoji_analyzer, "false");
 DEFINE_Int32(inverted_index_read_buffer_size, "4096");
 // tree depth for bkd index
 DEFINE_Int32(max_depth_in_bkd_tree, "32");

be/src/common/config.h

@@ -1456,6 +1456,8 @@ DECLARE_mInt64(snii_forced_spill_min_arena_bytes);
 DECLARE_mInt32(snii_spill_max_run_files_per_buffer);
 // dict path for chinese analyzer
 DECLARE_String(inverted_index_dict_path);
+// The kuromoji (Japanese) analyzer
+DECLARE_mBool(enable_kuromoji_analyzer);
 DECLARE_Int32(inverted_index_read_buffer_size);
 // tree depth for bkd index
 DECLARE_Int32(max_depth_in_bkd_tree);

be/src/storage/index/inverted/analyzer/analyzer.cpp

@@ -39,6 +39,7 @@
 #include "storage/index/inverted/analyzer/basic/basic_analyzer.h"
 #include "storage/index/inverted/analyzer/icu/icu_analyzer.h"
 #include "storage/index/inverted/analyzer/ik/IKAnalyzer.h"
+#include "storage/index/inverted/analyzer/kuromoji/KuromojiAnalyzer.h"
 #include "storage/index/inverted/char_filter/char_replace_char_filter_factory.h"
 #include "storage/index/inverted/token_filter/common_grams_filter.h"
 
@@ -88,7 +89,8 @@ bool InvertedIndexAnalyzer::is_builtin_analyzer(const std::string& analyzer_name
            analyzer_name == INVERTED_INDEX_PARSER_CHINESE ||
            analyzer_name == INVERTED_INDEX_PARSER_ICU ||
            analyzer_name == INVERTED_INDEX_PARSER_BASIC ||
-           analyzer_name == INVERTED_INDEX_PARSER_IK;
+           analyzer_name == INVERTED_INDEX_PARSER_IK ||
+           analyzer_name == INVERTED_INDEX_PARSER_KUROMOJI;
 }
 
 AnalyzerPtr InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserType parser_type,
@@ -126,6 +128,22 @@ AnalyzerPtr InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserTy
             ik_analyzer->setMode(false);
         }
         analyzer = std::move(ik_analyzer);
+    } else if (parser_type == InvertedIndexParserType::PARSER_KUROMOJI) {
+        if (!config::enable_kuromoji_analyzer) {
+            throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                            "kuromoji analyzer is disabled by default. Set "
+                            "enable_kuromoji_analyzer=true in "
+                            "be.conf (or via the BE config HTTP API) to enable it.");
+        }
+
+        std::string kuromoji_mode = parser_mode;
+        if (kuromoji_mode.empty() || kuromoji_mode == INVERTED_INDEX_PARSER_COARSE_GRANULARITY) {
+            kuromoji_mode = INVERTED_INDEX_PARSER_KUROMOJI_SEARCH;
+        }
+        auto kuromoji_analyzer = std::make_shared<KuromojiAnalyzer>();
+        kuromoji_analyzer->setMode(kuromoji_mode_from_string(kuromoji_mode));
+        kuromoji_analyzer->initDict(config::inverted_index_dict_path + "/kuromoji");
+        analyzer = std::move(kuromoji_analyzer);
     } else {
         // default
         analyzer = std::make_shared<lucene::analysis::SimpleAnalyzer<char>>();

be/src/storage/index/inverted/analyzer/kuromoji/KuromojiAnalyzer.h

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analyzer/kuromoji/KuromojiTokenizer.h"
+#include "storage/index/inverted/analyzer/kuromoji/dict/kuromoji_dictionary.h"
+
+namespace doris::segment_v2 {
+
+class KuromojiAnalyzer : public Analyzer {
+public:
+    KuromojiAnalyzer() {
+        _lowercase = true;
+        _ownReader = false;
+    }
+    ~KuromojiAnalyzer() override = default;
+
+    bool isSDocOpt() override { return true; }
+
+    // Loads (once, process-wide) the IPADIC dictionary from `dictPath`.
+    void initDict(const std::string& dictPath) override {
+        dict_ = inverted_index::kuromoji::KuromojiDictionary::get_or_load(dictPath);
+        if (dict_ == nullptr) {
+            throw doris::Exception(
+                    doris::ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                    "kuromoji dictionary could not be loaded from {}; ensure system.bin, "
+                    "matrix.bin, chardef.bin and unkdict.bin are present in the BE package",
+                    dictPath);
+        }
+    }
+
+    void setMode(KuromojiMode mode) { mode_ = mode; }
+
+    TokenStream* tokenStream(const TCHAR* fieldName, lucene::util::Reader* reader) override {
+        auto* tokenizer = _CLNEW KuromojiTokenizer(mode_, _lowercase, _ownReader, dict_);
+        tokenizer->reset(reader);
+        return (TokenStream*)tokenizer;
+    }
+
+    TokenStream* reusableTokenStream(const TCHAR* fieldName,
+                                     lucene::util::Reader* reader) override {
+        if (tokenizer_ == nullptr) {
+            tokenizer_ = std::make_unique<KuromojiTokenizer>(mode_, _lowercase, _ownReader, dict_);
+        }
+        tokenizer_->reset(reader);
+        return (TokenStream*)tokenizer_.get();
+    }
+
+private:
+    const inverted_index::kuromoji::KuromojiDictionary* dict_ {nullptr};
+    KuromojiMode mode_ {KuromojiMode::Search};
+    std::unique_ptr<KuromojiTokenizer> tokenizer_;
+};
+
+} // namespace doris::segment_v2

be/src/storage/index/inverted/analyzer/kuromoji/KuromojiMode.h

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+
+#include "common/exception.h"
+
+namespace doris::segment_v2 {
+
+// Segmentation mode, mirroring Lucene's JapaneseTokenizer.Mode. Normal returns
+// the minimum-cost segmentation. Search additionally decomposes long compounds
+// into their shorter parts (via a length-based cost penalty) for better search
+// recall. Extended applies the Search penalty and also splits unknown
+// (out-of-vocabulary) words into per-character unigrams.
+enum class KuromojiMode { Normal, Search, Extended };
+
+inline KuromojiMode kuromoji_mode_from_string(const std::string& mode) {
+    if (mode.empty() || mode == "search") {
+        return KuromojiMode::Search;
+    }
+    if (mode == "normal") {
+        return KuromojiMode::Normal;
+    }
+    if (mode == "extended") {
+        return KuromojiMode::Extended;
+    }
+    throw doris::Exception(doris::ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+                           "Invalid kuromoji parser_mode: '{}', must be search, normal or extended",
+                           mode);
+}
+
+} // namespace doris::segment_v2