09_vector_retrievalGPU 向量检索引擎/张磊

09_vector_retrieval/张磊/.gitignore

@@ -0,0 +1,8 @@
+build/
+data/*.bin
+data/*.idx
+data/*.npy
+*.log
+test_index_tmp.idx
+test_fp16_tmp.bin
+test_fp32_tmp.bin

09_vector_retrieval/张磊/CMakeLists.txt

@@ -0,0 +1,63 @@
+cmake_minimum_required(VERSION 3.18)
+project(vsearch LANGUAGES CXX CUDA)
+
+# ---------------------------------------------------------------------------
+# GPU vector search engine (CUDA): exact baseline + IVF-Flat / IVF-PQ
+# ---------------------------------------------------------------------------
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CUDA_STANDARD 17)
+set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+
+if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+  # Covers common datacenter / desktop NVIDIA GPUs.
+  # Override with: -DCMAKE_CUDA_ARCHITECTURES="80;86"
+  set(CMAKE_CUDA_ARCHITECTURES "70;75;80;86;89" CACHE STRING "CUDA architectures")
+endif()
+
+option(VSEARCH_BUILD_TESTS "Build host and GPU tests" ON)
+option(VSEARCH_ENABLE_OMP "Enable OpenMP in host code when available" ON)
+
+find_package(CUDAToolkit REQUIRED)
+
+if(VSEARCH_ENABLE_OMP)
+  find_package(OpenMP QUIET)
+endif()
+
+add_library(vsearch_core STATIC
+  src/binary_io.cpp
+  src/config.cpp
+  src/cpu_reference.cpp
+  src/result_io.cpp
+  src/metric.cpp
+  cuda/engine.cu
+)
+
+target_include_directories(vsearch_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_link_libraries(vsearch_core PUBLIC CUDA::cudart CUDA::cublas)
+target_compile_options(vsearch_core PRIVATE
+  $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
+)
+set_target_properties(vsearch_core PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+)
+
+if(OpenMP_CXX_FOUND AND VSEARCH_ENABLE_OMP)
+  target_link_libraries(vsearch_core PUBLIC OpenMP::OpenMP_CXX)
+endif()
+
+add_executable(vsearch src/main.cpp)
+target_link_libraries(vsearch PRIVATE vsearch_core)
+
+if(VSEARCH_BUILD_TESTS)
+  enable_testing()
+
+  add_executable(test_host tests/test_host.cpp)
+  target_link_libraries(test_host PRIVATE vsearch_core)
+  add_test(NAME host_tests COMMAND test_host)
+
+  add_executable(test_gpu tests/test_gpu.cu)
+  target_link_libraries(test_gpu PRIVATE vsearch_core)
+  add_test(NAME gpu_tests COMMAND test_gpu)
+endif()

09_vector_retrieval/张磊/Makefile.corex

@@ -0,0 +1,56 @@
+# Iluvatar CoreX (天数智芯) build for the GPU vector search engine.
+#
+# The CoreX toolkit ships an nvcc-compatible *stub* and a customized clang
+# driver. CMake's CUDA language support expects a real nvcc, so this simple
+# Makefile compiles every .cu through the CoreX clang `-x ivcore` path and
+# links against the CoreX runtime / cuBLAS libraries directly.
+
+COREX ?= /usr/local/corex
+CXX   := $(COREX)/bin/clang++
+INCS  := -I$(COREX)/include -Iinclude
+LIBS  := -L$(COREX)/lib64 -lcudart -lcublas -lpthread
+CFLAGS := -std=c++17 -O2 -fPIC -Wall
+CUFLAGS := -x ivcore --cuda-path=$(COREX) $(INCS) $(CFLAGS)
+CUFLAGS += -DVSEARCH_COREX=1
+HFLAGS := $(INCS) $(CFLAGS) -DVSEARCH_COREX=1
+
+BUILD := build_corex
+CORE_SRCS := src/binary_io.cpp src/config.cpp src/cpu_reference.cpp \
+             src/result_io.cpp src/metric.cpp cuda/engine.cu
+CORE_OBJS := $(patsubst %.cpp,$(BUILD)/%.o,$(patsubst %.cu,$(BUILD)/%.o,$(CORE_SRCS)))
+
+BIN := vsearch
+TEST_HOST := test_host
+TEST_GPU := test_gpu
+
+.PHONY: all clean tests
+
+all: $(BUILD)/$(BIN) $(BUILD)/$(TEST_HOST) $(BUILD)/$(TEST_GPU)
+
+$(BUILD)/%.o: %.cu
+	@mkdir -p $(dir $@)
+	$(CXX) $(CUFLAGS) -c $< -o $@
+
+$(BUILD)/%.o: %.cpp
+	@mkdir -p $(dir $@)
+	$(CXX) $(HFLAGS) -c $< -o $@
+
+$(BUILD)/$(BIN): $(CORE_OBJS) src/main.cpp
+	@mkdir -p $(BUILD)
+	$(CXX) $(HFLAGS) -c src/main.cpp -o $(BUILD)/src/main.o
+	$(CXX) $(BUILD)/src/main.o $(CORE_OBJS) $(LIBS) -o $@
+
+$(BUILD)/$(TEST_HOST): $(CORE_OBJS) tests/test_host.cpp
+	@mkdir -p $(BUILD)/tests
+	$(CXX) $(HFLAGS) -c tests/test_host.cpp -o $(BUILD)/tests/test_host.o
+	$(CXX) $(BUILD)/tests/test_host.o $(CORE_OBJS) $(LIBS) -o $@
+
+$(BUILD)/$(TEST_GPU): $(CORE_OBJS) tests/test_gpu.cu
+	@mkdir -p $(BUILD)/tests
+	$(CXX) $(CUFLAGS) -c tests/test_gpu.cu -o $(BUILD)/tests/test_gpu.o
+	$(CXX) $(BUILD)/tests/test_gpu.o $(CORE_OBJS) $(LIBS) -o $@
+
+tests: all
+
+clean:
+	rm -rf $(BUILD)

09_vector_retrieval/张磊/Makefile.maca

@@ -0,0 +1,51 @@
+# 沐曦 MetaX（MACA）平台构建。
+#
+# 沐曦通过 cu-bridge 提供 CUDA 兼容层：
+#   * 编译器 : cucc（nvcc 风格 wrapper，内部调用 mxgpu_llvm 的 mxcc）
+#   * 头文件 : cu-bridge/include（cuda_runtime.h / cublas_v2.h / cub 等）
+#             与 /opt/maca/include（mcc、mcblas 等原生头）
+#   * 库     : /opt/maca/lib（libmcblas = cuBLAS 对应实现）
+# 与 NVIDIA / CoreX 共用同一份源码，不需要任何源码改动（沐曦的 64 位原子加与
+# double 精度均正常，因此无需 VSEARCH_COREX 那样的 float 累计分支）。
+
+MACA  ?= /opt/maca
+CUCC  := $(MACA)/tools/cu-bridge/bin/cucc
+CXX   := g++
+INCS  := -I$(MACA)/tools/cu-bridge/include -I$(MACA)/include -Iinclude
+LIBS  := -L$(MACA)/lib -L$(MACA)/tools/cu-bridge/lib -lmcblas
+CFLAGS := -std=c++17 -O2 -fPIC -Wall
+
+BUILD := build_maca
+CORE_SRCS := src/binary_io.cpp src/config.cpp src/cpu_reference.cpp \
+             src/result_io.cpp src/metric.cpp cuda/engine.cu
+CORE_OBJS := $(patsubst %.cpp,$(BUILD)/%.o,$(patsubst %.cu,$(BUILD)/%.o,$(CORE_SRCS)))
+
+.PHONY: all clean
+
+all: $(BUILD)/vsearch $(BUILD)/test_host $(BUILD)/test_gpu
+
+$(BUILD)/%.o: %.cu
+	@mkdir -p $(dir $@)
+	$(CUCC) $(INCS) $(CFLAGS) -c $< -o $@
+
+$(BUILD)/%.o: %.cpp
+	@mkdir -p $(dir $@)
+	$(CXX) $(INCS) $(CFLAGS) -c $< -o $@
+
+$(BUILD)/vsearch: $(CORE_OBJS) src/main.cpp
+	@mkdir -p $(BUILD)/src
+	$(CXX) $(INCS) $(CFLAGS) -c src/main.cpp -o $(BUILD)/src/main.o
+	$(CUCC) $(BUILD)/src/main.o $(CORE_OBJS) $(LIBS) -o $@
+
+$(BUILD)/test_host: $(CORE_OBJS) tests/test_host.cpp
+	@mkdir -p $(BUILD)/tests
+	$(CXX) $(INCS) $(CFLAGS) -c tests/test_host.cpp -o $(BUILD)/tests/test_host.o
+	$(CUCC) $(BUILD)/tests/test_host.o $(CORE_OBJS) $(LIBS) -o $@
+
+$(BUILD)/test_gpu: $(CORE_OBJS) tests/test_gpu.cu
+	@mkdir -p $(BUILD)/tests
+	$(CUCC) $(INCS) $(CFLAGS) -c tests/test_gpu.cu -o $(BUILD)/tests/test_gpu.o
+	$(CUCC) $(BUILD)/tests/test_gpu.o $(CORE_OBJS) $(LIBS) -o $@
+
+clean:
+	rm -rf $(BUILD)

09_vector_retrieval/张磊/Makefile.musa

@@ -0,0 +1,56 @@
+# 摩尔线程 MUSA 平台构建。
+#
+# MUSA 提供原生 API（musa_runtime.h / mublas_v2.h / CUB for MUSA），不提供
+# CUDA 兼容头，因此 engine.cu 通过 -DVSEARCH_MUSA=1 启用一层名称映射
+# （cuda* -> musa*, cublas* -> mublas*），其余实现代码不变。
+#
+# 构建与运行：
+#   make -f Makefile.musa -j
+#   export LD_LIBRARY_PATH=/usr/local/musa/lib:/usr/local/musa/lib64
+#   ./build_musa/test_host && ./build_musa/test_gpu
+
+MUSA ?= /usr/local/musa
+MCC  := $(MUSA)/bin/mcc
+CXX  := g++
+INCS := -I$(MUSA)/include -Iinclude
+HINCS := -Iinclude
+LIBS := -L$(MUSA)/lib -lmusart -lmublas
+CFLAGS  := -std=c++17 -O2 -fPIC -Wall
+# -x musa：把源文件按 MUSA 语言编译（若按 .cu 推断，mcc 会走 CUDA 前端并报
+# 找不到 CUDA 安装）。--musa-path 显式给出 MUSA 安装根目录。
+CUFLAGS := -x musa --musa-path=$(MUSA) $(INCS) $(CFLAGS) -DVSEARCH_MUSA=1
+
+BUILD := build_musa
+CORE_SRCS := src/binary_io.cpp src/config.cpp src/cpu_reference.cpp \
+             src/result_io.cpp src/metric.cpp cuda/engine.cu
+CORE_OBJS := $(patsubst %.cpp,$(BUILD)/%.o,$(patsubst %.cu,$(BUILD)/%.o,$(CORE_SRCS)))
+
+.PHONY: all clean
+
+all: $(BUILD)/vsearch $(BUILD)/test_host $(BUILD)/test_gpu
+
+$(BUILD)/%.o: %.cu
+	@mkdir -p $(dir $@)
+	$(MCC) $(CUFLAGS) -c $< -o $@
+
+$(BUILD)/%.o: %.cpp
+	@mkdir -p $(dir $@)
+	$(CXX) $(HINCS) $(CFLAGS) -c $< -o $@
+
+$(BUILD)/vsearch: $(CORE_OBJS) src/main.cpp
+	@mkdir -p $(BUILD)/src
+	$(CXX) $(HINCS) $(CFLAGS) -c src/main.cpp -o $(BUILD)/src/main.o
+	$(MCC) $(BUILD)/src/main.o $(CORE_OBJS) $(LIBS) -o $@
+
+$(BUILD)/test_host: $(CORE_OBJS) tests/test_host.cpp
+	@mkdir -p $(BUILD)/tests
+	$(CXX) $(HINCS) $(CFLAGS) -c tests/test_host.cpp -o $(BUILD)/tests/test_host.o
+	$(MCC) $(BUILD)/tests/test_host.o $(CORE_OBJS) $(LIBS) -o $@
+
+$(BUILD)/test_gpu: $(CORE_OBJS) tests/test_gpu.cu
+	@mkdir -p $(BUILD)/tests
+	$(MCC) $(CUFLAGS) -c tests/test_gpu.cu -o $(BUILD)/tests/test_gpu.o
+	$(MCC) $(BUILD)/tests/test_gpu.o $(CORE_OBJS) $(LIBS) -o $@
+
+clean:
+	rm -rf $(BUILD)