From 3532f85d7bc4b7f62726f7023c42849431fe3f61 Mon Sep 17 00:00:00 2001 From: Honggyu Kim Date: Thu, 23 Jul 2026 21:28:15 +0900 Subject: [PATCH 1/3] Add dtoctl wrapper to drive DTO without manual LD_PRELOAD setup Using the DTO library today requires the user to understand two orthogonal mechanisms before a single program can be offloaded to DSA. The library has to be injected into the target through LD_PRELOAD, and its behavior is steered entirely through a set of roughly twenty DTO_* environment variables that must be discovered by reading the manual and then exported by hand. This is easy to get wrong: the LD_PRELOAD path and the variable names are stringly typed, an exported variable silently leaks into every later command in the same shell, and there is no validation or discoverability at the point of use. Provide a small wrapper, dtoctl, that turns this into a single self-describing command line. It follows the same model as hmctl from the hmsdk project: parse options, translate them into the environment the library expects, and then exec the target program so no state lingers in the caller's shell. On top of that model dtoctl also takes care of the injection itself by prepending the resolved libdto.so to LD_PRELOAD, which is possible because DTO works by transparently interposing on the standard memory APIs. Every DTO_* variable is exposed as an option so the mapping stays obvious and greppable. Only options that are actually given are written to the child environment; anything left unset keeps whatever the caller already exported or the library's own default, and an explicit option wins over an inherited variable. The library path is resolved from -l/--library, then $DTO_LIBRARY, then a compiled-in default, and execvp() is used so a bare command name on PATH works as well as an explicit path. Build and install dtoctl with both Make and CMake. Set its compiled-in library path to match the installation directory, since CMake's default /usr/local/lib differs from the Makefile's /usr/lib64. Allow that path to be overridden with DTOCTL_LIBPATH. With this in place the previous incantation $ export LD_PRELOAD=/usr/lib64/libdto.so $ export DTO_WAIT_METHOD=busypoll $ export DTO_CPU_SIZE_FRACTION=0.33 $ ./prog collapses to $ dtoctl -w busypoll -c 0.33 ./prog Signed-off-by: Honggyu Kim --- .gitignore | 1 + CMakeLists.txt | 9 ++ Makefile | 11 +- dtoctl.c | 366 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 385 insertions(+), 2 deletions(-) create mode 100644 dtoctl.c diff --git a/.gitignore b/.gitignore index ebb181e..977a993 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dto-test* libdto.so* build*/ tests/baselines*/ +dtoctl diff --git a/CMakeLists.txt b/CMakeLists.txt index 4806e1f..59a00cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,15 @@ if (HAS_WAITPKG) target_compile_options(dto PRIVATE -mwaitpkg -march=native) endif() +# Build dtoctl with a default preload path matching the library installation. +get_filename_component(DTOCTL_LIBDIR "${CMAKE_INSTALL_LIBDIR}" ABSOLUTE + BASE_DIR "${CMAKE_INSTALL_PREFIX}") +set(DTOCTL_LIBPATH "${DTOCTL_LIBDIR}/libdto.so" CACHE FILEPATH + "Path to libdto.so that dtoctl prepends to LD_PRELOAD") +add_executable(dtoctl dtoctl.c) +target_compile_definitions(dtoctl PRIVATE LIBDTO_PATH="${DTOCTL_LIBPATH}") +install(TARGETS dtoctl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + # Build dto-test and dto-test-wodto add_executable(dto-test dto-test.c) target_link_libraries(dto-test PRIVATE DTO::dto pthread) diff --git a/Makefile b/Makefile index 06cc2e1..c200ea1 100644 --- a/Makefile +++ b/Makefile @@ -2,20 +2,27 @@ # # SPDX-License-Identifier: MIT -all: libdto dto-test-wodto +all: libdto dtoctl dto-test-wodto DML_LIB_CXX=-D_GNU_SOURCE +# Path to libdto.so that dtoctl prepends to LD_PRELOAD (matches "make install"). +DTOCTL_LIBPATH ?= /usr/lib64/libdto.so + libdto: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_STATS_SUPPORT -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg libdto_nostats: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg +dtoctl: dtoctl.c + gcc -O2 -Wall dtoctl.c -DLIBDTO_PATH=\"$(DTOCTL_LIBPATH)\" -o dtoctl + install: cp libdto.so.1.0 /usr/lib64/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so + cp dtoctl /usr/bin/ install-local: ln -sf ./libdto.so.1.0 ./libdto.so.1 @@ -28,4 +35,4 @@ dto-test-wodto: dto-test.c gcc -g dto-test.c $(DML_LIB_CXX) -o dto-test-wodto -lpthread clean: - rm -rf *.o *.so dto-test + rm -rf *.o *.so dto-test dtoctl diff --git a/dtoctl.c b/dtoctl.c new file mode 100644 index 0000000..707cd3a --- /dev/null +++ b/dtoctl.c @@ -0,0 +1,366 @@ +/* Copyright (C) 2026 SK hynix, Inc. */ +/* SPDX-License-Identifier: MIT */ + +/* + * dtoctl -- a CLI wrapper that runs a program with the DSA Transparent Offload + * (DTO) library preloaded. It converts command line options into the DTO_* + * environment variables and prepends libdto.so to LD_PRELOAD so that users do + * not need to export the variables or set up LD_PRELOAD manually. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* for asprintf() */ +#endif + +#include +#include +#include +#include +#include +#include + +#ifndef LIBDTO_PATH +#define LIBDTO_PATH "/usr/lib64/libdto.so" +#endif + +/* argp option keys for the long-only options (no short equivalents) */ +enum { + OPT_UMWAIT_DELAY = 0x100, + OPT_OVERLAPPING_MEMMOVE, + OPT_LOG_FILE, + OPT_LOG_LEVEL, + OPT_STATS_FILE, + OPT_NO_MEMCPY, + OPT_NO_MEMMOVE, + OPT_NO_MEMSET, + OPT_NO_MEMCMP, + OPT_NO_CACHE_CONTROL, + OPT_NO_AUTO_ADJUST, + OPT_STDC_ONLY, + OPT_STATS, +}; + +struct opts { + int idx; + char *exename; + + /* library / preload */ + const char *library; + + /* value options (NULL means "not specified") */ + const char *wait_method; + const char *min_bytes; + const char *cpu_fraction; + const char *numa_aware; + const char *wq_list; + const char *umwait_delay; + const char *overlapping_memmove; /* "0" or "1" after validation */ + const char *log_file; + const char *log_level; + const char *stats_file; + + /* flag options (false means "not specified") */ + bool no_memcpy; + bool no_memmove; + bool no_memset; + bool no_memcmp; + bool no_cache_control; + bool no_auto_adjust; + bool stdc_only; + bool stats; +}; + +struct opts opts; + +/* (a part of) output in --help option (generated by argp runtime) */ +const char *argp_program_bug_address = "https://github.com/intel/DTO/issues"; + +/* Option groups are separated using entries with only a .doc field. */ +static struct argp_option dtoctl_options[] = { + {.doc = "Library:", .group = 1}, + {.name = "library", + .key = 'l', + .arg = "PATH", + .doc = "Path to libdto.so to prepend to LD_PRELOAD. Overrides $DTO_LIBRARY " + "and the compiled-in default (" LIBDTO_PATH ")"}, + + {.doc = "Offload tuning:", .group = 2}, + {.name = "wait-method", + .key = 'w', + .arg = "METHOD", + .doc = "How to wait for DSA completion: yield, busypoll, umwait, or tpause " + "(sets DTO_WAIT_METHOD; default: busypoll)"}, + {.name = "min-bytes", + .key = 'b', + .arg = "BYTES", + .doc = "Minimum operation size offloaded to DSA (sets DTO_MIN_BYTES; " + "default: 65536)"}, + {.name = "cpu-fraction", + .key = 'c', + .arg = "FRACTION", + .doc = "Fraction (0.0 <= f < 1.0) of each operation done on CPU in parallel " + "to DSA (sets DTO_CPU_SIZE_FRACTION; default: 0.0)"}, + {.name = "numa-aware", + .key = 'n', + .arg = "MODE", + .doc = "NUMA awareness: 0 disable, 1 buffer-centric, 2 cpu-centric " + "(sets DTO_IS_NUMA_AWARE; default: 0)"}, + {.name = "wq-list", + .key = 'q', + .arg = "LIST", + .doc = "Semicolon-separated list of DSA WQs to use, e.g. \"wq0.0;wq2.0\" " + "(sets DTO_WQ_LIST; default: auto-discover all WQs)"}, + {.name = "umwait-delay", + .key = OPT_UMWAIT_DELAY, + .arg = "CYCLES", + .doc = "Delay in cycles for the umwait instruction (sets DTO_UMWAIT_DELAY; " + "default: 100000)"}, + {.name = "overlapping-memmove", + .key = OPT_OVERLAPPING_MEMMOVE, + .arg = "WHERE", + .doc = "Where to run memmove with overlapping buffers: cpu or dsa " + "(sets DTO_OVERLAPPING_MEMMOVE_ACTION; default: cpu)"}, + + {.doc = "Disable DSA offload for specific operations (DSA is used by default):", + .group = 3}, + {.name = "no-memcpy", + .key = OPT_NO_MEMCPY, + .doc = "Use system memcpy instead of DSA (sets DTO_DSA_MEMCPY=0)"}, + {.name = "no-memmove", + .key = OPT_NO_MEMMOVE, + .doc = "Use system memmove instead of DSA (sets DTO_DSA_MEMMOVE=0)"}, + {.name = "no-memset", + .key = OPT_NO_MEMSET, + .doc = "Use system memset instead of DSA (sets DTO_DSA_MEMSET=0)"}, + {.name = "no-memcmp", + .key = OPT_NO_MEMCMP, + .doc = "Use system memcmp instead of DSA (sets DTO_DSA_MEMCMP=0)"}, + + {.doc = "Other toggles:", .group = 4}, + {.name = "no-cache-control", + .key = OPT_NO_CACHE_CONTROL, + .doc = "Clear the DSA cache control flag to avoid cache pollution " + "(sets DTO_DSA_CC=0; default: cache control on)"}, + {.name = "no-auto-adjust", + .key = OPT_NO_AUTO_ADJUST, + .doc = "Disable auto tuning of cpu-fraction and min-bytes " + "(sets DTO_AUTO_ADJUST_KNOBS=0; default: auto tuning on)"}, + {.name = "stdc-only", + .key = OPT_STDC_ONLY, + .doc = "Use only the standard C memory functions, no DSA offload " + "(sets DTO_USESTDC_CALLS=1; default: DSA offload enabled)"}, + + {.doc = "Statistics and logging:", .group = 5}, + {.name = "stats", + .key = OPT_STATS, + .doc = "Enable stats collection; for debugging/profiling only " + "(sets DTO_COLLECT_STATS=1; default: off)"}, + {.name = "stats-file", + .key = OPT_STATS_FILE, + .arg = "PATH", + .doc = "Write the stats histogram to PATH instead of standard output " + "(sets DTO_STATS_FILE; default: standard output)"}, + {.name = "log-file", + .key = OPT_LOG_FILE, + .arg = "PATH", + .doc = "Redirect DTO output to PATH (suffixed by pid) (sets DTO_LOG_FILE; " + "default: standard output)"}, + {.name = "log-level", + .key = OPT_LOG_LEVEL, + .arg = "LEVEL", + .doc = "Verbosity of DTO logging: 0, 1, or 2 (sets DTO_LOG_LEVEL; " + "default: 0)"}, + + {NULL}, +}; + +static error_t parse_option(int key, char *arg, struct argp_state *state) +{ + struct opts *opts = state->input; + + switch (key) { + case 'l': + opts->library = arg; + break; + case 'w': + opts->wait_method = arg; + break; + case 'b': + opts->min_bytes = arg; + break; + case 'c': + opts->cpu_fraction = arg; + break; + case 'n': + opts->numa_aware = arg; + break; + case 'q': + opts->wq_list = arg; + break; + case OPT_UMWAIT_DELAY: + opts->umwait_delay = arg; + break; + case OPT_OVERLAPPING_MEMMOVE: + if (!strcmp(arg, "cpu")) + opts->overlapping_memmove = "0"; + else if (!strcmp(arg, "dsa")) + opts->overlapping_memmove = "1"; + else { + fprintf(stderr, + "Error: --overlapping-memmove must be 'cpu' or 'dsa'.\n\n"); + argp_usage(state); + } + break; + case OPT_NO_MEMCPY: + opts->no_memcpy = true; + break; + case OPT_NO_MEMMOVE: + opts->no_memmove = true; + break; + case OPT_NO_MEMSET: + opts->no_memset = true; + break; + case OPT_NO_MEMCMP: + opts->no_memcmp = true; + break; + case OPT_NO_CACHE_CONTROL: + opts->no_cache_control = true; + break; + case OPT_NO_AUTO_ADJUST: + opts->no_auto_adjust = true; + break; + case OPT_STDC_ONLY: + opts->stdc_only = true; + break; + case OPT_STATS: + opts->stats = true; + break; + case OPT_LOG_FILE: + opts->log_file = arg; + break; + case OPT_LOG_LEVEL: + opts->log_level = arg; + break; + case OPT_STATS_FILE: + opts->stats_file = arg; + break; + + case ARGP_KEY_ARG: + if (state->arg_num) + return ARGP_ERR_UNKNOWN; + if (opts->exename == NULL) { + /* remaining options will be processed in ARGP_KEY_ARGS */ + return ARGP_ERR_UNKNOWN; + } + break; + + case ARGP_KEY_ARGS: + /* process remaining non-option arguments */ + opts->exename = state->argv[state->next]; + opts->idx = state->next; + break; + + case ARGP_KEY_NO_ARGS: + case ARGP_KEY_END: + if (state->arg_num < 1) + argp_usage(state); + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static void setup_child_environ(struct opts *opts) +{ + const char *lib, *cur; + char *preload; + + /* value options: only set when explicitly specified */ + if (opts->wait_method) + setenv("DTO_WAIT_METHOD", opts->wait_method, 1); + if (opts->min_bytes) + setenv("DTO_MIN_BYTES", opts->min_bytes, 1); + if (opts->cpu_fraction) + setenv("DTO_CPU_SIZE_FRACTION", opts->cpu_fraction, 1); + if (opts->numa_aware) + setenv("DTO_IS_NUMA_AWARE", opts->numa_aware, 1); + if (opts->wq_list) + setenv("DTO_WQ_LIST", opts->wq_list, 1); + if (opts->umwait_delay) + setenv("DTO_UMWAIT_DELAY", opts->umwait_delay, 1); + if (opts->overlapping_memmove) + setenv("DTO_OVERLAPPING_MEMMOVE_ACTION", opts->overlapping_memmove, 1); + if (opts->log_file) + setenv("DTO_LOG_FILE", opts->log_file, 1); + if (opts->log_level) + setenv("DTO_LOG_LEVEL", opts->log_level, 1); + if (opts->stats_file) + setenv("DTO_STATS_FILE", opts->stats_file, 1); + + /* flag options: only set when explicitly specified */ + if (opts->no_memcpy) + setenv("DTO_DSA_MEMCPY", "0", 1); + if (opts->no_memmove) + setenv("DTO_DSA_MEMMOVE", "0", 1); + if (opts->no_memset) + setenv("DTO_DSA_MEMSET", "0", 1); + if (opts->no_memcmp) + setenv("DTO_DSA_MEMCMP", "0", 1); + if (opts->no_cache_control) + setenv("DTO_DSA_CC", "0", 1); + if (opts->no_auto_adjust) + setenv("DTO_AUTO_ADJUST_KNOBS", "0", 1); + if (opts->stdc_only) + setenv("DTO_USESTDC_CALLS", "1", 1); + if (opts->stats) + setenv("DTO_COLLECT_STATS", "1", 1); + + /* resolve the libdto path and prepend it to LD_PRELOAD */ + lib = opts->library ? opts->library : getenv("DTO_LIBRARY"); + if (lib == NULL) + lib = LIBDTO_PATH; + + cur = getenv("LD_PRELOAD"); + if (cur == NULL || *cur == '\0') { + setenv("LD_PRELOAD", lib, 1); + return; + } + + /* + * LD_PRELOAD is a colon separated list, so it can be much longer than a + * single path. Allocate the result instead of formatting it into a + * fixed size buffer, which would silently drop part of what the caller + * had already preloaded. + */ + if (asprintf(&preload, "%s:%s", lib, cur) < 0) { + perror("cannot build LD_PRELOAD"); + exit(EXIT_FAILURE); + } + setenv("LD_PRELOAD", preload, 1); + free(preload); +} + +int main(int argc, char *argv[]) +{ + struct argp argp = { + .options = dtoctl_options, + .parser = parse_option, + .args_doc = " [...]", + .doc = "dtoctl -- run a program with DSA Transparent Offload (DTO) preloaded", + }; + + argp_parse(&argp, argc, argv, ARGP_IN_ORDER, NULL, &opts); + + /* pass only the target program and its arguments to execvp() */ + argc -= opts.idx; + argv += opts.idx; + + setup_child_environ(&opts); + + execvp(opts.exename, argv); + perror(opts.exename); + + return -1; +} From eaa786e071d25a8a9243ba8877aac6afc2aea9ab Mon Sep 17 00:00:00 2001 From: Honggyu Kim Date: Sat, 5 Sep 2026 08:59:50 +0900 Subject: [PATCH 2/3] Add a dtoctl man page The dtoctl options mirror the DTO_* environment variables, but a user coming to the tool for the first time still needs a single authoritative place that explains what each option does, what values it accepts, and how the library path and LD_PRELOAD are resolved. Scattering that information across the source help text alone makes it hard to consult without running the tool. Add a manual page in the same troff-friendly markdown form the hmsdk project uses for hmctl, and render it as a section 1 man page for the user command. Besides the per-option reference it documents the library resolution order and the override semantics, and it gathers the common tuning goals - latency, power, and cache-pollution avoidance - as ready-to-run examples such as $ dtoctl -w umwait -c 0.0 --no-auto-adjust ./prog so users can map an intent directly onto a command instead of reverse engineering it from the individual variables. Keep the markdown as the source form and generate dtoctl.1 with pandoc during the Makefile and CMake builds. Both build systems install the generated page under man1, so it can be read with $ man 1 dtoctl Pandoc is optional: when it is unavailable, print a message recommending its installation and skip manual generation. The Makefile can still install an already generated page. Ignore its generated doc/dtoctl.1 and remove it on make clean; CMake keeps its generated page in the build directory. Signed-off-by: Honggyu Kim --- .gitignore | 1 + CMakeLists.txt | 18 +++++ Makefile | 25 ++++++- doc/dtoctl.md | 181 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 doc/dtoctl.md diff --git a/.gitignore b/.gitignore index 977a993..b9711b7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ libdto.so* build*/ tests/baselines*/ dtoctl +/doc/dtoctl.1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 59a00cc..90e4b39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,24 @@ add_executable(dtoctl dtoctl.c) target_compile_definitions(dtoctl PRIVATE LIBDTO_PATH="${DTOCTL_LIBPATH}") install(TARGETS dtoctl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +# Build and install the optional section 1 manual. +find_program(PANDOC_EXECUTABLE NAMES pandoc) +if(PANDOC_EXECUTABLE) + set(DTOCTL_MAN_PAGE "${CMAKE_CURRENT_BINARY_DIR}/doc/dtoctl.1") + add_custom_command( + OUTPUT "${DTOCTL_MAN_PAGE}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/doc" + COMMAND "${PANDOC_EXECUTABLE}" --standalone --to=man + "${CMAKE_CURRENT_SOURCE_DIR}/doc/dtoctl.md" -o "${DTOCTL_MAN_PAGE}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/doc/dtoctl.md" + COMMENT "Generating dtoctl(1) manual" + VERBATIM) + add_custom_target(dtoctl-man ALL DEPENDS "${DTOCTL_MAN_PAGE}") + install(FILES "${DTOCTL_MAN_PAGE}" DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) +else() + message(STATUS "Skipping dtoctl manual generation and installation: install pandoc to build the man page.") +endif() + # Build dto-test and dto-test-wodto add_executable(dto-test dto-test.c) target_link_libraries(dto-test PRIVATE DTO::dto pthread) diff --git a/Makefile b/Makefile index c200ea1..8b4b2cf 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,16 @@ # # SPDX-License-Identifier: MIT -all: libdto dtoctl dto-test-wodto +all: libdto dtoctl dto-test-wodto man + +.PHONY: man install-man DML_LIB_CXX=-D_GNU_SOURCE # Path to libdto.so that dtoctl prepends to LD_PRELOAD (matches "make install"). DTOCTL_LIBPATH ?= /usr/lib64/libdto.so +PANDOC ?= pandoc +MANDIR ?= /usr/share/man libdto: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_STATS_SUPPORT -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg @@ -18,12 +22,28 @@ libdto_nostats: dto.c dtoctl: dtoctl.c gcc -O2 -Wall dtoctl.c -DLIBDTO_PATH=\"$(DTOCTL_LIBPATH)\" -o dtoctl -install: +man: doc/dtoctl.1 + +doc/dtoctl.1: doc/dtoctl.md + @if command -v "$(PANDOC)" >/dev/null 2>&1; then \ + "$(PANDOC)" --standalone --to=man $< -o $@; \ + else \ + echo "Skipping dtoctl manual generation: install pandoc to build the man page."; \ + fi + +install: install-man cp libdto.so.1.0 /usr/lib64/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so cp dtoctl /usr/bin/ +install-man: man + @if [ -f doc/dtoctl.1 ]; then \ + install -D -m 644 doc/dtoctl.1 "$(MANDIR)/man1/dtoctl.1"; \ + else \ + echo "Skipping dtoctl manual installation: no generated man page is available."; \ + fi + install-local: ln -sf ./libdto.so.1.0 ./libdto.so.1 ln -sf ./libdto.so.1.0 ./libdto.so @@ -36,3 +56,4 @@ dto-test-wodto: dto-test.c clean: rm -rf *.o *.so dto-test dtoctl + rm -f doc/dtoctl.1 diff --git a/doc/dtoctl.md b/doc/dtoctl.md new file mode 100644 index 0000000..af9f5e7 --- /dev/null +++ b/doc/dtoctl.md @@ -0,0 +1,181 @@ +% DTOCTL(1) Dtoctl User Manuals +% Honggyu Kim +% 2026 + +NAME +==== +dtoctl - run a program with DSA Transparent Offload (DTO) preloaded + + +SYNOPSIS +======== +dtoctl [_options_] COMMAND [_command-options_] + + +DESCRIPTION +=========== +The **dtoctl** tool runs `COMMAND` with the DSA Transparent Offload library +(**libdto.so**) preloaded, so that its `memcpy`, `memmove`, `memset`, and +`memcmp` calls are transparently offloaded to Intel DSA (Data Streaming +Accelerator). + +Without **dtoctl**, users must set **LD_PRELOAD** to point at **libdto.so** and +export a set of **DTO_\*** environment variables to control DTO behavior. +**dtoctl** replaces that manual setup: it converts its command line options into +the corresponding **DTO_\*** environment variables, prepends **libdto.so** to +**LD_PRELOAD**, and then executes `COMMAND`. + +Only the options that are explicitly given are applied. Options that are not +given leave any pre-existing **DTO_\*** environment variables (or the DTO +built-in defaults) untouched, and an explicitly given option overrides a +previously exported variable of the same name. + + +LIBRARY RESOLUTION +================== +The path of **libdto.so** to preload is resolved in the following order: + +1. The `-l`, `--library` option, if given. +2. The **DTO_LIBRARY** environment variable, if set. +3. The compiled-in default path (`/usr/local/lib/libdto.so` for a default + CMake build, or `/usr/lib64/libdto.so` for the Makefile build). + +The resolved path is prepended to any existing **LD_PRELOAD** value. + + +OPTIONS +======= + +Library +------- +-l _PATH_, \--library=_PATH_ +: Path to **libdto.so** to prepend to **LD_PRELOAD**. Overrides + **DTO_LIBRARY** and the compiled-in default. + Default: the compiled-in library path (see LIBRARY RESOLUTION). + +Offload tuning +-------------- +-w _METHOD_, \--wait-method=_METHOD_ +: How to wait for DSA completion: _yield_, _busypoll_, _umwait_, or _tpause_. + Sets **DTO_WAIT_METHOD**. Default: _busypoll_. + +-b _BYTES_, \--min-bytes=_BYTES_ +: Minimum operation size offloaded to DSA. Smaller operations run on the CPU. + Sets **DTO_MIN_BYTES**. Default: 65536. + +-c _FRACTION_, \--cpu-fraction=_FRACTION_ +: Fraction (0.0 <= f < 1.0) of each operation performed on the CPU in parallel + to DSA. Sets **DTO_CPU_SIZE_FRACTION**. Default: 0.0. + +-n _MODE_, \--numa-aware=_MODE_ +: NUMA awareness: _0_ disable, _1_ buffer-centric, _2_ cpu-centric. + Sets **DTO_IS_NUMA_AWARE**. Default: _0_. + +-q _LIST_, \--wq-list=_LIST_ +: Semicolon-separated list of DSA work queues to use, e.g. "wq0.0;wq2.0". + Names must match those under /dev/dsa/. Sets **DTO_WQ_LIST**. + Default: auto-discover all available WQs. + +\--umwait-delay=_CYCLES_ +: Delay in cycles for the umwait instruction. Sets **DTO_UMWAIT_DELAY**. + Default: 100000. + +\--overlapping-memmove=_WHERE_ +: Where to run memmove with overlapping buffers: _cpu_ or _dsa_. + Sets **DTO_OVERLAPPING_MEMMOVE_ACTION** (cpu -> 0, dsa -> 1). Default: _cpu_. + +Disable DSA offload for specific operations +------------------------------------------- +By default every one of these operations is offloaded to DSA; each option below +turns a single one back into its standard C library call on the CPU. + +\--no-memcpy +: Use the system memcpy instead of DSA. Sets **DTO_DSA_MEMCPY=0**. + +\--no-memmove +: Use the system memmove instead of DSA. Sets **DTO_DSA_MEMMOVE=0**. + +\--no-memset +: Use the system memset instead of DSA. Sets **DTO_DSA_MEMSET=0**. + +\--no-memcmp +: Use the system memcmp instead of DSA. Sets **DTO_DSA_MEMCMP=0**. + +Other toggles +------------- +\--no-cache-control +: Clear the DSA cache control flag to avoid cache pollution. + Sets **DTO_DSA_CC=0**. Default: cache control is on. + +\--no-auto-adjust +: Disable auto tuning of cpu-fraction and min-bytes. + Sets **DTO_AUTO_ADJUST_KNOBS=0**. Default: auto tuning is on. + +\--stdc-only +: Use only the standard C memory functions, no DSA offload. + Sets **DTO_USESTDC_CALLS=1**. Default: DSA offload is enabled. + +Statistics and logging +---------------------- +\--stats +: Enable stats collection. For debugging/profiling only, as it slows down the + workload. Sets **DTO_COLLECT_STATS=1**. Default: off. + +\--stats-file=_PATH_ +: Write the stats histogram to _PATH_ instead of standard output. + Sets **DTO_STATS_FILE**. Default: standard output. + +\--log-file=_PATH_ +: Redirect DTO output to _PATH_ (the file name is suffixed by the pid). + Sets **DTO_LOG_FILE**. Default: standard output. + +\--log-level=_LEVEL_ +: Verbosity of DTO logging: _0_, _1_, or _2_. Sets **DTO_LOG_LEVEL**. + Default: _0_. + +-?, \--help +: Print help message and list of options with description. + +\--usage +: Print usage string. + + +EXAMPLES +======== +Run a program with DTO using the installed **libdto.so**: + + $ dtoctl ./prog + +The equivalent of the manual setup + + $ export LD_PRELOAD=/usr/lib64/libdto.so + $ export DTO_WAIT_METHOD=busypoll + $ export DTO_CPU_SIZE_FRACTION=0.33 + $ export DTO_AUTO_ADJUST_KNOBS=1 + $ ./prog + +becomes a single command: + + $ dtoctl -w busypoll -c 0.33 ./prog + +Latency reduction mode (auto tuning on, busy polling): + + $ dtoctl -w busypoll ./prog + +Power reduction mode (offload everything to DSA, wait with umwait): + + $ dtoctl -w umwait -c 0.0 --no-auto-adjust ./prog + +Avoid cache pollution (offload everything, clear cache control): + + $ dtoctl -w yield -c 0.0 --no-auto-adjust --no-cache-control ./prog + +Use a locally built library and a specific set of work queues, and collect +stats: + + $ dtoctl -l ./libdto.so.1.0 -q "wq0.0;wq2.0;wq4.0;wq6.0" --stats ./prog + + +SEE ALSO +======== +**ld.so**(8) From bb10ddedaac250213523e6433c793ecd518a7d89 Mon Sep 17 00:00:00 2001 From: Honggyu Kim Date: Sat, 5 Sep 2026 08:59:50 +0900 Subject: [PATCH 3/3] Document dtoctl in the README The README is where users first learn how to configure and run DTO, and until now it only described the manual LD_PRELOAD and DTO_* export workflow. A reader following it has no reason to know that dtoctl exists or that it removes most of that ceremony, so the easier path stays hidden precisely where people look for guidance. Introduce dtoctl after the environment variable reference and the common usage models that accompany it, so that discussion stays in one piece and the new section can point back at those usage models. Contrast the old export sequence with the equivalent $ dtoctl -w busypoll -c 0.33 ./prog and spell out the resolution order for libdto.so along with the rule that only the options actually passed are applied. Keep the per-option reference in dtoctl(1) instead of repeating it here. The option list already exists in the --help output and in the manual, and a third copy in the README would only be one more place to forget when an option changes. Signed-off-by: Honggyu Kim --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index bd3e239..4175970 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,39 @@ Although not the only usage models of DTO, the following are some common ones: DTO_WAIT_METHOD=yield or umwait (saves either cycles or power) +## Using dtoctl + +Setting `LD_PRELOAD` and exporting the `DTO_*` environment variables by hand is +error prone. The `dtoctl` tool provides a simpler interface: it takes the DTO +settings as command line options, converts them to the corresponding `DTO_*` +environment variables, prepends `libdto.so` to `LD_PRELOAD`, and then runs the +target program. For example, instead of + +```bash +export LD_PRELOAD=/usr/local/lib/libdto.so +export DTO_WAIT_METHOD=busypoll +export DTO_CPU_SIZE_FRACTION=0.33 +export DTO_AUTO_ADJUST_KNOBS=1 +./prog +``` + +you can run + +```bash +dtoctl -w busypoll -c 0.33 ./prog +``` + +Every environment variable listed above has a corresponding option, and only the +options you pass are applied: a `DTO_*` variable you already exported is left +untouched, and an explicit option overrides it. The `libdto.so` path is resolved +from `-l/--library`, then `$DTO_LIBRARY`, then the compiled-in default +(`/usr/local/lib/libdto.so` for a CMake build, `/usr/lib64/libdto.so` for the +Makefile build), and is prepended to any existing `LD_PRELOAD`. + +Run `dtoctl --help` for the option list, or `man 1 dtoctl` for the full manual, +which documents each option and gives a ready-to-run command for each of the +usage models above. + ## Build Pre-requisite packages: