diff --git a/03_hadamard_tc/a962695448-rgb/.gitattributes b/03_hadamard_tc/a962695448-rgb/.gitattributes new file mode 100644 index 00000000..0cddae0f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/.gitattributes @@ -0,0 +1,19 @@ +# Keep the experimental source hash reproducible across platforms. +src/tune_launch.cu text eol=lf +platforms/iluvatar/shared_baseline.cu text eol=lf +platforms/iluvatar/hadamard_api.h text eol=lf +platforms/iluvatar/hadamard_api.cu text eol=lf +platforms/iluvatar/validate_and_benchmark.cu text eol=lf +platforms/iluvatar/run_platform.py text eol=lf +platforms/metax/hadamard_api.h text eol=lf +platforms/metax/hadamard_api.cu text eol=lf +platforms/metax/validate_and_benchmark.cu text eol=lf +platforms/metax/run_platform.py text eol=lf +platforms/biren/hadamard_api.h text eol=lf +platforms/biren/hadamard_api.su text eol=lf +platforms/biren/validate_and_benchmark.su text eol=lf +platforms/biren/run_platform.py text eol=lf +platforms/ascend/*.h text eol=lf +platforms/ascend/*.cpp text eol=lf +platforms/ascend/*.py text eol=lf +platforms/ascend/CMakeLists.txt text eol=lf diff --git a/03_hadamard_tc/a962695448-rgb/.gitignore b/03_hadamard_tc/a962695448-rgb/.gitignore new file mode 100644 index 00000000..86a5f072 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/.gitignore @@ -0,0 +1,10 @@ +/build/ +__pycache__/ +*.pyc +*.nsys-rep +*.sqlite +/results/iluvatar/**/validate_and_benchmark +/results/metax/**/validate_and_benchmark +/results/biren/**/validate_and_benchmark +/results/ascend/**/validate_and_benchmark +/results/ascend/**/build/ diff --git a/03_hadamard_tc/a962695448-rgb/EVIDENCE.md b/03_hadamard_tc/a962695448-rgb/EVIDENCE.md new file mode 100644 index 00000000..3028ddd0 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/EVIDENCE.md @@ -0,0 +1,64 @@ +# 验证证据与完整档案 + +本提交保留实现、必要测试、平台入口、简明报告及摩尔的核心复现材料。 +完整开发分支和历史实验不随此次精简删除。 + +## 固定档案 + +- [完整归档提交 3676727](https://github.com/a962695448-rgb/Learning-CUDA/tree/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb) +- [历史 results 目录](https://github.com/a962695448-rgb/Learning-CUDA/tree/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results) + +该 results 目录的 2,586 个历史文件通过固定提交引用。日志、失败记录、数据和 +实验脚本仍可在个人仓库查询；本 PR 的功能和测试不要求下载全部历史归档。 +复现某次历史实验时，应使用该实验清单指定的源码、环境和输入；历史说明中的 +results 路径属于完整档案检出。新运行可以在当前目录创建自己的结果目录。 + +## 重点审查入口 + +| 内容 | 入口与范围 | +|---|---| +| NVIDIA 最新生产验证 | [A100/4090 调用方式与自动规则](reports/a100-calling-validation.md)，原始数据链接固定到对应档案 | +| 独立参考与线程配置 | [A100 跨卡报告](reports/a100-validation.md)，包含固定 Dao 对照与全部负例 | +| 小维度行打包 | [A800/4090 报告](reports/packed-rows-validation.md)，按设备和 Graph/普通调用分别说明 | +| N256 显式融合布局 | [4090](reports/fused-layout-validation.md)、[A100](reports/fused-layout-a100-validation.md) | +| 五种国产平台 | 各 platforms 子目录 README，分别列出 SDK、设备、正确性和本机基线 | +| 摩尔原始计时与再生成 | [公开复现材料](platforms/moore/repro/README.zh-CN.md)，含 6,750 条计时、冻结对照和夹具生成器 | +| 代码来源与本轮检查 | [SUBMISSION_MANIFEST.json](SUBMISSION_MANIFEST.json) | + +## 首轮精简提交的验证边界 + +初次精简阶段没有改变 CUDA/MUSA 等计算源码。保留代码的 Git blob SHA 与固定档案一致； +当时的工作是精简提交、整理文档链接和执行 CPU 检查。此前 GPU 结果仍归属于 +其清单明确记录的实测版本，不将本轮整理写成新的 GPU 实验。 + +本地 CPU reference 检查、摩尔离线复算及数据完整性检查通过。原始 GPU 结果中 +测试范围、跳过项、慢例及计时方式均保留，跨设备及不同计时口径不直接相除。 + +## 2026-09-17 独立量化入口 + +新增显式 packed 量化入口的完整范围见 [报告](reports/quantize-packed-20260917.md)。 +[固定档案](https://github.com/a962695448-rgb/Learning-CUDA/tree/29bb34a0b2817e282a9554bd92c3911a3c010762/03_hadamard_tc/a962695448-rgb/results/quantize-packed-20260917) 包含最终双重性能门槛、正确性检查、源码和复现准备工具，也保存新增参数引入主机开销的初版实验。当前源文件与新测试哈希已在 SUBMISSION_MANIFEST.json 更新；未将先前的跨卡或国产平台记录当成本轮新入口的实测证据。 + +## 2026-09-17 单元素穷举与未采用候选 + +[报告](reports/singleton-quantization-20260917.md) 记录当前生产内核的完整单元素位型检查，以及未达收益门槛的符号简化候选。生产计算源码未变，新工具的指纹已登记；失败候选的源码和计时数据独立保留。 + +[本轮固定档案](https://github.com/a962695448-rgb/Learning-CUDA/tree/5e80eadd156c82a299e7cc7991bf46d1904243e6/03_hadamard_tc/a962695448-rgb/results/singleton-quantization-20260917) 保存完整原始结果与可验证的准备工具。 + +## 2026-09-17 成对量化 + +[最终报告](reports/paired-quantization-20260917.md) 记录 RTX4090 的分量编码、偏移、旧接口、长采样和 A/A 校准。新增 kernel 与测试的哈希已登记；三版源码及失败协议分别归档，不借用别的型号或历史源码作为本轮验证。 + +[本轮固定验证档案](https://github.com/a962695448-rgb/Learning-CUDA/tree/9b3bb79f3c67b6c5e284be7ded9559148b5ae448/03_hadamard_tc/a962695448-rgb/results/paired-quantization-20260917) 保留完整的所有阶段及原始判定。 + +## 2026-09-17 打包变换验证与成对候选 + +[本轮报告](reports/paired-hadamard-20260917.md) 列出基线验证与两个 REJECT 候选。计算/构建源码保持 b49fff4，新增验证器按实机通过的内容登记指纹；完整两版数据和源码通过报告中的固定归档链接复现。 + +## 2026-09-17 普通调用交叉诊断 + +[诊断报告](reports/host-attribution-20260917.md) 区分独立中位数与配对统计，并给出同源模块、缓冲池和测量顺序对照。固定证据通过报告链接访问；本次不新增生产计算变化或采纳结论。 + +## 2026-09-17 整合入口完整验证 + +[本轮报告](reports/integrated-pairs-20260917.md) 保存CLI六配置自测、接口回归和588设备/588主机对照。唯一未过判据为绝对耗时离散度；相对性能达标不自动覆盖该失败。原始数据可通过无损恢复工具逐文件校验。 diff --git a/03_hadamard_tc/a962695448-rgb/Makefile b/03_hadamard_tc/a962695448-rgb/Makefile new file mode 100644 index 00000000..97e13167 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/Makefile @@ -0,0 +1,34 @@ +CUDA_HOME ?= $(HOME)/.local/opt/cuda-12.8 +NVCC ?= $(CUDA_HOME)/bin/nvcc +ARCH ?= 89 +CXX ?= g++ +NVCCFLAGS ?= -O3 -std=c++17 -lineinfo -arch=sm_$(ARCH) + +.PHONY: all self-test benchmark cpu-test clean FORCE +all: build/hadamard + +build/config.txt: FORCE + @mkdir -p build + @printf '%s\n' '$(NVCC) $(NVCCFLAGS)' > build/config.tmp + @cmp -s build/config.tmp $@ || cp build/config.tmp $@ + @rm -f build/config.tmp + +build/hadamard: src/main.cu include/kernels.cuh include/contiguous256.cuh include/packed_rows.cuh include/row_policy.hpp include/reference.hpp build/config.txt + mkdir -p build + $(NVCC) $(NVCCFLAGS) -Iinclude src/main.cu -o $@ + +self-test: build/hadamard + ./build/hadamard --self-test + +benchmark: build/hadamard + ./build/hadamard --benchmark --csv results/benchmark.csv + +cpu-test: tests/cpu_reference_test.cpp include/reference.hpp tests/row_policy_test.cpp include/row_policy.hpp + mkdir -p build + $(CXX) -O3 -std=c++17 -Iinclude tests/cpu_reference_test.cpp -o build/cpu_reference_test + ./build/cpu_reference_test + $(CXX) -O3 -std=c++17 -Iinclude tests/row_policy_test.cpp -o build/row_policy_test + ./build/row_policy_test + +clean: + rm -f build/hadamard build/cpu_reference_test build/row_policy_test build/config.txt build/config.tmp diff --git a/03_hadamard_tc/a962695448-rgb/README.md b/03_hadamard_tc/a962695448-rgb/README.md new file mode 100644 index 00000000..80259543 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/README.md @@ -0,0 +1,134 @@ +# Hadamard 变换加速与 INT4 融合 + +2026 夏季训练营 Hadamard 项目的精简提交版本，提供 NVIDIA CUDA 实现、PyTorch +前向接口，以及五种国产平台的独立入口。完整开发记录见 [证据索引](EVIDENCE.md)。 + +## 功能与数值约定 + +- 对最后一维执行 Sylvester Hadamard 变换，支持 FP16/BF16 存储、FP32 内部计算。 +- 最后一维支持 1～256 的二次幂；CLI 使用 batch、seq、heads、dim 描述四维输入。 +- 同时提供普通 CUDA 基线、warp 寄存器实现和稠密 WMMA Tensor Core 对照。 +- 支持分步与融合 INT4：先将变换结果舍入到公开存储类型，再按行缩放、最近偶数舍入、 + 截断到 [-7,7] 并打包；分步与融合的字节及 scales 要求精确一致。 +- 默认 scale=1；归一化为 1/sqrt(dim)。全零行的量化 scale 为 1。 + +PyTorch 接受连续、非空的二维或四维 CUDA 张量，输出变换保持输入形状和 dtype。 +打包结果为 uint8，最后一维为 ceil(dim/2)；行 scales 为 FP32。 +接口使用调用方当前 stream，提供前向计算；不支持自动求导、FP8、任意步长和 dim>256。 + +## 快速开始 + +以下命令从本目录执行。CPU reference 检查只需要 C++17： + +~~~bash +make cpu-test +~~~ + +NVIDIA 构建需要 CUDA Toolkit 和 sm80+ GPU。按实际设备设置 ARCH： + +~~~bash +# A100 使用 ARCH=80；RTX 4090/4060 使用 ARCH=89。 +make CUDA_HOME=/usr/local/cuda ARCH=89 +./build/hadamard --self-test +python3 scripts/run_validation.py --label validation-new --benchmark +~~~ + +PyTorch 扩展通过当前 PyTorch ABI 编译： + +~~~bash +python3 scripts/build_torch_extension.py --verbose +~~~ + +~~~python +import torch +from scripts.build_torch_extension import load_extension + +op = load_extension() +x = torch.randn((2, 16), device="cuda", dtype=torch.float16) +y = op.hadamard(x, 1.0) +packed, row_scales = op.hadamard_int4(x, 1.0) +split_packed, split_scales = op.quantize_int4(y) +assert torch.equal(packed, split_packed) +assert torch.equal(row_scales, split_scales) +~~~ + +Hadamard 与融合接口默认 row_layout 为 original、线程数为 128。packed/auto、256 线程和 contiguous256 +融合布局均为显式选项，其设备及形状范围见 [完整接口与开发记录](docs/DEVELOPMENT.md)。 +Tensor Core 对照在部分已测场景中慢于 warp，相关结果完整保留。 + +## 平台入口 + +| 平台 | 构建与验收说明 | +|---|---| +| NVIDIA CUDA | 本页、[A100/4090 生产验证](reports/a100-calling-validation.md) | +| 天数 MR-V100 / COREX | [原生 API、Warp64 与结果](platforms/iluvatar/README.md) | +| 沐曦 C500 / MACA，25% sGPU 配额 | [原生 API 与结果](platforms/metax/README.md) | +| 壁仞 106M / SUPA | [原生 API、BF16 舍入与结果](platforms/biren/README.md) | +| 昇腾 910B1 / CANN | [Ascend C API 与结果](platforms/ascend/README.md) | +| 摩尔 S4000 / MUSA | [报告](platforms/moore/REPORT.zh-CN.md)、[公开复现材料](platforms/moore/repro/README.zh-CN.md) | + +这些后端使用各自的 SDK、编译器和设备 API，按对应 README 独立构建。 + +## 验证与性能范围 + +- NVIDIA 生产版本在 A100 与 4090 分别通过 1,876 组 CLI 输入的多种模式、 + 1,800 组固定版本第三方对照及接口/stream/偏移条件检查。 +- 分步、融合 INT4 的输出字节和行 scales 精确一致；原始失败与负例均可追溯。 +- 摩尔完整验证为 1,504 组输入、192 项 API 和 14 项 CLI 检查；精确量化优化的 + 三轮配对计时共 6,750 条，默认融合相对上一版在已测配置中为 2.012～2.898×。 +- 各平台记录各自的硬件、SDK、版本和计时方式；Graph、event 与端到端数据分别解释。 + +以上为首轮跨平台基线的归档结果，版本、计数与限制见 [证据索引](EVIDENCE.md)。 +后续增量优化与补测见下方专题报告，各次结论只覆盖对应源码、硬件和测试范围。 + +## 建议审查顺序 + +1. include/reference.hpp：独立参考、舍入和打包约定。 +2. include/kernels.cuh 及其他内核头文件：CUDA 实现与显式调优策略。 +3. src/main.cu、tests/cpu_reference_test.cpp：验证与基准入口。 +4. src/torch_binding.cu：张量契约和前向接口。 +5. 各国产平台目录与对应报告。 + +## 2026-09-16 执行上下文补测 + +新增 [120 组流与 CUDA Graph 回归](reports/execution-context-20260916.md)：独立 CPU 参考、输入变化、输入不可变性，以及默认流错误/旧输出负例。RTX 4090 D 实测全部通过；原计算内核和性能结论保持各自的历史验证范围。 + +## RTX 4090 D 自动布局优化 + +[限定范围的 auto 路由](reports/4090d-routing-20260916.md) 在 dim 1/2/4/8/16、4096～65536 行上选用已有 packed kernel。180 组配置经过三轮配对及正确性验证，受测 auto 路径的 Graph 设备执行加速为 1.31～7.53×。默认 original 保持不变；该数字不代表普通 Python 调用或模型端到端收益。 + +## 复用输出缓冲区 + +新增 [三个 out 前向接口](reports/out-buffers-20260917.md)，可复用预分配的输出。受测小、中批量的普通 Python 调用平均加速约 1.75～2.05×；输出由调用者持有，接口返回 None。原接口、默认参数及设备核函数源码保持不变。 + +## 小维度独立量化 + +新增显式 `quantize_int4_packed` 与 `quantize_int4_packed_out`，支持 N≤16。RTX4090D 的三轮分组设备加速约 2.87–3.17×，普通 allocating/out 调用整体几何平均约 1.36×/1.89×；原接口签名与默认路径保留。见 [使用方式、完整范围与原始实验取舍](reports/quantize-packed-20260917.md)。原始结果见[固定验证档案](https://github.com/a962695448-rgb/Learning-CUDA/tree/29bb34a0b2817e282a9554bd92c3911a3c010762/03_hadamard_tc/a962695448-rgb/results/quantize-packed-20260917)。 + +## 单元素量化穷举 + +新增 [FP16/BF16 单元素量化穷举](reports/singleton-quantization-20260917.md)，覆盖 131,072 种存储编码和 128,768 个有限值的独立参考。计算简化候选未达到预设性能门槛，已保留实验记录并继续使用原生产内核。 + +## 成对独立量化 + +N=2/4/8/16 的显式 packed 量化改为每线程处理相邻两值，保持原 API。RTX4090 最终分组设备几何平均加速约 1.36–1.43×，普通 allocating/out 目标约 1.07×/1.12×；所有对照及 A/A 校准通过。见 [完整范围、失败记录与复现入口](reports/paired-quantization-20260917.md)。 + +## 打包变换边界验证与负实验 + +新增 [packed Hadamard 验证器及两版实验记录](reports/paired-hadamard-20260917.md)。基线通过 128 项有限分量/偏移配置与 24 项流/Graph 检查；两版成对变换候选均未通过普通调用门槛，生产计算源码保持 b49fff4。 + +## 普通调用计时诊断 + +[三进程交叉诊断](reports/host-attribution-20260917.md) 保存同源双模块、共享/轮换输出池和测量顺序的3240个计时块。四种配置未复现跨进程稳定超过5%的候选差异；它不替代完整矩阵，原来的两次未采用判定和生产计算保持原有范围。 + +## 整合入口候选的完整验证 + +[完整CLI/接口/性能验证](reports/integrated-pairs-20260917.md) 的正确性与所有相对性能限值通过，但一个共同波动区间未过预设绝对稳定性判据，最终仍为REJECT。生产内核保持原版，完整日志和时间线已归档。 + +## 相对稳定性前瞻确认 + +[三轮新采样确认](reports/relative-stability-confirm-20260918.md) 仍有一项配对比值波动超出预登记门槛，最终未采用。全部原始结果已归档；生产计算保持原有版本，停止对该候选重复确认。 + +## N32–256 半warp变换验证 + +[完整三轮记录](reports/wide-pairs-20260918.md) 包含416项有限分量、512项溢出兼容及全部入口回归。678设备与1356主机记录完整，部分配置退化超出门槛，因此保留原生产实现；融合目标分组的收益也按未采用候选记录。 diff --git a/03_hadamard_tc/a962695448-rgb/SUBMISSION_MANIFEST.json b/03_hadamard_tc/a962695448-rgb/SUBMISSION_MANIFEST.json new file mode 100644 index 00000000..a6f4007f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/SUBMISSION_MANIFEST.json @@ -0,0 +1,529 @@ +{ + "archive_commit": "3676727fc21ea27bda668d743d10e52d7e138f68", + "upstream_base": "a17ce401ad3e49d0871e946d6ac6d7d09c35cb74", + "historical_results_files_referenced_externally": 2586, + "computation_and_build_sources_unchanged": false, + "sources": [ + { + "path": "Makefile", + "git_blob_sha": "97e13167aabe95e5461df101aaaf6cbf4be30d96", + "sha256": "61ff9c3d8ecd66773e8d59b6496201bbbfb661a12232e690d776dc8a1630e93f", + "archive_git_blob_sha": "288f6b6cfefc7f8e63e469ed9fa7d16d4021c9ca", + "archive_sha256": "8f15a347ded757dfcb6c54cb2444265e549c3245d928e2652d8195e342cd4d73" + }, + { + "path": "include/contiguous256.cuh", + "git_blob_sha": "5e940d50e42ddc3048c68d07e2d0a75ab5ef864b", + "sha256": "dd25a633e6740a20b77c1c19ab375ca4ee8190e4a63ea4b82b17669406c46102" + }, + { + "path": "include/kernels.cuh", + "git_blob_sha": "4618f3fd2964dff2341b6cdbfd402da00e55ae26", + "sha256": "12bf5330c4a9825312421fee9836f9fb8a1eeae83ebff346990aac2a29e275fe" + }, + { + "path": "include/packed_rows.cuh", + "git_blob_sha": "ca599d395e25992000ffb8c58d0fce678fc7141c", + "sha256": "6a0dfe4c020cd8c3fba5da4c35f1ceabdb2b2b948e1ab90553bc0cd1b9c93b30" + }, + { + "path": "include/reference.hpp", + "git_blob_sha": "e06ce3fef99a5b431ef0a0c23e0e59bbb5a582a7", + "sha256": "5be2751c9ebf3729bf6a59e19c9b91ad52a6b3a7d86720de4a23880f76deb3c4" + }, + { + "path": "include/row_policy.hpp", + "git_blob_sha": "84f9fce567cb156dab9e9d3642f4fb4a20f26653", + "sha256": "9f4bea3233106a446d650e8d4b9f96dfae22571abc2eed8a1cce0b4b5215253d", + "archive_git_blob_sha": "b2d64d5437446cecd69c49e1e1460afc4ff38c10", + "archive_sha256": "57c319d22f5e9c74539134bf6cb077a43ab81e4c0f4dacf23348379f1e562a62" + }, + { + "path": "platforms/ascend/CMakeLists.txt", + "git_blob_sha": "993c61ac0377441133a2431f2ec944788fac9daf", + "sha256": "3ce9073967af0d0be8016fe23c0b1e3b0fa1c90e1536276d896fec9c276cb6ef" + }, + { + "path": "platforms/ascend/hadamard_api.cpp", + "git_blob_sha": "8c4456af283714e9412f4eadaeddef075bbb97b9", + "sha256": "da5777047a5228d2a05e4149223f162ea27eeaed6ea28fd134563ff06ecc46a6" + }, + { + "path": "platforms/ascend/hadamard_api.h", + "git_blob_sha": "203f83a9af82e5020d790847a17cd1bbddc99c46", + "sha256": "e6e6197708bdbfedc9826f693a5dfe905121e488069efad2468453b9f27745f6" + }, + { + "path": "platforms/ascend/hadamard_kernel.cpp", + "git_blob_sha": "832bd7b344f48070fdff45c6302fc6405fd393cf", + "sha256": "a1e1d4c19d38058cf4875931abd8eda187d87b314539549d50f8a710dca771eb" + }, + { + "path": "platforms/ascend/run_platform.py", + "git_blob_sha": "d8609abb475012bd08beb53e213d122dcfad2944", + "sha256": "a1c08ec5fc9d0571a72e050fef74243b1f21a1d2fbee0654096183a4d3a484dd" + }, + { + "path": "platforms/ascend/validate_and_benchmark.cpp", + "git_blob_sha": "8aadb0254ac6dd5b9c64035f52d8724dd151ae49", + "sha256": "cbdbe64e0cecc4d8af6e3ea1e6acb318dabc03cb990090b2343c7f3085d58a4f" + }, + { + "path": "platforms/biren/hadamard_api.h", + "git_blob_sha": "1e8d545bfd309e64b9807f0bd44834bd3557c095", + "sha256": "1fb2e487891aa8aca3b5f03bef2647654af7f2fe8f41b76a2d57a6ae741b38e6" + }, + { + "path": "platforms/biren/hadamard_api.su", + "git_blob_sha": "6a4b0cdf34b398864ca4777ba417b97c67c92354", + "sha256": "310b54d1818c9b518f70c481a02ba6bd86e472c126ef261beff4524aef021396" + }, + { + "path": "platforms/biren/run_platform.py", + "git_blob_sha": "386e6e59c900a21624500a235c7712d7529662fe", + "sha256": "dfcdf9d7a4030ad8337c41f0f3fd53e0c5a88af9031af38ead36dd9cc6b41585" + }, + { + "path": "platforms/biren/validate_and_benchmark.su", + "git_blob_sha": "e4a976b8e80db8b80fef75b01d7588b4d6d18325", + "sha256": "fb8ce24e1250dcc7a0ebcfe845201be7892a5596ba6f7a96b9170e19a7517adb" + }, + { + "path": "platforms/iluvatar/analyze_runs.py", + "git_blob_sha": "700e7ba2d2c78a2015298c5fb79ec9ea3ced6a45", + "sha256": "ec9a66a6d33f8b84d0970f5e2e8ad5932885884f48914966119ec6be7f9e1142" + }, + { + "path": "platforms/iluvatar/hadamard_api.cu", + "git_blob_sha": "c4191e64a9fe77293cc5f6c1a087f8d3f1fb7921", + "sha256": "7b33cd38339efaeb36b5829338fda044109a9b1cd38f1b925cc79cbe063c7330" + }, + { + "path": "platforms/iluvatar/hadamard_api.h", + "git_blob_sha": "9beb6db9852c1e88ae1db937865e3f4499f316fc", + "sha256": "aaf22b677b43f67e3d8489529f03dcaea21a962a947a50927d05f55ed5e2f570" + }, + { + "path": "platforms/iluvatar/run_platform.py", + "git_blob_sha": "61d044c7dfddee9df8731d32ef6668fe24b18cfc", + "sha256": "f48906d8536822631a2c41e0bbd4e574b3ec09da8917fa7265b6049da0276bbe" + }, + { + "path": "platforms/iluvatar/shared_baseline.cu", + "git_blob_sha": "e6763cbc4bc0da18cba86e8e083290ee58a94459", + "sha256": "d8b00f6e60feaf0dd450ca66255e05bd7e0e01a060d911a9fc7c61779023f180" + }, + { + "path": "platforms/iluvatar/validate_and_benchmark.cu", + "git_blob_sha": "686b7b9b05cc5005f4ecbb6432b99f8358586975", + "sha256": "3712dc8513b5a298d572d32a6faacac468a47d21d73a7cc37a724613a7f7405d" + }, + { + "path": "platforms/metax/hadamard_api.cu", + "git_blob_sha": "b0c828ba755e56c5610545c01b59d10c5c875c7a", + "sha256": "407e6b7e962f9f14c910a6714c5443446bd4b99e33ea2c15fb018065c7f15b71" + }, + { + "path": "platforms/metax/hadamard_api.h", + "git_blob_sha": "324032daf4184fe40bf575dc2acf85621dfd6d3e", + "sha256": "d8e6d120dc38aa7256daef411e08956c4a045989b3975fb22c663bf50b2f9a4d" + }, + { + "path": "platforms/metax/run_platform.py", + "git_blob_sha": "a1fb213a05346838f665d681cf6d43659832d3c9", + "sha256": "1667504c3f15b1010ab36ad0c75c7524c4325cdee69ddf8e3d56ad4337677e08" + }, + { + "path": "platforms/metax/validate_and_benchmark.cu", + "git_blob_sha": "eccd6d3a5b123dcb8ca2ce8174ad3e53d64f42ab", + "sha256": "cce649557ed63f8459c5b03e6fc8943aef47dcf8dc26b10d24a659995752ad47" + }, + { + "path": "platforms/moore/Makefile", + "git_blob_sha": "67318f7889d58bb82b38030fd79a45d95b9e8140", + "sha256": "ccd363d6e1152d1fb3e47f856ecf05bbd431519329effb9f46924cc434e4917c" + }, + { + "path": "platforms/moore/analyze_results.py", + "git_blob_sha": "c9be695d6f643f43af54ef281bc729a9d11b8eed", + "sha256": "68efa54dda9b5d95d35746c5ca66a79fda1f77980c617c06218867897f4df0e2" + }, + { + "path": "platforms/moore/exact_int4.hpp", + "git_blob_sha": "2a1cd4043bc4af5282a63f1a6453f2948608dcc3", + "sha256": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a" + }, + { + "path": "platforms/moore/hadamard_api.h", + "git_blob_sha": "1c437ef4c5382a7471b3fff47c3f4fea67ca73f6", + "sha256": "cba4d5c187defec8f70083a02ba252da8d73f5fba5b1b7692efb91dd1a0c4d93" + }, + { + "path": "platforms/moore/hadamard_api.mu", + "git_blob_sha": "28a29c63950a230045be6b6856cd8ef2da97ec88", + "sha256": "f9c7fb325cbdc19e57e5c02ce58ec973197b75039fff38eb4b1fbac4daec4a80" + }, + { + "path": "platforms/moore/probe.mu", + "git_blob_sha": "da10eb7451460c4696563dcbdd1d6aac7cb3d2df", + "sha256": "2e777447c5153b55a7eee8bd7729170517637c4e04cb3cdbe89c6c51ec58bccd" + }, + { + "path": "platforms/moore/repro/analyze_optimization.py", + "git_blob_sha": "0536e7d642928679c37e42f5c9676b171e345c13", + "sha256": "8d6f11ca3fd7b8c709a0885502272d472d8169027fe29b652afd2d8c70f758d0" + }, + { + "path": "platforms/moore/repro/check_quant_cpu.cpp", + "git_blob_sha": "77c32413cf53d53685406e7c544cdae3a538c0c4", + "sha256": "8d4c6702c36a81f1cdef27df8e4b6bb413b92585d5169f21053d97ac1a450d1a" + }, + { + "path": "platforms/moore/repro/check_quant_cpu.py", + "git_blob_sha": "cb236b18cd9c7cb6478acffd227d6dded4aed9bc", + "sha256": "50d2a8115079c039541818765317bb6efb0efac24ed85b4586bef79e14c49227" + }, + { + "path": "platforms/moore/repro/check_quant_gpu.mu", + "git_blob_sha": "2455fabb0a8b37e5368f9e975bb61a4a31b59225", + "sha256": "e51771809fb54db5cdb9e63c142f5266f2105ce0777d24577e428941f89f3b5d" + }, + { + "path": "platforms/moore/repro/control/control_api.h", + "git_blob_sha": "744e12d740f1b494ce7c397f64ffa86070db49bf", + "sha256": "b40c05f707e806381b3b7440537a435fdb4137cf27b78d403b336b32bf722014" + }, + { + "path": "platforms/moore/repro/control/control_api.mu", + "git_blob_sha": "bda8354d20b68cb0a6f600de96b62573c91bc4af", + "sha256": "d3ebe8f53f076a66db0cdc5d8cca5092fa821c729336949c3e2913d45e5ad56c" + }, + { + "path": "platforms/moore/repro/paired_benchmark.mu", + "git_blob_sha": "7cc7fde3a512d661156a7bca0071cc4bdea90794", + "sha256": "138e6dd6ec9f9fd959162ae5edc0d27727e355ba5354cd1bd70bf702cfc69043" + }, + { + "path": "platforms/moore/repro/test_analyze_optimization.py", + "git_blob_sha": "069ad03ed2f19935c687466d0d0db8119ed7490a", + "sha256": "d05f1426a406c21aa37efccd5f0d09e3856f281ad84aa6706c13a6806216156e" + }, + { + "path": "platforms/moore/run_platform.py", + "git_blob_sha": "124bd82dc75aa8d2b9a9a2ec1ccd54c744ce7571", + "sha256": "d92b236ca4bcaff910921e2d0ef21f452968bb66f1bd98a1a157c0a7c8a88773" + }, + { + "path": "platforms/moore/validate_and_benchmark.mu", + "git_blob_sha": "2cb2977f5be413c81b409164dfcb6b8f43cce9ee", + "sha256": "2a23a7938bd3140742b3c45f6b85db08d539bf03d4cb44ee9a48be41a268459e" + }, + { + "path": "scripts/build_torch_extension.py", + "git_blob_sha": "8883aa543690dbb1ba61052b62842fcd222482fb", + "sha256": "b27c9f2e4867ca7a0de79554f1591689124c7b13eb9fb47f0d0300cb7d62402a" + }, + { + "path": "scripts/compare_reference.py", + "git_blob_sha": "392acd6158c2c0ae651cfc3b28e18232b8b50a39", + "sha256": "aca622b5768141b8f0da6d264e693d632f1f25ad6b64e8abc222cb099a704171" + }, + { + "path": "scripts/run_validation.py", + "git_blob_sha": "c309ce669a5fae89783251d2fd2c05f6e0135d92", + "sha256": "986ab36b0ba347ef3bfdabbf133f9c7123bd587ffa21aced76091f033136c55a" + }, + { + "path": "scripts/verify_block_threads.py", + "git_blob_sha": "0da6570b0628441bd774fce6f2958dabf00026f5", + "sha256": "689d9c6116f13571a37dc4853a99323b1aa08203e83b7128ba78e4430783b141" + }, + { + "path": "scripts/verify_tensor_metadata.py", + "git_blob_sha": "a176dbec5d88ded23b65b90a0c7074e7681be3c1", + "sha256": "c0e7e33bd82a1a582a31035ced93cd4a2b0df8f9d8c81a39df0892a76a6eaf12" + }, + { + "path": "src/main.cu", + "git_blob_sha": "6433184a6de7d3ba2abbe661c39728b733c92d16", + "sha256": "a77680855fe3bdd907b45651dc5144f6f067c134e7a3e22775d8d27713cac9bc" + }, + { + "path": "src/torch_binding.cu", + "git_blob_sha": "ca22740b15715725d250e573eb810f0a7807e0e8", + "sha256": "950be67c774ed66994e2dee87412a5423b7498a21d62c0072f4ef7a399fdefb1", + "archive_git_blob_sha": "8919d8e441a2972bc78edb4e0d5b7a23079ad276", + "archive_sha256": "0939e6e08f06080c245a171124c1c04a2342298eb69663d4aef4b583f346bed8" + }, + { + "path": "src/tune_launch.cu", + "git_blob_sha": "942b15c7d28befedb01fa099471a7312f86110c4", + "sha256": "8a9899f5180180752af20659aca31275279e3c2b1fc1d1baaea69a8b87198212" + }, + { + "path": "tests/cpu_reference_test.cpp", + "git_blob_sha": "9895de71da56836136db12ef0904e71e3f4bbc96", + "sha256": "d436dd007f22669dcb80df7057e1536b223b8e03f7f11f10a6257163939597cc" + } + ], + "closeout_checks": { + "cpu_reference": "PASS: 9 matrix-oracle/involution cases, rounding, packing, zero, invalid shape", + "moore_offline_analysis": "6750 observations; complete result equals the previously validated analysis", + "moore_analysis_tests": 6, + "new_gpu_tests": false + }, + "supplemental_test_sources": [ + { + "path": "scripts/verify_execution_context.py", + "git_blob_sha": "c9034a55d357025c455a3f48fba3ea7a04f4d68f", + "sha256": "a2c1e11c233e388b0b6beec3f6ccadf9cdaa0628f894a7711aaf5de16a165c53" + }, + { + "path": "tests/row_policy_test.cpp", + "git_blob_sha": "26207f53a5165d4f44c83f1b850ad226c180fbf4", + "sha256": "f40bf7370539863f61630a98501f096908bc13fd5115e663ce33c37a679606d4" + }, + { + "path": "scripts/verify_out_buffers.py", + "sha256": "07daf545e33a099e530586526c19afb79c80efdf2d78273150d1b659312a5cd8", + "git_blob_sha": "b3bccd0f09ea513cd8901c8c5be227db1e566b55" + }, + { + "path": "scripts/verify_quantize_packed.py", + "git_blob_sha": "92b63effad45ff1e125c6460cd1d24193d833078", + "sha256": "a3cfa1fb9e7b08e2e24f5c4855f497e0c1eafd7a79ace722c4822e6c41e7d57d" + }, + { + "path": "scripts/verify_singleton_quantization.py", + "sha256": "e839ed36f6f40c6c07be10d424e750570511b8066ad2f8c83275dc9c520d4977", + "git_blob_sha": "660b61300e21942499d8bd6a1725c53c3215c800" + }, + { + "path": "scripts/verify_paired_quantization.py", + "sha256": "e752d64fd429d6923bd2f94bb30d63f682214fc5771951b48199ced908c604c6", + "git_blob_sha": "f0249a5fe5cbf7d6a405d76c521994d37384b9bd" + }, + { + "path": "scripts/verify_hadamard_packed.py", + "sha256": "78c24e33b34d95dba6b5e55959eabf97e3be16428a9116015616b7d96a72b090", + "git_blob_sha": "326b4112d4bb7e8c113ef63821d0dfe0344f3911", + "validated_origin": "initial/isolated source/scripts/verify_paired_hadamard.py; byte-identical body" + } + ], + "supplemental_test_scope": "Prior source/build provenance retained. New packed Hadamard verification is validated on production b49fff4. Both paired-transform optimization candidates remain only in the evidence archive.", + "original_source_changes": [ + "Makefile", + "include/row_policy.hpp", + "src/torch_binding.cu" + ], + "original_sources_unchanged_count": 48, + "compute_kernels_unchanged": false, + "previous_validations": [ + { + "base_commit": "c591085756fec139c55f3de59c2a4ba0742dc5cc", + "device": "NVIDIA GeForce RTX 4090 D", + "unique_configurations": 180, + "paired_rounds": 3, + "comparisons": 540, + "new_route_context_configurations": 16, + "cpu_row_policy_checks": 270, + "decision": "ACCEPT" + }, + { + "base_commit": "bce004ecb2097a20207080f7507e4cfd2ee49d06", + "feature": "reusable output APIs", + "device": "NVIDIA GeForce RTX 4090 D", + "positive_configurations": 372, + "negative_checks": 63, + "mutation_checks": 4, + "execution_context_configurations": 24, + "paired_rounds": 3, + "benchmark_configurations": 72, + "benchmark_records": 216, + "decision": "ACCEPT", + "metric": "synchronized ordinary Python host wall time" + }, + { + "base_commit": "28fb3a95ca8bcdf53a66949cfde23820bf7e4005", + "feature": "explicit packed quantize-only APIs preserving legacy signatures", + "device": "NVIDIA GeForce RTX 4090 D", + "positive_configurations": 700, + "negative_condition_groups": 4, + "execution_context_configurations": 16, + "graph_replays": 64, + "detected_context_negative_controls": 32, + "legacy_positive_configurations": 372, + "legacy_negative_checks": 63, + "mutation_checks": 4, + "paired_rounds": 3, + "device_benchmark_records": 240, + "host_benchmark_records": 240, + "decision": "ACCEPT", + "metric": "CUDA Graph device event timing and separate synchronized ordinary Python host wall time", + "archive_commit": "29bb34a0b2817e282a9554bd92c3911a3c010762", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/29bb34a0b2817e282a9554bd92c3911a3c010762/03_hadamard_tc/a962695448-rgb/results/quantize-packed-20260917" + }, + { + "base_commit": "f9da76087f6cbf6aad52e9effa6cf3ac3aabccf0", + "feature": "exhaustive singleton INT4 verification; retain existing production kernel", + "device": "NVIDIA GeForce RTX 4090 D", + "unique_storage_encodings": 131072, + "unique_finite_values": 128768, + "exhaustive_configurations": 8, + "production_kernel_unchanged": true, + "correctness": "PASS for control and candidate", + "candidate_benchmark_decision": "REJECT", + "paired_rounds": 3, + "device_benchmark_records": 132, + "host_benchmark_records": 132, + "candidate_target_geomean_range": [ + 1.0265712845681487, + 1.0436901022093767 + ], + "minimum_target_geomean": 1.1, + "archive_commit": "5e80eadd156c82a299e7cc7991bf46d1904243e6", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/5e80eadd156c82a299e7cc7991bf46d1904243e6/03_hadamard_tc/a962695448-rgb/results/singleton-quantization-20260917" + }, + { + "base_commit": "fabe4aa1beb4f623564ba02ce8cdf24af5932b11", + "feature": "paired explicit quantize-only kernels with compile-time launch dimensions", + "device": "NVIDIA GeForce RTX 4090", + "finite_component_encodings": 128768, + "component_offset_configurations": 64, + "packed_positive_configurations": 700, + "packed_negative_conditions": 4, + "execution_context_configurations": 16, + "legacy_positive_configurations": 372, + "legacy_negative_checks": 63, + "mutation_checks": 4, + "paired_rounds": 3, + "device_benchmark_records": 288, + "host_benchmark_records": 288, + "same_function_calibrations": 9, + "decision": "ACCEPT", + "previous_candidate_decisions": [ + "REJECT generic launcher", + "REJECT runtime-dimension dispatcher", + "REJECT long-sampling runtime dispatcher" + ], + "target_device_geomean_range": [ + 1.3609850971796464, + 1.4253554874534755 + ], + "archive_commit": "9b3bb79f3c67b6c5e284be7ded9559148b5ae448", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/9b3bb79f3c67b6c5e284be7ded9559148b5ae448/03_hadamard_tc/a962695448-rgb/results/paired-quantization-20260917" + } + ], + "current_validation": { + "base_commit": "b49fff4ac57ae6702213077ebf7ff848cb8715d9", + "feature": "packed Hadamard verifier and rejected transform-pair experiments", + "device": "NVIDIA GeForce RTX 4090", + "production_computation_and_build_unchanged_from_base": true, + "baseline_component_offset_configurations": 128, + "baseline_execution_context_configurations": 24, + "finite_component_encodings": 128768, + "candidate_overflow_compatibility_cases_each": 512, + "candidate_trials": [ + { + "name": "initial", + "device_records": 588, + "host_records": 588, + "failed_host_records": 3, + "decision": "REJECT" + }, + { + "name": "isolated fused host dispatch", + "device_records": 588, + "host_records": 588, + "failed_host_records": 5, + "decision": "REJECT" + } + ], + "verification_decision": "PASS", + "optimization_adoption_decision": "REJECT", + "decision": "KEEP_BASELINE", + "archive_commit": "3ff75126f3b29458c9a2dfb2665349afb3a4e58e", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/3ff75126f3b29458c9a2dfb2665349afb3a4e58e/03_hadamard_tc/a962695448-rgb/results/paired-hadamard-20260917" + }, + "supplemental_kernel_sources": [ + { + "path": "include/packed_pairs.cuh", + "sha256": "0678a75afe8b02a0841bb2baad839404bb6ba6e7cdd27e9e42a0258b970d3add", + "git_blob_sha": "5314139ffb0aaff02f5330fc6720de214b90deeb" + } + ], + "diagnostic_followups": [ + { + "kind": "module-buffer-order host timing diagnostic", + "production_head": "55c8f48e3c75698ce1537c541e3f25831c74c4ed", + "processes": 3, + "cases": 4, + "conditions": 5, + "timed_blocks": 3240, + "same_source_calibrations_within_five_percent": true, + "adoption_decision": "NOT_APPLICABLE_DIAGNOSTIC_ONLY", + "prior_rejections_preserved": true, + "archive_commit": "dbe195f4a4c60947e42eb11cafe8aab689c6b7bc", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/dbe195f4a4c60947e42eb11cafe8aab689c6b7bc/03_hadamard_tc/a962695448-rgb/results/host-attribution-20260917" + }, + { + "kind": "integrated kernel entry candidate complete validation", + "base_commit": "4516fd4b1cd9afdd7cee499f09a4f614ef637400", + "device": "NVIDIA GeForce RTX 4090 D", + "candidate_cli_selftest_configurations": 6, + "cases_per_cli_configuration": 1876, + "device_records": 588, + "host_records": 588, + "target_groups": 24, + "same_source_calibrations": 18, + "relative_performance_gates_passed": true, + "absolute_time_stability_failures": 1, + "candidate_decision": "REJECT", + "production_compute_unchanged": true, + "archive_commit": "704b0b3625cf93048b21ec47ac578f9bc0c32c0b", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/704b0b3625cf93048b21ec47ac578f9bc0c32c0b/03_hadamard_tc/a962695448-rgb/results/integrated-pairs-20260917" + }, + { + "kind": "preregistered relative stability confirmation", + "production_head": "c72d7b723dae14cc7dc880bd8f2ed83cbbaef41a", + "preregistration_commit": "1f97d437370af9f5c102797308ad5fd1a8db9bef", + "device": "NVIDIA GeForce RTX 4090 D", + "overflow_compatibility_cases": 512, + "device_records": 588, + "host_records": 588, + "target_groups": 24, + "same_source_calibrations": 18, + "relative_stability_failures": 1, + "round_decisions": [ + "ACCEPT", + "REJECT", + "ACCEPT" + ], + "candidate_decision": "REJECT", + "production_compute_unchanged": true, + "prior_rejections_preserved": true, + "repeat_confirmation_stopped": true, + "archive_commit": "06b985425e3192fb58e0500ddaffde3476de78ef", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/06b985425e3192fb58e0500ddaffde3476de78ef/03_hadamard_tc/a962695448-rgb/results/relative-stability-confirm-20260918" + }, + { + "kind": "wide transform half-warp candidate complete validation", + "base_commit": "bde0256944778d3749547b798045c5a05d0dab35", + "device": "NVIDIA GeForce RTX 4090 D", + "finite_component_configurations": 416, + "overflow_compatibility_configurations": 512, + "new_execution_contexts": 64, + "full_execution_contexts": 120, + "device_records": 678, + "host_records": 1356, + "target_groups": 24, + "same_source_calibrations": 18, + "failed_gates": 210, + "candidate_decision": "REJECT", + "production_compute_unchanged": true, + "preregistered_commit": "95b13fa0f9519936e004b3ac6f120ab0a04e84ef", + "archive_commit": "c0a105c725343e3332c3465e125384bbd815d8eb", + "evidence_url": "https://github.com/a962695448-rgb/Learning-CUDA/tree/c0a105c725343e3332c3465e125384bbd815d8eb/03_hadamard_tc/a962695448-rgb/results/wide-pairs-20260918" + } + ] +} diff --git a/03_hadamard_tc/a962695448-rgb/docs/DEVELOPMENT.md b/03_hadamard_tc/a962695448-rgb/docs/DEVELOPMENT.md new file mode 100644 index 00000000..e5e19213 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/docs/DEVELOPMENT.md @@ -0,0 +1,365 @@ +# 完整接口与开发记录（归档说明） + +本页来自固定档案 3676727 的接口与历史验证说明，所有命令从项目根目录执行。当前精简提交和归档边界见 [README](../README.md) 与 [证据索引](../EVIDENCE.md)。 + +在输入最后一维执行 Sylvester Hadamard 变换，提供普通 CUDA 基线、warp 寄存器实现、Tensor Core 对照实现，以及变换与 INT4 量化融合。支持 FP16/BF16 和四维输入形状；验证与性能测量命令在同一可执行程序中。 + +本目录的命令行程序只需要 C++17 和 CUDA Toolkit。另提供 PyTorch 前向接口，用于接收真实张量，并与第三方 CUDA 库在同一张 GPU 上比较。 + +实施与后续优化的固定输入、计时和验收标准见 [实施与优化方案](../reports/implementation-optimization-plan.md)。 + +## 当前状态 + +当前生产源码为 `24dfef776ad73a4128cb6138674c5886c21e49c0`。N=1/2/4/8/16支持显式packed/auto，默认仍original；新增A100-SXM4-40GB经过独立筛选和留出验证的9条自动规则，已有A800/4090规则不变。详见[A100规则与调用方式报告](../reports/a100-calling-validation.md)。 + +最终源码在A100与4090分别通过同一1876项CLI矩阵的七种模式、原1800项Dao对照、280项大行数/偏移/stream条件及接口/CSV检查。A100另外完成160条原生安全夹具、memcheck和synccheck零错误；两卡nsys执行路径核验完成。全部原始数据、失败与哈希见[本轮档案](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_calling_20260908/README.md)。 + +最终Graph中，A100的64个启用配置、4090的62个启用配置三轮均减少至少5%耗时，相对同轮更快的原128/256线程路径，范围分别为5.24%～68.56%、5.51%～66.74%。两卡各24个回退控制的波动范围在约±0.31%内。这些是已测配置的均摊Graph结果，不是应用整体收益。 + +普通Python调用可使用已有位置参数写法：相对同一auto的关键字写法，12配置三轮的耗时减少范围为A100 6.22%～9.49%、4090 5.95%～12.31%。相对原默认调用仍有最高约3.45%的额外开销，默认original保留。旧A800/4090负例、A100首次PATH配置失败和时间线检查脚本的范围误判均独立保留。 + +此前N256显式 `fused_layout="contiguous256"` 的[4090记录](../reports/fused-layout-validation.md)和[A100记录](../reports/fused-layout-a100-validation.md)保留各自版本与负例。PR与正式提交待项目所有者验收。 + +## 已完成的基线与平台验证 + +- 2026-09-05 已在租赁的 **NVIDIA GeForce RTX 4090 24 GB（sm89）** 上编译命令行程序和 PyTorch 扩展；原有 RTX 4060 Laptop / WSL2 记录也保留。 +- 4090 的 **1,876 组自测通过**，覆盖 FP16/BF16、连续随机、正态和异常值输入，最大绝对误差为 `0.0078125`。35 个 warp 输出元素与已舍入的稠密矩阵参考存在容差内差异，原始日志完整保留。 +- CPU 对实际变换输出的量化、GPU 分步量化、GPU 融合量化的 packed bytes 与 scales 全量一致；15 项非法命令行参数检查通过。 +- 4090 的 16 组形状/精度/scale 配置已测量，共 110 行性能数据，包含 CPU、GPU 含复制时间及各 CUDA 路径。 +- 与固定版本 `fast_hadamard_transform` 的 **1,800 组真实 GPU 对照通过**，FP16 和 BF16 的最大绝对差均为 0；10 项非法张量输入检查、非默认 CUDA stream 检查通过。单卡环境没有验证多 GPU 切换。 +- 第三方比较另完成 12 组 CUDA Graph 性能测量，且每个捕获输出均与 eager 输出一致。本轮大批量 6 组的 `Dao 时间 / 本项目时间` 为 `1.200～1.926`；`[17,256]` 两种精度下本项目稍慢，负例完整保留。 +- Nsight Systems 已成功导出时间线和 kernel 统计。Nsight Compute 因 `ERR_NVGPUCTRPERM` 无法读取 GPU 性能计数器，失败日志保留，没有虚构带宽利用率或占用率结论。 +- NVIDIA 新增显式 `block_threads=128/256`（CLI 为 `--block-threads`），**默认仍为128**。两种选择均通过同一1,876组CLI矩阵；原1,800组Dao矩阵逐输入确认旧默认、显式128、显式256位一致，测试重复不累计为新用例。 +- 256线程的原24个候选与48个邻近配置，在64份独立输出的CUDA Graph条件下三轮均减少耗时至少5%，实测范围为6.53%～25.58%；仅覆盖下文明确的N=16/64范围，不自动派发、不保证其他形状更快。 +- A100-SXM4-40GB 已重新按 sm80 编译并完成同一1,876组CLI矩阵、1,800组固定Dao对照及线程兼容检查；预定72配置三轮也已完成。CUDA 与九齿的 A100 结果分别记录，当前尚无上游合并结果。 +- 天数 MR-V100 已完成独立 COREX 原生适配：三种实现通过 1,504 组变换、180 项接口及 14 项命令行检查，三轮性能原始样本完整保存。源码和原始结果见下方平台入口，不与 NVIDIA 的测试数量相加。 +- 沐曦 C500 的 25% sGPU / 16000 MiB 配额也完成独立 MACA 原生验证：相同 1,504 组矩阵、180 项接口和 14 项命令行检查通过，三轮共 4,050 条计时样本。重复同一矩阵的跨平台执行不累计为更多独立算法用例。 +- 壁仞 106M 完成原生 SUPA 适配，显式处理旧 SDK 的 BF16 截断与项目最近偶数舍入约定的差异。完整矩阵、量化一致性、三轮基准及小批量留出验证通过；同时保留 Warp32 比共享实现更慢的形状和未推广的实验。 +- 昇腾 910B1 完成原生 Ascend C/CANN 9.0 适配：主矩阵1,496组、多核专项128组和有限大行数专项2组通过，分步/融合INT4与CPU参考精确一致。可选矢量缩放的三轮交错对照完整保留8,400条观测；不使用会改变量化中点结果的矢量除法。 + +下文初轮命令行结果对应实现基线 `6f8e15a2db63a1816c2da6632848a1945380cf21`；初轮 eager 比较使用修复本地安装来源识别后的脚本 `766283ba7352250d5def06fbe62428c74e546917`；CUDA Graph 与 profiler 记录对应 `0b29fcf9031193f49319b2d4132df4d1ef6a4a74`。原始文件及 SHA-256 清单见[实测证据](#实测证据)。 + +## 平台入口 + +- 摩尔线程 MTT S4000 / MUSA 5.1.0：[原生 API 与优化报告](../platforms/moore/REPORT.zh-CN.md)。精确 INT4 整数量化通过 1,504 组完整输入、192 项 API 与 14 项 CLI 检查；三轮 6,750 条配对计时中，默认融合相对上一版加速 2.012～2.898×。17,527,208 项 CPU 与 2,074,568 项重叠 GPU 量化检查通过。本轮 77 份文本证据及测试夹具已本地校验。 + +- NVIDIA A100：[跨卡验收报告](../reports/a100-validation.md)与[完整档案](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_20260906/README.md)。同轮128/256线程配对的72配置，每轮均减少至少5%耗时，范围6.19%～26.09%；仅限预定N=16/64形状。默认128的原12组Graph仍保留两个较慢的`[17,256]`配置，不把eager比值当作纯kernel收益。 + +| 平台 | 交付入口与实测边界 | +|---|---| +| NVIDIA RTX 4090 / 4060 | 本页的 CUDA 命令行、PyTorch 前向扩展与参考库对照；下方 `sm80+` 构建要求针对这条路径 | +| 天数 Iluvatar MR-V100 / COREX 4.4.0 | [原生 C++ API、Warp64 复现与完整报告](../platforms/iluvatar/README.md)，[固定源码与原始结果清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/manifest.json)；独立构建，不使用 NVIDIA `sm_89` 目标 | +| 沐曦 MetaX C500 / MACA 3.0 / sGPU 25% | [原生 C++ API、配额与复现报告](../platforms/metax/README.md)，[固定源码与原始结果清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/manifest.json)；cu-bridge 使用 CUDA 命名生成 MACA 设备代码，实际为分区算力 | +| 壁仞 Biren106M / SUPA 1.10 | [原生 API、BF16 舍入、Warp32 及毫秒性能表](../platforms/biren/README.md)，[完整矩阵与原始基准](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/manifest.json)，[交错留出实验](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/manifest.json)；使用 BRCC 和原生 `su*` 接口，不依赖 CUDA runtime | +| 昇腾 Ascend910B1 / CANN 9.0 | [原生异步 API、量化中点问题与完整复现](../platforms/ascend/README.md)，[原始探针、完整验证与交错A/B](https://github.com/a962695448-rgb/Learning-CUDA/tree/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend)；使用真实 NPU、ACL 与 Ascend C，16位存储接口独立于CUDA类型 | + +天数 Warp64 在实测 N=64/128/256、五档行数、两种精度的三轮中，变换相对本平台基础实现为 1.1907～3.3867 倍，融合路径为 1.5245～7.3599 倍。它们是同设备、同输入语义的事件区间比较；不与 NVIDIA 的 Graph 或端到端时间相除，也不宣称所有形状或其他国产设备具备同样收益。平台报告保留共享内存候选的退化样例和初始化故障修复记录。 + +沐曦在自己的相同规模矩阵中，Warp64 变换相对本平台基础实现为 1.0697～2.5394 倍，融合路径为 1.2900～6.0572 倍；Warp64 内部融合相比分步降低耗时 21.40%～41.46%。所有分母均来自同轮沐曦实测，不比较不同芯片或不同计算配额的绝对快慢；报告保留跨轮波动及共享内存候选的退化。 + +壁仞保留共享实现作为默认；小批量单 warp 发射在已测 rows=32/63/64、N=64、两种精度的融合路径中，相对共享实现每轮减少耗时 10.09%～12.68%。这不是全范围保证：N=256 和较大行数仍有显著负例，量化线程均衡实验也只作为可选消融。具体方法选择、原始失败、阈值外控制和计时边界均在该平台报告中说明。 + +昇腾的`--vector-scale`仅改变VectorGather的缩放计算，量化保持NPU标量除法；默认关闭。120个向量路径/形状/精度组合中97个在三轮每轮均减少耗时至少5%，其余含未达门槛与混合方向的结果。对已测M=257/4096/16384、N=64/128/256，两种精度的变换耗时减少40.89%～79.02%，融合减少21.97%～29.33%。这些是同block_dim=32的OFF/ON构建比较，不能解释为相对CPU或NVIDIA的加速；小输入与Vector对Scalar的负例均保留。 + +## 数学与接口约定 + +输入形状为 `[batch, seq, heads, dim]`，按行连续存储，前三维展平成 `rows=batch*seq*heads`。当前支持 `dim ∈ {1,2,4,8,16,32,64,128,256}`。 + +变换定义为：`y[row,j] = scale * sum_i H[j,i] * x[row,i]`，其中 `H[j,i] = (-1)^popcount(j & i)`。 + +- 默认 `scale=1`，为未归一化变换，和参考库默认 scale 对齐。 +- `--normalize` 或 `--scale normalized` 使用 `scale=1/sqrt(dim)`。 +- 输入和输出均为所选 FP16/BF16，内部加减与 Tensor Core 累加使用 FP32。 +- 当前输入由确定性生成器按形状和精度构造；自测包含边界形状、全零、脉冲、交替符号、随机、正态和异常值模式。该命令行程序不解析外部二进制张量文件。 +- 当前构建要求 sm80+，以同时验证 BF16 Tensor Core；RTX 4060/4090 为 sm89，A100 为 sm80。 + +## 量化约定 + +首版明确采用 per-row 对称 INT4，量化整数范围 `[-7,7]`： + +1. Hadamard 结果先舍入为输出 FP16/BF16。 +2. 每行 `s=max(abs(y))/7`；全零行固定 `s=1`。 +3. `q=clamp(round_to_nearest_even(y/s),-7,7)`。 +4. 每个字节放两个 4-bit 补码数：偶数元素在低半字节，奇数元素在高半字节。N=1 时高半字节为零。 + +融合路径刻意保留第 1 步的舍入，使其与“先变换写回，再量化”采用相同数值语义；不能为了减少一次舍入而破坏融合一致性。每行 scale 单独输出，packed 存储确实每字节两个数。 + +FP8、随机符号旋转和更长维度属于当前未实现的扩展，不计入已完成能力。 + +## 实现路径 + +- `naive_global`：先转 FP32，每个蝶形阶段启动一个 kernel，最后舍入写回。用于可读基线。 +- `warp`：每个 warp 处理一行；短距离蝶形使用 shuffle，跨 32 个元素的蝶形在每个线程的寄存器数组中完成。 +- `tensor_core`：WMMA 执行稠密 Hadamard 矩阵乘，作为不同算法的 Tensor Core 对照；只用于 dim>=16。 +- `split_int4`：warp 变换后再调用独立量化 kernel。 +- `fused_int4`：一条 warp kernel 完成变换、输出精度舍入、行最大值归约、量化及打包。 + +当前 Tensor Core 方法做了比 FWHT 更多的运算，而且有共享内存中转。在 4090 的大批量、dim=256 配置中明显慢于 warp；小配置的差距较小。该结果只适用于当前 WMMA 实现，不能外推到所有 Tensor Core Hadamard 算法。 + +## 构建和运行 + +Linux 或 WSL2，已配置 NVIDIA 驱动和 CUDA Toolkit： + +```bash +# 在本目录执行。4090 服务器使用 CUDA 12.8。 +export CUDA_HOME=/usr/local/cuda-12.8 +export PATH="$CUDA_HOME/bin:$PATH" +make CUDA_HOME="$CUDA_HOME" ARCH=89 +make cpu-test +./build/hadamard --self-test + +# 单一配置。日志文件记录实际硬件与测量范围。 +./build/hadamard --benchmark --batch 4 --seq 128 --heads 8 \ + --dim 256 --dtype fp16 --csv results/my-run.csv + +# 完整自测、非法参数检查和 16 组基准；label 不得和已有结果混用。 +python3 scripts/run_validation.py --label rtx4090-new-run --benchmark +``` + +A100 上重编： + +```bash +make CUDA_HOME=/usr/local/cuda ARCH=80 +./build/hadamard --self-test +python3 scripts/run_validation.py --label a100-first-run --benchmark +``` + +Makefile 会记录 NVCC 路径和编译参数；修改 ARCH 后自动重编，避免把 sm89 二进制直接当成 A100 构建。 + +## PyTorch 张量接口 + +实测环境为 Python `3.12.3`、PyTorch `2.6.0a0+ecf3bae40a.nv25.01`、PyTorch CUDA `12.8`。服务器已有可用 PyTorch 时，使用它构建扩展；无需为了本项目替换系统 PyTorch。 + +```bash +# 在本目录、已启用 CUDA PyTorch 的 Python 环境内执行。 +export MAX_JOBS=1 +export NVCC_THREADS=1 +export TORCH_CUDA_ARCH_LIST=8.9 +python scripts/build_torch_extension.py --verbose +``` + +第一次调用会编译，后续复用本目录 `build/torch_extension/` 的产物。以下例子可以直接保存为 Python 文件运行： + +```python +import torch +from scripts.build_torch_extension import load_extension + +op = load_extension() +x = torch.randn((2, 16), device="cuda", dtype=torch.float16) +y = op.hadamard(x, 1.0) # 只做变换，保持 x 的形状和 dtype +packed, row_scales = op.hadamard_int4(x, 1.0) # 融合变换和 INT4 量化 +split_packed, split_scales = op.quantize_int4(y) +assert torch.equal(packed, split_packed) +assert torch.equal(row_scales, split_scales) +``` + +输入必须是连续、非空的 CUDA FP16/BF16 张量，形状为 `[rows, dim]` 或 `[batch, seq, heads, dim]`，最后一维取前述 1～256 的二次幂。接口是前向计算，不支持 `requires_grad=True`。`packed` 的 dtype 为 `uint8`，最后一维为 `ceil(dim/2)`；`row_scales` 为 FP32，形状是输入去掉最后一维。接口使用调用者当前 CUDA stream。 + +带有 PyTorch 惰性负号标记的视图会明确拒绝，请先调用 `x.resolve_neg()` 将逻辑值物化，再传入接口。普通负数张量保持支持。 + +## N256 显式融合布局 + +```python +# x 的最后一维必须为256；旧三参数调用保持兼容。 +packed_original, scales_original = op.hadamard_int4(x, 1.0, 128) +packed, scales = op.hadamard_int4(x, 1.0, 128, fused_layout="contiguous256") +``` + +```bash +./build/hadamard --benchmark --batch 1 --seq 17 --heads 1 --dim 256 \ + --dtype fp16 --block-threads 128 --fused-layout contiguous256 --csv results/new-layout.csv +./build/hadamard --self-test --fused-layout contiguous256 +``` + +该选项只改变融合路径；独立变换和独立量化继续使用原 kernel。自测仍执行全矩阵，仅 N256 的融合项切换布局。CSV 增加第 21 列 `fused_layout`，仅融合行填写实际选项；旧表头拒绝混写，复现时使用新文件名。 + +## 小维度显式行布局 + +```python +# x 的最后一维为1/2/4/8/16；旧调用默认仍是original。 +y = op.hadamard(x, 1.0, 128, "auto") +packed, row_scales = op.hadamard_int4(x, 1.0, 128, "original", "auto") +y_packed = op.hadamard(x, 1.0, block_threads=256, row_layout="packed") +``` + +```bash +./build/hadamard --self-test --row-layout auto +./build/hadamard --benchmark --batch 1 --seq 4096 --heads 1 \ + --dim 8 --dtype fp16 --row-layout auto --csv results/rows-auto-new.csv +``` + +`packed`明确要求N≤16；auto可在已有规则中选择256线程，否则回退到调用者指定的128/256。自测的packed模式只切换N≤16，其他维度仍执行原实现。非original行布局不能与contiguous256组合。A100-SXM4-40GB也已完成独立规则验证。关键字写法仍支持，位置参数可减少所测调用开销；普通调用相对原默认仍有少量成本。自动阈值、未采用的N16融合和有限覆盖范围见[最新报告](../reports/a100-calling-validation.md)。 + +CSV默认original仍为21列；packed/auto采用24列，在末尾增加`requested_row_layout`、`resolved_row_layout`、`quantize_block_threads`。`warp_block_threads`记录实际选择的变换/融合线程，split的独立量化保留调用者线程并单列记录。不同表头拒绝混写，使用新的输出文件。 + +## 显式选择 NVIDIA 线程数 + +默认row_layout="original"时，省略线程参数仍使用128；显式auto的选择规则另见上一节。以下为早期128/256线程实验： 本次接口源码可定位至[`24849f6`](https://github.com/a962695448-rgb/Learning-CUDA/commit/24849f61ef06350f4e8bcd224ef93d97622c9744)；实机原字节与提交后的LF内容核查分别保存在下方归档。正确性已覆盖原全部dim=1～256的二次幂，性能证据仅覆盖N=16/64：变换M=4096/16384及各自M±1，融合INT4为M=4096及M±1。更一般的 M 范围和其他 GPU 仍需分别验证；已测 A100 的范围与结果见 [A100 报告](../reports/a100-validation.md)。 + +```python +# 保留原调用；x仍须满足前述CUDA/形状/连续存储约束。 +y_default = op.hadamard(x) # 默认128 +# scale仍为原来的第二个参数；block_threads追加为可选参数。 +y_256 = op.hadamard(x, 1.0, block_threads=256) +packed, row_scales = op.hadamard_int4(x, 1.0, block_threads=256) +split_packed, split_scales = op.quantize_int4(y_256, block_threads=256) +``` + +```bash +# 同一1,876组完整矩阵，明确选择256线程；默认命令仍使用128。 +./build/hadamard --self-test --block-threads 256 +./build/hadamard --benchmark --batch 4 --seq 128 --heads 8 \ + --dim 64 --dtype fp16 --block-threads 256 --csv results/threads256-new-run.csv + +# 独立构建缓存；复用原1,800组Dao矩阵比较默认/128/256，输出文件须为新文件。 +python scripts/verify_block_threads.py \ + --reference-repo /data/infinitensor-2026/fast-hadamard-transform \ + --build-directory /tmp/hadamard-thread-check-new-run \ + --json results/thread-api-check-new-run.json +``` + +CLI选项只影响warp变换、独立量化、split/fused和warp含复制路径；naive、Tensor Core和CPU路径不变。不支持的线程值会明确拒绝。输入检查、设备guard、当前CUDA stream和原来的输出精度/INT4舍入语义均保留。 + +计时日志同时显示us和ms。初次线程接口扩展时，CSV在原18列后追加`warp_block_threads`和`mean_ms`；当前original还含第21列`fused_layout`，packed/auto的24列另见上节。`mean_ms`严格按`mean_us/1000`换算；不涉及warp的行将线程字段留空。已有旧表头的CSV会被拒绝追加，请使用新的结果文件，避免不同格式混写。 + +[256线程独立Graph复核与原始源码](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_thread_promotion_20260905/README.md)保存三轮全部72配置及原始采样；[生产接口集成验证](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_api_integration_20260906/README.md)保存28个原始文件及对应源码SHA。集成后只复测了六个既有代表配置，256线程相对128减少耗时7.12%～25.34%，没有扩大性能搜索。Graph均摊时间仍不等于独立单kernel延迟或端到端时间；未知驻留CUDA上下文及单卡限制均在归档中说明。 + +## 固定第三方版本并复现比较 + +参考库固定为 Dao-AILab 的提交 `e7706faf8d1c3b9f241e36860640ad1dac644ede`。下面使用实测服务器的源码目录；已有干净的固定版本目录时，直接复用，不重复 clone。 + +```bash +REFERENCE_ROOT=/data/infinitensor-2026/fast-hadamard-transform +git clone https://github.com/Dao-AILab/fast-hadamard-transform.git "$REFERENCE_ROOT" +git -C "$REFERENCE_ROOT" checkout --detach e7706faf8d1c3b9f241e36860640ad1dac644ede + +# 使用同一个 Python/PyTorch 环境，保留其 C++ ABI 设置。 +export MAX_JOBS=1 +export NVCC_THREADS=1 +export FAST_HADAMARD_TRANSFORM_FORCE_BUILD=TRUE +export FAST_HADAMARD_TRANSFORM_SKIP_CUDA_BUILD=FALSE +python -m pip install --no-deps --no-build-isolation --no-cache-dir "$REFERENCE_ROOT" + +# 回到本项目目录；每轮结果使用新的文件名。 +python scripts/compare_reference.py --reference-repo "$REFERENCE_ROOT" \ + --benchmark --json results/third_party_rtx4090-new-run.json +``` + +参考库有自己的多架构编译参数；`TORCH_CUDA_ARCH_LIST=8.9` 只保证本项目扩展的目标，不会覆盖参考库写死的 `-gencode`。`MAX_JOBS=1` 与 `NVCC_THREADS=1` 限制编译并发，适合本次 20 GB 内存服务器。不要人为改成与已安装 PyTorch 不一致的 C++ ABI。 + +比较脚本检查固定源码提交、受跟踪文件是否干净、实际安装来源及 `.so` 哈希。两边使用完全相同的输入、dtype 和 FP32 `scale`：默认均为 1，归一化时均为 `1/sqrt(dim)`。本次 1,800 组对照包含两种精度、两种 scale、多种形状、4 种输入模式和 3 个随机种子，同时检查融合与分步 INT4 结果完全一致。 + +`third_party_rtx4090_eager.json` 保留初轮 eager 测量。增加 CUDA Graph 后重新验证了同一组 1,800 个正确性用例，再将 eager 和 Graph 两种口径保存为新的 `third_party_rtx4090_graph.json`；这不是新增 1,800 个独立用例。Graph 数据不会覆盖或混入初轮记录。 + +## 如何判定正确 + +- 独立 CPU 参考采用 FP64 稠密矩阵求和，没有复用 GPU 的蝶形实现；再按公开输出 dtype 舍入。 +- FP16 最大绝对误差严格 `<1e-2`，BF16 严格 `<5e-2`。 +- 小用例对所有元素作稠密比较；大批量选 32 行作独立稠密比较，但分步/融合量化仍覆盖全部元素。 +- 量化独立 oracle 对实际 warp 输出进行 CPU 量化，并与两个 GPU 路径逐字节、逐 scale 比较。 +- 若变换仅在允许的误差内接近稠密参考，不会把恰在量化阈值附近的离散差异误称为融合错误；此类差异单独记录。只有 warp 变换与稠密参考完全相同时，才强制要求稠密量化也精确相等。 +- CPU 专用测试还覆盖 Hadamard 二次变换还原、最近偶数舍入、补码打包及非法长度。 + +## 性能测量口径 + +- `kernel_only`：CUDA events，先预热、再多次执行取均值；排除分配、输入复制和 Hadamard 矩阵准备。 +- `cpu_compute`：优化编译的单线程 FP32 FWHT，使用主机时钟，不包含重置输入的复制。 +- `host_e2e`：已有缓冲区上的 pageable H2D + warp 变换 + D2H，用主机时钟计时。 +- 吞吐量为输入元素数/秒，不等于 FLOP/s，也不假称物理内存带宽。 +- 命令行扫描是单轮重复均值：每条 CUDA 路径 300 次，CPU 与含复制路径各 20 次。它不是跨次运行的置信区间。 +- 第三方 eager 比较在 CUDA event 区间内反复调用会分配输出的 PyTorch API，每组 200 次、共 5 组，取 5 个组均值的中位数。它排除了 H2D/D2H、编译和校验，但可能包含 Python 调用与 kernel 发射之间的 GPU 空闲时间，不能与命令行 `kernel_only` 列直接相除。 +- CUDA Graph 将每条路径的 64 次调用捕获到各自的图中，保留全部输出；每组重放 20 次，共 5 组，并交替两条路径的测量顺序。图重放使用固定输入与输出地址，计时除以每组 1,280 次调用，再取组间中位数。它去除了逐次 Python 发射和分配，但仍含捕获的 GPU 工作与均摊的图调度开销，**不是单个 kernel 的独立延迟**。 +- 旧 RTX 4060 Laptop 日志受 WSL/WDDM 调度影响，与本次 Linux 4090 结果分开保存。 + +4090 代表结果如下，单位均为 **微秒（us）**，`scale=1`。CPU 与 GPU 含复制两列用于观察把主机数据送入 GPU 是否划算；其余列用于比较 GPU 内部实现。 + +| 输入形状 | dtype | CPU FWHT | GPU 含复制 | warp | WMMA | 分步 INT4 | 融合 INT4 | +|---|---|---:|---:|---:|---:|---:|---:| +| `[4,128,8,256]` | FP16 | 2246.584 | 401.763 | 3.157 | 34.434 | 6.246 | 3.956 | +| `[4,128,8,256]` | BF16 | 2222.205 | 432.524 | 3.164 | 33.888 | 6.274 | 3.959 | +| `[1,1,17,256]` | FP16 | 9.244 | 11.065 | 2.068 | 7.772 | 3.993 | 2.270 | + +在 FP16 `[4,128,8,256]` 中，普通多次 kernel 基线 `naive_global` 为 `39.059 us`，warp 为 `3.157 us`；融合 INT4 相比本项目分步 INT4 约快 `1.58 倍`。当输入原本在 CPU 上时，含复制的 GPU 总时间约为 CPU FWHT 的 `1/5.59`，远小于仅看 GPU kernel 能算出的表面加速比。 + +小规模并不保证端到端收益：`[1,1,17,256]` FP16 的 CPU 为 `9.244 us`，GPU 含复制为 `11.065 us`。这组负例保留在同一 CSV 中；大规模收益不能外推到所有形状。 + +初轮第三方 eager PyTorch API 的 12 组 `Dao 时间 / 本项目时间` 为 `2.151～2.747`。例如 FP16 `[4,128,8,256]` 为本项目 `7.357 us`、Dao `15.846 us`。这说明当前接口及发射方式在该测量区间内更快，**不能证明纯 CUDA kernel 快了 2 倍**。 + +以下列出第二轮完整的 12 组 CUDA Graph 结果，单位为每次调用均摊的 **us**，`scale=1`。比值大于 1 表示本项目用时较少，小于 1 表示 Dao 用时较少。 + +| 输入形状 | dtype | 本项目 | Dao | Dao / 本项目 | +|---|---|---:|---:|---:| +| `[17,16]` | FP16 | 0.9912 | 1.0072 | 1.0161 | +| `[4,128,8,16]` | FP16 | 1.6976 | 3.2696 | 1.9260 | +| `[17,64]` | FP16 | 1.0152 | 1.0320 | 1.0165 | +| `[4,128,8,64]` | FP16 | 1.7383 | 3.3008 | 1.8989 | +| `[17,256]` | FP16 | 1.1256 | 1.0656 | **0.9467** | +| `[4,128,8,256]` | FP16 | 2.7920 | 3.3664 | 1.2057 | +| `[17,16]` | BF16 | 0.9912 | 1.0096 | 1.0186 | +| `[4,128,8,16]` | BF16 | 1.6984 | 3.2696 | 1.9251 | +| `[17,64]` | BF16 | 1.0144 | 1.0296 | 1.0150 | +| `[4,128,8,64]` | BF16 | 1.7368 | 3.3000 | 1.9000 | +| `[17,256]` | BF16 | 1.1464 | 1.0687 | **0.9322** | +| `[4,128,8,256]` | BF16 | 2.8104 | 3.3712 | 1.1995 | + +本轮大批量配置有明确的时间差，小规模 dim=16/64 的结果接近，不能据约 1% 的差距推广出稳定优势。`[17,256]` 的两种精度均是本项目更慢。更严格的 Graph 口径缩小了 eager 比较中的优势，说明前面的两倍左右比值不能简单归因于蝶形计算本身。上述12组仍是单卡、一次运行中的分组重复测量，没有跨设备或跨日期置信区间。 + +## Profiler 复现与结论边界 + +下面沿用实测日志的配置，只将输出改成新的文件名前缀，避免覆盖证据。在本项目目录、程序已编译后执行： + +```bash +nsys profile --trace=cuda,nvtx,osrt --sample=none --cpuctxsw=none \ + --output results/profile_rtx4090-new-run \ + ./build/hadamard --benchmark --batch 4 --seq 128 --heads 8 \ + --dim 256 --dtype fp16 --warmup 1 --repetitions 3 +nsys stats --report cuda_gpu_kern_sum,cuda_api_sum --format csv \ + results/profile_rtx4090-new-run.nsys-rep > results/profile_rtx4090-new-run_stats.txt + +ncu --set basic --kernel-name 'regex:warp_kernel' --launch-count 1 \ + --export results/profile_rtx4090-new-run_ncu \ + ./build/hadamard --benchmark --batch 4 --seq 128 --heads 8 \ + --dim 256 --dtype fp16 --warmup 1 --repetitions 3 +``` + +Nsight Systems 退出码为 0，原始 `.nsys-rep` 时间线保存在本地。此格式可能包含运行环境信息，因此公开仓库提供统计和日志，原始二进制不随代码发布。该次 **profiler 运行内部**的 kernel 统计如下： + +| 实际执行的 kernel | 实例数 | 中位数（us） | +|---|---:|---:| +| WMMA `tensor_core_kernel` | 5 | 34.400 | +| 只做变换的 `warp_kernel<...,true,false>` | 14 | 2.624 | +| 只做量化的 `warp_kernel<...,false,true>` | 5 | 2.496 | +| 融合变换量化的 `warp_kernel<...,true,true>` | 5 | 3.424 | + +统计证明上述路径均实际执行，也显示当前 WMMA 路径在该配置下耗时较长。这里聚合了整段 trace 中的同名实例，各路径实例数不同；不能将两个中位数相加当作一次分步调用延迟。这些数据受 profiler 与短测量配置影响，与前面的常规基准分开解读；日志里 profiler 附加后的程序计时也不能替代常规 CSV。 + +Nsight Compute 退出码为 1，明确报 `ERR_NVGPUCTRPERM`。当前容器没有目标 GPU 的性能计数器权限，需要算力平台开放权限后再测。尚未获得 ncu 的带宽利用率、occupancy 等硬件计数器结果；当前报告不据时间线推断这些指标已经测得。 + +## 实测证据 + +以下 4090 文件从服务器结果逐字节保存；文件大小、SHA-256 和对应提交见 [rtx4090_initial_manifest.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/rtx4090_initial_manifest.json) 与 [rtx4090_graph_profile_manifest.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/rtx4090_graph_profile_manifest.json)。保留完整数据，包括负例、容差内差异及 profiler 权限失败。 + +| 文件 | 内容 | +|---|---| +| [validation_rtx4090_initial.log](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/validation_rtx4090_initial.log) | 1,876 组自测、15 项非法参数及全部基准命令输出 | +| [benchmark_rtx4090_initial.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/benchmark_rtx4090_initial.csv) | 16 组配置、110 行 CPU/GPU 原始均值 | +| [third_party_rtx4090_eager.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/third_party_rtx4090_eager.json) | 1,800 组第三方对照、环境与二进制来源、12 组 eager 原始采样 | +| [third_party_rtx4090_graph.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/third_party_rtx4090_graph.json) | 同组正确性复测、第二轮 eager 与 12 组 CUDA Graph 原始区间及采样 | +| [profile_rtx4090_nsys.log](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/profile_rtx4090_nsys.log) | nsys 完整命令、退出码及程序输出 | +| [profile_rtx4090_nsys_stats.txt](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/profile_rtx4090_nsys_stats.txt) | CUDA kernel 与 API 调用统计 | +| [profile_rtx4090_ncu.log](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/profile_rtx4090_ncu.log) | ncu 完整命令与 `ERR_NVGPUCTRPERM` 失败证据 | + +早期 4060 证据仍在 `results/validation_sm89_expanded.log`、`results/benchmark_sm89_expanded.csv`、`results/build_sm89_dryrun.log`；初始 580 组 exact-grid 日志为 `results/self_test_sm89.log`。这些历史结果不能标成 4090 或 A100 数据。 + +## 后续验收 + +- 本轮A100-SXM4-40GB与4090的新版验证已完成；其他设备型号、更广形状与真实应用仍按新实验分别验证，不能继承本轮有限范围。九齿的A100记录在九齿仓库独立保存。 +- 在平台允许硬件计数器采集后补充 ncu；结合已取得的 nsys 时间线，进一步检查访存、寄存器与当前 WMMA 路径，评估分解 Hadamard 的 Tensor Core 算法。 +- 提交前核对训练营“包含测试”与通用“无测试代码”的措辞冲突；保留完整开发验证证据。 + +## 参考 + +- [本季度官方题目](https://github.com/InfiniTensor/Learning-CUDA/tree/2026-summer-project/03_hadamard_tc) +- [参考库及其 scale 约定](https://github.com/Dao-AILab/fast-hadamard-transform/blob/e7706faf8d1c3b9f241e36860640ad1dac644ede/README.md) + +本实现自行编写；参考库作为固定版本的独立 GPU 对照，不将其公开 A100 数据计作本项目实测。 diff --git a/03_hadamard_tc/a962695448-rgb/docs/optimization_candidates.md b/03_hadamard_tc/a962695448-rgb/docs/optimization_candidates.md new file mode 100644 index 00000000..355de117 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/docs/optimization_candidates.md @@ -0,0 +1,135 @@ +# Hadamard 下一轮优化：依据、实验与验收 + +记录日期：2026-09-05。本文是待执行实验方案，不代表已经取得新的性能提升。当前接口范围为连续 FP16/BF16 输入，末维为 1～256 的二次幂；先优化既有范围，再判断是否扩展。 + +## 先做什么 + +**首先实验小批量的 block 大小，随后做成对元素的向量化存取，最后试验分解式 Tensor Core。**第一项不改变变换数学与舍入过程，最容易验证收益和排除假设；第三项技术风险较高，应独立保留实验结果，不能先写入默认快路径。 + +当前 4090 足以启动三项实验。A100 用于跨架构重编、复测和九齿的官方验收；无需为 Hadamard 开发同时租多张 NVIDIA 卡。国产平台适配属于另一条交付线，不能替代这三个 CUDA 优化实验。 + +## 已有证据 + +数据来自仓库内的 [Graph 原始报告](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/third_party_rtx4090_graph.json)、[命令行 CSV](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/benchmark_rtx4090_initial.csv) 和 [profiler 权限日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/profile_rtx4090_ncu.log)。Graph 报告的源码提交记录在 [manifest](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/rtx4090_graph_profile_manifest.json) 中。 + +| 实测项 | FP16 | BF16 | 能支持的判断 | +|---|---:|---:|---| +| M=17、N=256，本项目 Graph 均摊 us | 1.1256 | 1.1464 | 两种精度都存在小批量负例 | +| 同形状 Dao Graph 均摊 us | 1.0656 | 1.0687 | 本项目分别慢约 5.6%、7.3% | +| M=4096、N=256，本项目 Graph 均摊 us | 2.7920 | 2.8104 | 大批量当前有优势，优化不能损伤它 | +| 同形状 Dao Graph 均摊 us | 3.3664 | 3.3712 | 当前优势约为 1.20 倍 | +| M=4096、N=256，warp kernel-only us | 3.157 | 3.164 | 与下一行采用同一种命令行计时口径 | +| 同形状 dense WMMA kernel-only us | 34.434 | 33.888 | 现有稠密 WMMA 明显较慢 | + +Graph 数值是多次捕获调用的均摊重放时间，不是单个 kernel 的独立延迟；不能和命令行 CSV 直接相除。初轮 eager API 的两倍左右优势也不能用于解释 kernel 的计算效率。 + +源代码可确认的结构：`src/main.cu` 和 `src/torch_binding.cu` 均固定 128 线程，即每 block 四个 warp、每 warp 一行。M=17 因此只有五个 block。N=256 时，每 lane 持有八个 FP32 值，五级 lane 间蝶形对每个值各执行一次 shuffle，随后三级蝶形在寄存器内完成。**这些是代码事实；占用率、带宽饱和与具体 stall 原因尚未测得。**现有 ncu 失败原因是 `ERR_NVGPUCTRPERM`。 + +## 候选一：小批量按形状选择 block 大小 + +**修改位置与映射：**保留 `warp_kernel` 本体；发射配置在 32、64、128 线程中离线比较，令 `rows_per_block=threads/32`，`grid=ceil(M/rows_per_block)`。M=17 对应 17、9、5 个 block，每行仍由一个完整 warp 处理。只在已验证的 `(N, M 范围, GPU 架构)` 上选择获胜配置；其余形状继续用原来的 128 线程。命令行与 PyTorch 包装必须共用同一派发规则，防止报告与实际 API 不一致。 + +**预期与不确定性：**减少每 block 的行数可能使少量行分布到更多 SM，改善五个 block 的并行度；但每行依赖链、总 warp 数与总运算不变，也可能受发射调度底限限制而完全没有收益。五个 block 本身不能证明这就是负例成因。 + +**数值约束：**线程内与 lane 间运算顺序、FP32 中间值、输出精度舍入、scale 与 INT4 打包全部不改；应逐位等于旧版输出，包括融合 bytes 和 scales。尾行退出仍必须是整个 warp 一起退出。 + +**有界实验：**先扫 `M={1,2,4,8,16,17,31,32,33,64,128}`、`N={64,128,256}`，两种 dtype、两种 scale。获胜候选再跑现有全部正确性用例和完整性能集。不要启动运行时自动调优或无限搜索。 + +**接受门槛：**在三个独立进程运行中，M=17/N=256 两种 dtype 的 Graph 中位时间均比各自同时测得的旧版低至少 5%，并尽量消除与 Dao 的差距；门槛是预设目标，不是已测收益。非目标形状若有超过 3% 且可复现的退化，只缩小派发范围。达不到门槛则保留旧派发并记录负实验，不为了“已经优化”而合入规则。 + +**资源：**现有 4090 即可；不要求 ncu 权限才能判定快慢。A100 必须独立选配置，不能把 sm89 的形状阈值直接当作 sm80 最优值。 + +## 候选二：成对存取与就地 INT4 打包 + +**修改位置与映射：**为 `N>=64` 增加独立实验 kernel，每个 lane 的第 k 对元素改为 `(2*lane+64*k, 2*lane+64*k+1)`，用一个 32-bit 存取装载两个 FP16/BF16，立即解包成两个 FP32。N=256 时每 lane 仍有八个 FP32 值，但只需四组成对装载/写回。先做对内 stride=1 蝶形；stride=2、4、8、16、32 对应 lane XOR 的 1、2、4、8、16；stride=64、128 在本线程的成对寄存器组之间完成。逐元素保留原版从低位到高位的蝶形顺序。 + +量化时同一线程已拥有相邻两个数，完成相同的 FP32 除法、最近偶数取整与 clamp 后可直接写一个 byte，省去原版为了取得相邻量化值而执行的 shuffle。不要把 FP16/BF16 的成对存取误写成低精度加法；所有算术仍在 FP32 中完成。不要用近似倒数乘法代替 `values/scale`，避免阈值附近的 INT4 不一致。 + +**必要边界：**连续张量可能有奇数 storage offset，不能仅凭 `is_contiguous()` 假定指针满足四字节对齐。检查输入/输出指针对齐后进入向量路径，否则走原标量路径；为偏移一元素的连续张量加测试。N=1/2/4/8/16/32 先保持现状，以限制首轮改动。 + +**预期与不确定性：**可能减少存取指令及融合量化的邻值交换。原版跨 lane 存取本来就连续，不能宣称这是“修复不合并访存”；新的映射也不保证减少变换 shuffle 数，N=256 仍为五级、每级八个 FP32 shuffle。寄存器数、代码大小和指令调度可能抵消收益，应检查 ptxas 输出和实际计时；没有 ncu 数据时不宣称解决带宽瓶颈。 + +**有界实验与门槛：**先只做 N=256、M=17/4096，分别测纯变换和 fused INT4。要求旧版、新版、CPU 独立量化 oracle 的融合 bytes/scales 完全一致，新旧纯变换逐位一致。至少一个目标路径在三个独立运行中稳定改善 5%，其他派发范围无可复现的超过 3% 退化，才扩展到 N=64/128；否则保留独立实验，不作为默认路径。不能只测纯变换再声称融合也变快。 + +**资源：**现有 4090、CUDA Toolkit 和现有测试环境即可。ncu 权限有助于解释寄存器、内存事务、指令和 stall，但不是执行实验的前提。A100 后续复测同样必需。 + +## 候选三：H16 Tensor Core 与 FP32 高位蝶形分解 + +**为何值得独立试验：**当前 dense WMMA 为 N 个输出计算 N 项点积，存在 O(N²) 运算；每个输出列 tile 还重复把 16 行输入搬入共享内存。现有 M=4096/N=256 大约 34 us 的时间支持淘汰它作为快路径，但不能证明 Tensor Core 这一硬件不适合 Hadamard。 + +**具体实验：**利用 `H_N = H_(N/16) ⊗ H_16`。一个 block 处理 16 行，协作装载一次 `[16,N]` 输入到共享内存。先对每行的每个连续 16 元素执行 `X_tile × H16`：一个 warp 负责一个 16×16 tile，使用输入 FP16/BF16、FP32 accumulator；N=256 时尝试八个 warp，各负责两个列 tile。将中间结果以 **FP32** 写入共享内存，block 同步后，由每个 warp 依次处理两行，执行 stride=16、32、64、128 的剩余 FP32 蝶形，最终仅一次舍入为输出 dtype。 + +这比把两个 H16 矩阵乘直接串起来更适合当前数值合同：第二次低精度 WMMA 若重新把中间 FP32 值转成 FP16/BF16，会引入额外舍入，因此本实验明确让剩余阶段用 FP32 蝶形完成。H16 系数为可精确表示的 ±1。N=256 的输入共享缓冲约 8 KiB、中间 FP32 缓冲约 16 KiB，另加少量 H16 存储；具体寄存器与共享内存资源以编译输出为准。 + +**数值约束：**WMMA 首阶段的求和顺序与旧版蝶形不同，不能预先承诺对旧版或 Dao 逐位相同。必须通过既有 FP16 `<1e-2`、BF16 `<5e-2` 的独立稠密误差检查，并记录差异；不提高容差。该路径若增加融合量化，必须先舍入为输出 dtype，然后以其自己的分步变换结果验证 packed bytes/scales 完全相等，不能拿另一种变换的微小数值差异掩盖融合错误。 + +**有界实验与门槛：**首轮只做 N=256、M=4096/65536 的两种 dtype、两种 scale；保留 M=17 尾行检查但不把它作为预期收益场景。先要求正确性和至少比当前 dense WMMA 快两倍，才继续实现/验证其融合量化。只有在三个独立运行中稳定比同形状 warp 快至少 5%，且完整正确性验证通过，才考虑默认派发。若仅优于旧 dense WMMA 而仍慢于 warp，结果可作为 Tensor Core 算法对照与消融，不能称为最快实现。 + +**资源：**4090 可开发；A100 对架构相关表现尤其有价值，无需 H100/5090 才能研究。ncu 对解释 Tensor Core 活跃度、共享内存与 barrier 开销最有帮助；没有权限时保留 nsys 与未插桩计时，明确因果解释的限制。 + +## 各候选共用的验证办法 + +保持原始提交与现有报告不动，为每个候选生成独立源码提交、构建参数、机器信息和新结果文件。正确性必须先过现有 1,876 组自测、1,800 组固定版本 Dao 对照，以及独立 CPU 量化/分步/融合精确一致检查；另外加入该候选新引入的边界。重复运行同一套用例不计为新增独立测试数量。 + +先用小集合筛选，再对选中版本做三个独立进程复测。每个进程按已有 Graph 方法保留原始组样本，轮换旧版/新版/Dao 的测量顺序，记录时钟、温度与是否存在其他 GPU 任务；对近似持平的结果不作胜出结论。Graph、eager、kernel-only、host-e2e 分别报告，profile 附加后的时间不得混入无 profiler 的性能表。扩展 M=65536 时另加循环使用不同输入缓冲的测量，区别固定地址缓存复用与更大工作集；两种实现必须使用同一条件。 + +**平台提供的最小硬条件：**可持续访问的 Linux NVIDIA 单卡实例；兼容的驱动、CUDA 编译环境与 PyTorch；代码和实验结果持久化空间。现有 4090 的 20 GB 主机内存可继续以单并发编译，若另选 A100，32 GB 以上主机内存、约 100 GB 可用持久化磁盘能减少依赖构建与双项目保存压力，属于便利配置而非算法最低要求。需要平台管理员允许 ncu GPU performance counters；仅在容器内取得 root 不保证权限有效。提前确认比盲目更换更昂贵的 GPU 更有价值。 + +不要因为三项候选存在就承诺“已优化到极致”。交付应同时呈现成功优化、负实验、未解决边界和相应复现命令，由证据决定最后保留哪些实现。 + +## 首轮发射实验程序 + +新增的 `src/tune_launch.cu` 直接调用现有 `warp_kernel`，不改变默认 API。首轮限于 `N={16,64,256}`、`M={1,17,4096,16384}`，两种 dtype、两种 scale、纯变换/融合 INT4 两种模式、32/64/128/256 线程。共 96 个形状/精度/scale/模式组合、384 个发射配置；各取五组原始数据，完整成功时 CSV 有 1,920 行样本。它用于筛选，尚未替代候选一中更密的小 M 验收扫描。 + +```bash +# 在本项目目录执行；A100 改为 -arch=sm_80。 +mkdir -p build +nvcc -O3 -std=c++17 -lineinfo -arch=sm_89 -Iinclude \ + src/tune_launch.cu -o build/tune_launch +# 使用新的文件名，保留 stderr 中的测量范围及最终 PASS/FAIL。 +./build/tune_launch > results/launch_sweep_4090_run1.csv \ + 2> results/launch_sweep_4090_run1.log +``` + +每个配置在计时前逐元素与 128 线程基线比较；输出先写入哨兵，避免旧值掩盖漏写。融合 packed bytes/scales 还与对基线实际舍入输出进行的 CPU 量化全量比较，变换对前最多四行另作 FP64 稠密检查。捕获后的实际结果在计时前后均再次校验。程序一旦发生不一致或 CUDA 错误就返回非零，不能把部分 CSV 当成整轮通过。 + +每张图包含 64 次顺序 kernel 调用，每组重放 20 次，计时除以 1,280；五组轮换起点和测量方向。所有配置重用同一组固定输入/输出缓冲，输出不相互依赖，但图内同一 stream 的调用保持顺序。分配、复制、编译与验证均不计时；该固定工作集可能受到缓存复用影响。**此程序与第三方比较脚本保留 64 个独立输出的图结构不同，只能用它内部的同时测量结果筛选发射参数；不能直接把两份报告的 us 相除。**选中候选后，仍需回到原始第三方 Graph 流程复测，并在独立进程和更大工作集下确认收益。 + +### 实验程序的同步缺陷修复 + +首轮 4090 扫描在部分用例后发生严格输出比较失败，整轮无效，不能据其部分 CSV 选择默认参数。本地 4060 复现时，失败位置会在变换精确比较与稠密参考检查之间变化。检查发现输入使用默认 stream 的 pageable `cudaMemcpy`，而 kernel 使用另一个 `cudaStreamNonBlocking` stream，没有建立复制完成到 kernel 读取之间的依赖。官方说明：pageable H2D 同步 API 返回时，暂存可能已完成，但到设备的 DMA 仍可能未完成。[CUDA API 同步语义](https://docs.nvidia.com/cuda/cuda-runtime-api/api-sync-behavior.html) + +修复仅将输入改为 `cudaMemcpyAsync(..., stream.handle)`，使 H2D、基线和候选 kernel 位于同一条显式 stream；复制仍在计时区间外。全部 `memcmp`、CPU 量化 oracle 和原容差保持不变。增加失败诊断的 dtype、M/N、scale、模式、阶段、线程数、首个差异索引、实际/预期数值及位模式。最终源码 SHA256 为 `8a9899f5180180752af20659aca31275279e3c2b1fc1d1baaea69a8b87198212`。 + +修复后本地 RTX 4060 Laptop / WSL 的完整 96 个用例、384 个配置检查通过，输出 1,920 个样本，退出码 0。此记录用于确认实验程序能正确执行，不计作 4090 性能成绩。随后 4090 已用修复版本完成下面的三轮扫描。现有产品 kernel 和正式 API 没有因本次 harness 修复而修改。 + +## 4090 三轮试验结果：只在明确范围内继续验证 + +**首轮筛选发现 24/96 个用例有稳定达到门槛的配置，全部为 256 线程；尚未修改生产派发。**这 24 个用例已经包含 dtype 和 scale 的不同组合，三次重复不计为 72 个独立用例，也不是新增算法正确性测试种类。完整实验覆盖 96 个用例、每轮 384 个配置、三轮共 5,760 个原始分组样本；三轮各自全部检查通过。 + +原始 CSV/log 保存在 [results/launch_tuning](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/launch_tuning/manifest.json)，从服务器导出的本地原件逐字节复制，SHA256 收录于清单。可用标准库脚本重算： + +```bash +python results/launch_tuning/analyze.py +``` + +[case_medians.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/launch_tuning/case_medians.csv) 保留全部 384 个配置的三轮中位时间、各轮 128 线程基线及成对比较；[summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/launch_tuning/summary.json) 包含稳定获胜配置、小 M 目标形状和明确负例。每轮每配置五个原始样本先取中位数，再分别对同轮基线计算 `时间减少百分比=100×(1−候选时间/128线程时间)`。只有**三轮全部至少减少 5% 时间**才记为本轮稳定获胜；没有把三轮样本混成一个中位数来掩盖某轮退化。 + +| 本轮可继续验证的精确范围 | 候选线程数 | 包含用例数 | 三轮、两精度、两 scale 的时间减少范围 | +|---|---:|---:|---:| +| transform，M=4096，N=16 | 256 | 4 | 14.48%～14.83% | +| transform，M=4096，N=64 | 256 | 4 | 9.26%～9.52% | +| transform，M=16384，N=16 | 256 | 4 | 25.64%～25.83% | +| transform，M=16384，N=64 | 256 | 4 | 13.89%～14.43% | +| fused_int4，M=4096，N=16 | 256 | 4 | 7.48%～9.29% | +| fused_int4，M=4096，N=64 | 256 | 4 | 8.87%～9.18% | + +这些是六个精确的 `(mode,M,N)` 组合，各覆盖 FP16/BF16 和 `scale=1/1/sqrt(N)`，**不能扩写为所有 M≥4096、所有 dim、全部融合路径都获益**。例如 fused_int4 的 M=16384 没有满足当前 5% 门槛。精确范围内三轮没有超过 3% 的退化，但范围外必须保留原配置，进一步的派发阈值需要补测邻近 M 值。 + +小批量目标尚未解决：M=17/N=256/scale=1 的 transform 改成 32 线程，FP16 三轮只减少约 2.93%～3.25%，BF16 约 2.94%～3.11%，低于预设门槛。FP16 第一轮为 0.9536 us 对 128 线程的 0.9840 us。这个差值属于本次固定缓冲图，不能拿它与前面第三方图的 1.1256/1.0656 us 直接比较，更不能因此宣称已经反超 Dao。 + +负例也说明不能全局改线程数：M=17/N=256/scale=1 的 FP16 transform 使用 256 线程时，三轮均比 128 线程慢 15.50%～15.70%；同形状 fused_int4 慢 6.77%～6.80%。M=4096/N=256/scale=1 的 FP16 transform 虽然第一轮从 2.4136 us 降为 2.3847 us，但三轮仅改善 1.13%～1.20%，不满足门槛。在 288 个非基线候选配置中，91 个配置三轮均出现超过 3% 的耗时增加，98 个配置至少一轮如此；全部保留在派生 CSV 中。 + +**测量范围与仍缺的条件：**三轮来自同一台 Linux RTX 4090 单卡服务器、同一次工作会话中的三个独立进程；CSV 实际记录 `sm89`、CUDA runtime `12080`。runner 确认测量时没有并行运行九齿 GPU 全量套件。本轮没有采集时钟、温度、功耗、硬件计数器，也没有跨日期和跨设备重复，不能声称已证明占用率原因、带宽瓶颈或获得严格置信区间。本轮未单独归档编译器/驱动版本，不将历史环境清单冒充本轮完整环境采样。 + +下一步先把上述六个精确组合接入独立实验调用入口，在保留 64 个独立输出的原始 Graph 条件、既有正确性流程和邻近 M 形状上复测；通过后才讨论生产派发。同时继续候选二的 N=256 成对存取与 INT4 打包，针对仍未达到门槛的小 M 问题。候选三保留为 Tensor Core 算法实验，不因当前发射配置结果而预先承诺收益。A100 到位后需要重编并重新选择配置。 diff --git a/03_hadamard_tc/a962695448-rgb/include/contiguous256.cuh b/03_hadamard_tc/a962695448-rgb/include/contiguous256.cuh new file mode 100644 index 00000000..5e940d50 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/include/contiguous256.cuh @@ -0,0 +1,108 @@ +#pragma once + +#include "kernels.cuh" + +namespace hadamard { + +template __device__ inline T storage_from_bits(unsigned short bits); +template <> __device__ inline __half storage_from_bits<__half>(unsigned short bits) { + return __ushort_as_half(bits); +} +template <> __device__ inline __nv_bfloat16 storage_from_bits<__nv_bfloat16>(unsigned short bits) { + return __ushort_as_bfloat16(bits); +} +template __device__ inline unsigned short storage_bits(T value); +template <> __device__ inline unsigned short storage_bits(__half value) { + return __half_as_ushort(value); +} +template <> __device__ inline unsigned short storage_bits(__nv_bfloat16 value) { + return __bfloat16_as_ushort(value); +} + +template +__global__ void contiguous256_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float transform_scale) { + const int lane = threadIdx.x % 32; + const std::size_t row = static_cast(blockIdx.x) * (blockDim.x / 32) + threadIdx.x / 32; + // The whole warp takes this guard before any row pointer is dereferenced. + if (row >= rows) return; + const T* source = input + row * 256 + lane * 8; + float values[8]; + if ((reinterpret_cast(source) & 15) == 0) { + const uint4 raw = *reinterpret_cast(source); + const unsigned int words[4] = {raw.x, raw.y, raw.z, raw.w}; +#pragma unroll + for (int k = 0; k < 8; ++k) + values[k] = as_float(storage_from_bits(static_cast(words[k / 2] >> (16 * (k % 2))))); + } else { + // Contiguous tensors may begin at a legal two-byte storage offset. +#pragma unroll + for (int k = 0; k < 8; ++k) values[k] = as_float(source[k]); + } + if constexpr (Transform) { + // Global index bits 0,1,2, followed by 3,4,5,6,7: same stage order. +#pragma unroll + for (int stride = 1; stride < 8; stride *= 2) { +#pragma unroll + for (int k = 0; k < 8; ++k) { + if (!(k & stride)) { + const float a = values[k], b = values[k + stride]; + values[k] = a + b; + values[k + stride] = a - b; + } + } + } +#pragma unroll + for (int stride = 1; stride < 32; stride *= 2) { +#pragma unroll + for (int k = 0; k < 8; ++k) { + const float mine = values[k]; + const float peer = __shfl_xor_sync(0xffffffff, mine, stride); + values[k] = (lane & stride) ? peer - mine : mine + peer; + } + } +#pragma unroll + for (int k = 0; k < 8; ++k) + values[k] = as_float(as_storage(values[k] * transform_scale)); + } + if constexpr (!Quantize) { + T* destination = output + row * 256 + lane * 8; + if ((reinterpret_cast(destination) & 15) == 0) { + unsigned int words[4]; +#pragma unroll + for (int k = 0; k < 4; ++k) + words[k] = static_cast(storage_bits(as_storage(values[k * 2]))) | + (static_cast(storage_bits(as_storage(values[k * 2 + 1]))) << 16); + *reinterpret_cast(destination) = make_uint4(words[0], words[1], words[2], words[3]); + } else { +#pragma unroll + for (int k = 0; k < 8; ++k) destination[k] = as_storage(values[k]); + } + } + if constexpr (Quantize) { + float magnitude = 0; +#pragma unroll + for (int k = 0; k < 8; ++k) magnitude = fmaxf(magnitude, fabsf(values[k])); +#pragma unroll + for (int stride = 16; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor_sync(0xffffffff, magnitude, stride)); + const float scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + if (lane == 0) scales[row] = scale; + unsigned int bytes = 0; +#pragma unroll + for (int pair = 0; pair < 4; ++pair) { + const int low = max(-7, min(7, __float2int_rn(values[pair * 2] / scale))); + const int high = max(-7, min(7, __float2int_rn(values[pair * 2 + 1] / scale))); + bytes |= static_cast((low & 15) | ((high & 15) << 4)) << (pair * 8); + } + std::uint8_t* destination = packed + row * 128 + lane * 4; + if ((reinterpret_cast(destination) & 3) == 0) { + *reinterpret_cast(destination) = bytes; + } else { +#pragma unroll + for (int k = 0; k < 4; ++k) destination[k] = static_cast(bytes >> (k * 8)); + } + } +} + +} // namespace hadamard diff --git a/03_hadamard_tc/a962695448-rgb/include/kernels.cuh b/03_hadamard_tc/a962695448-rgb/include/kernels.cuh new file mode 100644 index 00000000..4618f3fd --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/include/kernels.cuh @@ -0,0 +1,138 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace hadamard { + +template __host__ __device__ float as_float(T value); +template <> __host__ __device__ inline float as_float(__half value) { return __half2float(value); } +template <> __host__ __device__ inline float as_float(__nv_bfloat16 value) { return __bfloat162float(value); } +template __host__ __device__ T as_storage(float value); +template <> __host__ __device__ inline __half as_storage(float value) { return __float2half_rn(value); } +template <> __host__ __device__ inline __nv_bfloat16 as_storage(float value) { return __float2bfloat16_rn(value); } + +template +__global__ void to_float_kernel(const T* input, float* output, std::size_t size) { + const std::size_t i = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (i < size) output[i] = as_float(input[i]); +} + +__global__ void butterfly_stage(const float* input, float* output, std::size_t size, int stride) { + const std::size_t i = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (i < size) { + const float mine = input[i], peer = input[i ^ stride]; + output[i] = (i & stride) ? peer - mine : mine + peer; + } +} + +template +__global__ void from_float_kernel(const float* input, T* output, std::size_t size, float scale) { + const std::size_t i = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (i < size) output[i] = as_storage(input[i] * scale); +} + +template +__global__ void warp_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float transform_scale) { + constexpr int Items = (N + 31) / 32; + const int lane = threadIdx.x % 32; + const std::size_t row = static_cast(blockIdx.x) * (blockDim.x / 32) + threadIdx.x / 32; + // An entire warp exits together: every shuffle below has a full active warp. + if (row >= rows) return; + float values[Items]; +#pragma unroll + for (int k = 0; k < Items; ++k) + values[k] = lane + 32 * k < N ? as_float(input[row * N + lane + 32 * k]) : 0; + if constexpr (Transform) { +#pragma unroll + for (int stride = 1; stride < (N < 32 ? N : 32); stride *= 2) { +#pragma unroll + for (int k = 0; k < Items; ++k) { + const float mine = values[k]; + const float peer = __shfl_xor_sync(0xffffffff, mine, stride); + values[k] = (lane & stride) ? peer - mine : mine + peer; + } + } +#pragma unroll + for (int stride = 1; stride < Items; stride *= 2) { +#pragma unroll + for (int k = 0; k < Items; ++k) { + if (!(k & stride)) { + const float a = values[k], b = values[k + stride]; + values[k] = a + b; + values[k + stride] = a - b; + } + } + } + } + // Preserve the exact transform-then-quantize contract, including output rounding. +#pragma unroll + for (int k = 0; k < Items; ++k) { + if (lane + 32 * k < N) { + if constexpr (Transform) values[k] = as_float(as_storage(values[k] * transform_scale)); + if constexpr (!Quantize) output[row * N + lane + 32 * k] = as_storage(values[k]); + } else values[k] = 0; + } + if constexpr (Quantize) { + float magnitude = 0; +#pragma unroll + for (int k = 0; k < Items; ++k) magnitude = fmaxf(magnitude, fabsf(values[k])); +#pragma unroll + for (int stride = 16; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor_sync(0xffffffff, magnitude, stride)); + const float scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + if (lane == 0) scales[row] = scale; +#pragma unroll + for (int k = 0; k < Items; ++k) { + const int q = max(-7, min(7, __float2int_rn(values[k] / scale))); + const int next = __shfl_down_sync(0xffffffff, q, 1); + const int index = lane + 32 * k; + if (!(lane & 1) && index < N) { + const int high = index + 1 < N ? (next & 15) : 0; + packed[row * ((N + 1) / 2) + index / 2] = static_cast((q & 15) | (high << 4)); + } + } + } +} + +// Dense H_N multiplication is an intentionally distinct Tensor Core algorithm. +// Compare it to FWHT experimentally; using Tensor Cores does not imply a speedup. +template +__global__ void tensor_core_kernel(const T* input, const T* matrix, T* output, + std::size_t rows, float scale) { + using namespace nvcuda; + __shared__ __align__(32) T a[16 * N]; + __shared__ __align__(32) float c[16 * 16]; + const std::size_t row_start = static_cast(blockIdx.x) * 16; + const int column_start = blockIdx.y * 16; + for (int i = threadIdx.x; i < 16 * N; i += blockDim.x) { + const std::size_t row = row_start + i / N; + a[i] = row < rows ? input[row * N + i % N] : as_storage(0); + } + __syncthreads(); + if (threadIdx.x < 32) { + wmma::fragment af; + wmma::fragment bf; + wmma::fragment cf; + wmma::fill_fragment(cf, 0.0f); +#pragma unroll + for (int k = 0; k < N; k += 16) { + wmma::load_matrix_sync(af, a + k, N); + wmma::load_matrix_sync(bf, matrix + k * N + column_start, N); + wmma::mma_sync(cf, af, bf, cf); + } + wmma::store_matrix_sync(c, cf, 16, wmma::mem_row_major); + } + __syncthreads(); + for (int i = threadIdx.x; i < 256; i += blockDim.x) { + const std::size_t row = row_start + i / 16; + if (row < rows) output[row * N + column_start + i % 16] = as_storage(c[i] * scale); + } +} + +} // namespace hadamard diff --git a/03_hadamard_tc/a962695448-rgb/include/packed_pairs.cuh b/03_hadamard_tc/a962695448-rgb/include/packed_pairs.cuh new file mode 100644 index 00000000..5314139f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/include/packed_pairs.cuh @@ -0,0 +1,33 @@ +#pragma once + +#include "kernels.cuh" + +namespace hadamard { + +// Each thread owns adjacent input values and writes their complete output byte. +// Scalar input loads retain the public contract for legal two-byte offsets. +template +__global__ void quantize_pairs_kernel(const T* input, std::uint8_t* packed, + float* scales, std::size_t rows) { + static_assert(N >= 2 && N <= 16 && (N & (N - 1)) == 0); + constexpr int Lanes = N / 2; + const std::size_t pair = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const std::size_t row = pair / Lanes; + const int lane = threadIdx.x % Lanes; + const bool valid = row < rows; + const float low = valid ? as_float(input[pair * 2]) : 0.0f; + const float high = valid ? as_float(input[pair * 2 + 1]) : 0.0f; + float magnitude = fmaxf(fabsf(low), fabsf(high)); +#pragma unroll + for (int stride = Lanes / 2; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor_sync(0xffffffff, magnitude, stride, Lanes)); + const float scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + const int q_low = max(-7, min(7, __float2int_rn(low / scale))); + const int q_high = max(-7, min(7, __float2int_rn(high / scale))); + if (valid) { + packed[pair] = static_cast((q_low & 15) | ((q_high & 15) << 4)); + if (lane == 0) scales[row] = scale; + } +} + +} // namespace hadamard diff --git a/03_hadamard_tc/a962695448-rgb/include/packed_rows.cuh b/03_hadamard_tc/a962695448-rgb/include/packed_rows.cuh new file mode 100644 index 00000000..ca599d39 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/include/packed_rows.cuh @@ -0,0 +1,44 @@ +#pragma once + +#include "kernels.cuh" + +namespace hadamard { + +template +__global__ void packed_rows_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float transform_scale) { + static_assert(N >= 1 && N <= 16 && (N & (N - 1)) == 0); + const std::size_t element = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const std::size_t row = element / N; + const int lane = threadIdx.x % N; + const bool valid = row < rows; + // Every lane reaches all shuffles, including complete masked subgroups at the tail. + float value = valid ? as_float(input[element]) : 0.0f; + if constexpr (Transform) { +#pragma unroll + for (int stride = 1; stride < N; stride *= 2) { + const float peer = __shfl_xor_sync(0xffffffff, value, stride, N); + value = (lane & stride) ? peer - value : value + peer; + } + value = as_float(as_storage(value * transform_scale)); + } + if constexpr (!Quantize) { + if (valid) output[element] = as_storage(value); + } else { + float magnitude = fabsf(value); +#pragma unroll + for (int stride = N / 2; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor_sync(0xffffffff, magnitude, stride, N)); + const float scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + if (valid && lane == 0) scales[row] = scale; + const int q = max(-7, min(7, __float2int_rn(value / scale))); + int next = 0; + if constexpr (N > 1) next = __shfl_down_sync(0xffffffff, q, 1, N); + if (valid && !(lane & 1)) { + const int high = N > 1 ? (next & 15) : 0; + packed[row * ((N + 1) / 2) + lane / 2] = static_cast((q & 15) | (high << 4)); + } + } +} + +} // namespace hadamard diff --git a/03_hadamard_tc/a962695448-rgb/include/reference.hpp b/03_hadamard_tc/a962695448-rgb/include/reference.hpp new file mode 100644 index 00000000..e06ce3fe --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/include/reference.hpp @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace hadamard { + +inline bool power_of_two(std::size_t n) { return n && !(n & (n - 1)); } + +// Sylvester ordering. The public transform is unnormalized unless scale is set. +template +void fwht(F* values, std::size_t rows, std::size_t dim, F scale = F(1)) { + if (!power_of_two(dim)) throw std::invalid_argument("dim must be a power of two"); + for (std::size_t row = 0; row < rows; ++row) { + F* x = values + row * dim; + for (std::size_t stride = 1; stride < dim; stride *= 2) { + for (std::size_t base = 0; base < dim; base += 2 * stride) { + for (std::size_t j = 0; j < stride; ++j) { + const F a = x[base + j], b = x[base + j + stride]; + x[base + j] = a + b; + x[base + j + stride] = a - b; + } + } + } + for (std::size_t j = 0; j < dim; ++j) x[j] *= scale; + } +} + +// Independent O(N^2) oracle: no butterfly code is shared with the kernels. +inline std::vector dense_reference(const std::vector& x, + std::size_t dim, double scale = 1) { + if (!power_of_two(dim) || x.size() % dim) + throw std::invalid_argument("invalid reference shape"); + std::vector y(x.size(), 0.0); + for (std::size_t row = 0; row < x.size() / dim; ++row) { + for (std::size_t out = 0; out < dim; ++out) { + double sum = 0; + for (std::size_t in = 0; in < dim; ++in) { + auto bits = out & in; + bool odd = false; + while (bits) { odd = !odd; bits &= bits - 1; } + sum += (odd ? -1.0 : 1.0) * x[row * dim + in]; + } + y[row * dim + out] = sum * scale; + } + } + return y; +} + +// Round-to-nearest, ties-to-even, independent of the host rounding mode. +inline int nearest_even(float value) { + const float lower = std::floor(value); + const float fraction = value - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + return q; +} + +struct Int4Result { + std::vector packed; + std::vector scales; +}; + +// Per-row symmetric INT4, [-7,7]; even element in low nibble. Zero row: scale=1. +// Input must already be rounded to the transform's FP16/BF16 output dtype. +inline Int4Result quantize_int4(const std::vector& values, std::size_t dim) { + if (!dim || values.size() % dim) throw std::invalid_argument("invalid quantization shape"); + const std::size_t rows = values.size() / dim, bytes = (dim + 1) / 2; + Int4Result out{std::vector(rows * bytes, 0), std::vector(rows)}; + for (std::size_t r = 0; r < rows; ++r) { + float magnitude = 0; + for (std::size_t j = 0; j < dim; ++j) { + if (!std::isfinite(values[r * dim + j])) + throw std::invalid_argument("INT4 quantization requires finite values"); + magnitude = std::max(magnitude, std::abs(values[r * dim + j])); + } + const float scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + out.scales[r] = scale; + for (std::size_t j = 0; j < dim; ++j) { + const int q = std::clamp(nearest_even(values[r * dim + j] / scale), -7, 7); + out.packed[r * bytes + j / 2] |= static_cast((q & 15) << (4 * (j % 2))); + } + } + return out; +} + +} // namespace hadamard diff --git a/03_hadamard_tc/a962695448-rgb/include/row_policy.hpp b/03_hadamard_tc/a962695448-rgb/include/row_policy.hpp new file mode 100644 index 00000000..84f9fce5 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/include/row_policy.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include + +namespace hadamard { + +enum class RowLayout { Original, Packed, Auto }; +enum class RowDevice { Unknown, A800, RTX4090, A100, RTX4090D }; +struct RowChoice { bool packed; int threads; }; + +inline RowLayout parse_row_layout(const std::string& name) { + if (name == "original") return RowLayout::Original; + if (name == "packed") return RowLayout::Packed; + if (name == "auto") return RowLayout::Auto; + throw std::invalid_argument("row_layout must be original, packed, or auto"); +} + +inline RowDevice row_device(const char* name) { + if (std::strcmp(name, "NVIDIA A800-SXM4-40GB") == 0) return RowDevice::A800; + if (std::strcmp(name, "NVIDIA GeForce RTX 4090") == 0) return RowDevice::RTX4090; + if (std::strcmp(name, "NVIDIA A100-SXM4-40GB") == 0) return RowDevice::A100; + if (std::strcmp(name, "NVIDIA GeForce RTX 4090 D") == 0) return RowDevice::RTX4090D; + return RowDevice::Unknown; +} + +inline RowChoice choose_rows(RowLayout layout, RowDevice device, std::size_t rows, + int dim, bool fused, int fallback_threads) { + if (layout == RowLayout::Original || dim < 1 || dim > 16 || (dim & (dim - 1))) + return {false, fallback_threads}; + if (layout == RowLayout::Packed) return {true, fallback_threads}; + // This model has separate measured bounds; do not inherit the 4090 rules. + if (device == RowDevice::RTX4090D) + return rows >= 4096 && rows <= 65536 ? RowChoice{true, 256} + : RowChoice{false, fallback_threads}; + int index = 0; + for (int n = dim; n > 1; n /= 2) ++index; + // Zero disables a rule. Only independently accepted device-specific ranges are installed. + // BEGIN GENERATED RULES + constexpr std::size_t a800_transform[5] = {4096, 4096, 4096, 4096, 4096}; + constexpr std::size_t a800_fused[5] = {1, 1, 16, 256, 0}; + constexpr std::size_t rtx4090_transform[5] = {4096, 4096, 4096, 4096, 4096}; + constexpr std::size_t rtx4090_fused[5] = {1, 1, 256, 64, 0}; + constexpr std::size_t a100_transform[5] = {4096, 256, 4096, 4096, 4096}; + constexpr std::size_t a100_fused[5] = {1, 1, 16, 64, 0}; + // END GENERATED RULES + std::size_t minimum = 0; + if (device == RowDevice::A800) minimum = (fused ? a800_fused : a800_transform)[index]; + if (device == RowDevice::RTX4090) minimum = (fused ? rtx4090_fused : rtx4090_transform)[index]; + if (device == RowDevice::A100) minimum = (fused ? a100_fused : a100_transform)[index]; + return minimum && rows >= minimum ? RowChoice{true, 256} : RowChoice{false, fallback_threads}; +} + +} // namespace hadamard diff --git a/03_hadamard_tc/a962695448-rgb/platforms/ascend/CMakeLists.txt b/03_hadamard_tc/a962695448-rgb/platforms/ascend/CMakeLists.txt new file mode 100644 index 00000000..993c61ac --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/ascend/CMakeLists.txt @@ -0,0 +1,41 @@ +cmake_minimum_required(VERSION 3.16) +project(hadamard_ascend LANGUAGES C CXX) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(ASCEND_CANN_PACKAGE_PATH "/usr/local/Ascend/cann-9.0.0" CACHE PATH "Actual CANN installation") +set(SOC_VERSION "Ascend910B1" CACHE STRING "Verified target SOC") +set(RUN_MODE "npu" CACHE STRING "Real NPU execution") +if(NOT RUN_MODE STREQUAL "npu") + message(FATAL_ERROR "This backend requires RUN_MODE=npu; CPU/simulator results are not NPU evidence") +endif() +set(ASCENDC_CMAKE_FILE "${ASCEND_CANN_PACKAGE_PATH}/aarch64-linux/tikcpp/ascendc_kernel_cmake/ascendc.cmake" + CACHE FILEPATH "Installed SDK Kernel Launch entry") +if(NOT EXISTS "${ASCENDC_CMAKE_FILE}") + message(FATAL_ERROR "Missing Ascend C CMake entry: ${ASCENDC_CMAKE_FILE}") +endif() +include("${ASCENDC_CMAKE_FILE}") + +option(ENABLE_VECTOR_SCALE "Experiment: use vector Muls for VectorGather scale only" OFF) +if(ENABLE_VECTOR_SCALE) + set(HADAMARD_VECTOR_SCALE_VALUE 1) +else() + set(HADAMARD_VECTOR_SCALE_VALUE 0) +endif() +ascendc_library(hadamard_ascend_kernels STATIC hadamard_kernel.cpp) +ascendc_compile_definitions(hadamard_ascend_kernels PRIVATE + HADAMARD_ASCEND_VECTOR_SCALE=${HADAMARD_VECTOR_SCALE_VALUE}) +add_library(hadamard_ascend STATIC hadamard_api.cpp) +target_include_directories(hadamard_ascend PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") +target_link_libraries(hadamard_ascend PUBLIC hadamard_ascend_kernels host_intf_pub) +add_dependencies(hadamard_ascend hadamard_ascend_kernels) + +option(BUILD_VALIDATION "Build standalone validation and benchmark executable" ON) +if(BUILD_VALIDATION) + add_executable(validate_and_benchmark validate_and_benchmark.cpp) + target_compile_definitions(validate_and_benchmark PRIVATE + HADAMARD_ASCEND_VECTOR_SCALE=${HADAMARD_VECTOR_SCALE_VALUE}) + target_include_directories(validate_and_benchmark PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../include") + target_compile_options(validate_and_benchmark PRIVATE -fno-fast-math -ffp-contract=off -frounding-math) + target_link_libraries(validate_and_benchmark PRIVATE hadamard_ascend) + add_dependencies(validate_and_benchmark hadamard_ascend) +endif() diff --git a/03_hadamard_tc/a962695448-rgb/platforms/ascend/README.md b/03_hadamard_tc/a962695448-rgb/platforms/ascend/README.md new file mode 100644 index 00000000..ad9e1dd9 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/ascend/README.md @@ -0,0 +1,200 @@ +# 昇腾 910B1 原生 Ascend C 后端 + +本目录实现原生 Ascend C/CANN Hadamard 变换和融合 INT4。固定源码 `f614762a69e75524db65b47fbf7d6d01836db438` 的 vector-scale OFF/ON 两种构建，均已在真实 Ascend910B1 完成主矩阵、grid 专项、大行数变换专项和接口检查；源码、日志和配置分别保存，重复矩阵不相加。 + +严格交错 OFF/ON 对照和独立统计复核已完成。120 个向量路径/形状/精度条件中，97 个在三轮每轮都减少至少 5% 耗时，11 个三轮均更快但未全部达到 5%，12 个方向混合；不能宣称所有条件都更快或应用整体提速。`--vector-scale` 默认保持 OFF，建议只在已测适用范围显式启用并复核。本文是实验与交付记录，不是设备释放回执。 + +## 实机环境与原生执行路径 + +| 项目 | 已核实信息 | +|---|---| +| 实际 SoC | `Ascend910B1`，ACL SoC 查询及 NPU 日志相符 | +| CANN | `/usr/local/Ascend/cann-9.0.0` | +| 主机 | aarch64；Python 3.11.15，GCC 12.3.1/openEuler 工具链 | +| 设备工具 | `npu-smi 25.5.1`，初始设备状态 Health=OK | +| 初始 HBM 报告 | 65536 MB 总量；设备状态快照不证明独占、固定频率或无其他负载 | +| 构建 | CMake + CANN 自带 Ascend C kernel CMake，目标 `Ascend910B1`、`RUN_MODE=npu` | + +[初始环境与源码清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/initial_probes/manifest.json) 保留设备查询、目录信息及自写探针。实现直接使用 ACL caller stream 与 NPU kernel；`ScalarButterfly` 中的“标量”同样在 NPU 上执行，**不是 CPU 回退**。CPU 代码只负责输入、独立参考、检查和控制流程，不代替设备输出。 + +项目只引用已安装 CANN 的 `kernel_operator.h`、ACL 和 Ascend C CMake 接口，不复制私人 SDK 头文件实现。SDK 产生的内核对象、launch 头文件和构建缓存位于忽略的 `build/ascend/`，不混进公开源代码或结果档案。 + +## 两种变换与量化的数值约定 + +- `Method::ScalarButterfly`：NPU 标量蝶形基线。 +- `Method::VectorGather`：使用 NPU Gather/矢量操作实现蝶形，内部 FP32。 +- `--vector-scale`：仅将 `VectorGather` 变换后的乘 scale 步骤改为矢量 `Muls`；默认关闭。它不改变下面的量化除法，也不是重新启用矢量 `Div`。 + +输入/输出为 FP16/BF16，内部计算使用 FP32。Hadamard 支持 N=1～256 的 2 次幂，常用 transform scale 为 1 或 `1/sqrt(N)`。每行 INT4 为 `[-7,7]`，量化 scale 为 `max(abs(x))/7`、全零行取 1；采用最近偶数舍入，偶数元素存低四位，N=1 的空高四位为零。融合路径先舍入到公开 FP16/BF16 存储类型，再读回 FP32 量化，保持分步语义。 + +### 为什么量化没有使用矢量 Div + +独立除法探针在同一实机上发现：矢量 `Div` 的 256 个输入中有 64 个结果与项目 FP32 位级参考不同，而 NPU 标量 C++ `/` 的 256 个结果全部逐位相同。输入保持与哨兵检查通过。原始 [矢量日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/division_probe/div_vector_run.log)、[两路径日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/division_probe/div_scalar_run.log)、[配置/源码与清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/division_probe/manifest.json) 完整保留。 + +其中一个影响 INT4 的边界是：参考除法得到 `3.49999976`，矢量结果变成 `3.5`。随后最近偶数舍入会分别得到 3 和 4，不能只因浮点误差很小就判定量化等价。生产实现因此使用已经单独核验的 **NPU 标量除法**计算量化 scale 和 `x/scale`，不采用该矢量 Div 或近似倒数替代。 + +探针控制器的 `PROBE_FINISHED` 只表示流程结束；矢量运行退出 1，启用标量的 `--mode both` 运行也因仍包含矢量失败而退出 1。必须分别读取子路径结果，不能把整批称为 PASS，也不能将该精度契约差异直接称为厂商 bug。 + +`f614762` 增加固定串联回归：输入 `[0.75,0.25]` 的未归一化变换为 `[1,0.5]`，量化 scale 的 FP32 位模式应为 `0x3e124925`，packed 输出应为 `0x37`。两 dtype、两 Method 各一例，共四个合同检查，分步/融合及哨兵均通过；它们增加在 API 检查计数中，不把例内的多次调用拆成多个用例。 + +## 公共 C++ API + +接口与实现分别见 [hadamard_api.h](hadamard_api.h)、[hadamard_api.cpp](hadamard_api.cpp) 和 [hadamard_kernel.cpp](hadamard_kernel.cpp)，命名空间为 `hadamard::ascend`。 + +| 入口 | 行为 | +|---|---| +| `transform` | 变换至同 dtype 输出，允许完全原位 | +| `quantize_int4` | 量化已经存储为 FP16/BF16 的输出；两种 Method 共用同一个 NPU 量化内核 | +| `transform_int4` | 融合变换与量化，输出 packed bytes 和 float scales | + +`StorageType::FP16/BF16` 指定类型，输入/变换输出使用 `std::uint16_t*` 承载对应原始位模式。连续 `[batch,seq,heads,head_dim]` 经溢出检查后展平为 `[rows,n]`，`rows=batch*seq*heads`、`n=head_dim`；不接受任意 stride 或自动重排非连续视图。 + +调用方分配当前设备上容量足够的缓冲区：输入/变换输出各 `rows*n` 个 16 位元素，packed 为 `rows*ceil(n/2)` 字节，scale 为 `rows` 个 float。输入及变换输出至少 2 字节对齐、scales 至少 4 字节，packed 无额外地址对齐要求。transform 只允许完全原位或互不重叠；量化涉及的各缓冲区必须互不重叠。 + +返回值为 `aclError`，参数/发射错误需立即检查。非空调用只在传入的 `aclrtStream` 发射，不分配、不复制、不等待；异步执行错误在调用方同步 stream/event 时检查。`rows=0` 在检查枚举、N、scale、block_dim 后直接返回，允许空指针/空 stream，不发射内核。 + +`block_dim` 支持 1～32，默认 1。它是本实现暴露的启动参数，不代表最优值，也不能直接等同其他 CUDA 平台的线程块含义。调用方保证 stream、设备归属、容量和生命周期；API 不扫描设备数据，输入与 FP32 中间值应有限，结果应在目标类型的有限范围内。 + +## 构建和复现命令 + +从项目 `03_hadamard_tc/a962695448-rgb` 目录、配套 aarch64 CANN 环境执行。结果目录和关联构建目录必须是新目录。runner 仅在子进程中加载已安装 CANN `set_env.sh` 并设置必要搜索路径，不安装或替换驱动/框架，也不将完整环境变量写进结果。 + +```bash +# 快速检查与 M17/N128 的单点计时，只用于试跑。 +python3 platforms/ascend/run_platform.py \ + --cann-root /usr/local/Ascend/cann-9.0.0 --quick --pilot-benchmark \ + --output results/ascend/reproduce_quick_01 + +# 固定 f614762 源码，vector-scale OFF：完整验证，默认不做基准。 +python3 platforms/ascend/run_platform.py \ + --cann-root /usr/local/Ascend/cann-9.0.0 \ + --output results/ascend/reproduce_scale_off_01 + +# 相同源码，vector-scale ON：单独完整验证。 +python3 platforms/ascend/run_platform.py \ + --cann-root /usr/local/Ascend/cann-9.0.0 --vector-scale \ + --output results/ascend/reproduce_scale_on_01 +``` + +[CMakeLists.txt](CMakeLists.txt) 要求 `RUN_MODE=npu`，默认 `SOC_VERSION=Ascend910B1`；使用实际安装的 `aarch64-linux/tikcpp/ascendc_kernel_cmake/ascendc.cmake`。`ENABLE_VECTOR_SCALE` 默认 OFF，runner 的 `--vector-scale` 会将它设为 ON，并核对生成 CMakeCache 与结果 JSON 的开关状态，防止测试到另一种构建。 + +runner [run_platform.py](run_platform.py) 的主验证 block_dim 默认 1，可用 `--block-dim` 指定；只有显式加 `--benchmark` 或 `--pilot-benchmark` 才计时。普通小基准为 rows=1/17/257、N=64/128/256、两 dtype；pilot 为 rows=17/N=128。默认预热 3 次、每组重复 5 次、五组，可显式调整 `--warmup`、`--repeats`、`--groups`。两方法各有 transform/split/fused 六路径，加一个共用的 `quant_only`，不是两套独立量化算法。 + +`--quick` 不是完整矩阵；`--skip-stress` 会令 `full_suite_complete=false`，不能跳过大行数专项后继续宣称完整套件通过。SDK 生成物和可执行文件在 `build/ascend/<本次目录>`；公开结果目录保存配置/构建日志、CLI 拒绝日志、验证 JSON/日志、可选原始 CSV 及运行摘要。 + +## 已归档的真实正确性结果 + +| 固定版本/构建 | 主矩阵 | grid 专项 | 大行数专项 | API 检查 | CLI 拒绝检查 | +|---|---:|---:|---:|---:|---:| +| `1b305d3e3a07b28b6879596babf47aadbf84ba0c` | 1496 | 128 | 2 | 184 | 17 | +| `f614762a69e75524db65b47fbf7d6d01836db438` OFF | 1496 | 128 | 2 | 188 | 17 | +| 同一 `f614762` ON | 1496 | 128 | 2 | 188 | 17 | + +三套完整记录均为 `execution=npu`、`full_matrix=true`、`full_suite_complete=true`。主矩阵与专项分列，避免把不同测试范围混成一个数字；相同矩阵跨构建不累计成新覆盖。旧 `1b305` 是 184 个 API 检查，新的 188 是增加四个固定除法串联回归后的计数。 + +主矩阵覆盖 N=1～256 的全部 2 次幂、rows=1/3/17/257、两种 transform scale（N=1 不重复）、零值、单位脉冲、均匀/正态/离群值分布及多个种子。两 dtype 各 748 组、3,122,218 个元素，ScalarButterfly/VectorGather/原位路径及 CPU FP32 FWHT 输出位模式一致；全部元素另与已舍入 dtype 的独立 FP64 稠密参考比较。 + +| 精度 | 主矩阵最大绝对误差：已舍入 FP64 | 主矩阵最大绝对误差：未舍入 FP64 | +|---|---:|---:| +| FP16 | 0.00390625 | 0.015620231628418 | +| BF16 | 0.000003814697265625 | 0.124984741210938 | + +FP16 `<1e-2`、BF16 `<5e-2` 的判定只对应已舍入参考；未舍入列包含输出存储精度误差，不声称它也低于相同阈值。CPU 对实际 NPU 变换输出量化、设备分步及融合的 packed bytes 和 float scales 精确一致。 + +grid 专项在 rows=33、N=1/256、block_dim=1～32 上运行，每 dtype 64 组、271,392 元素，合计 128 组；其误差单独记录。大行数专项为每 dtype 一例 rows=262145、N=1、block_dim=32，只覆盖两种 transform 和 VectorGather 原位变换；**不声称大行数量化已测，也不声称已经覆盖大于 2^32 的索引**。 + +正式输出分别见 [1b305 全量](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/production_runs/full_1b305/validation.json)、[f614762 OFF](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/production_runs/scale_off_f614762/validation.json)、[f614762 ON](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/production_runs/scale_on_f614762/validation.json)。四份完整/quick 运行摘要的七个源码 hash 已与对应 Git 对象核对，产物 hash 与下载原件一致；旧 quick/pilot 不当作额外完整验证。 + +最新 OFF 二进制 SHA256 为 `fe0b0cd7d3998e61ce8f6ac6a1d8dd17f9964bc41d1ea68d68011ff7010b70d3`；ON 为 `b738db0bb242945b9e16b9c919c9a6378c26c854edeac0caace3034b447b99fa`。它们来自同一源码、不同明确 CMake 开关，独立构建与测试；本机二进制不放入公开档案。 + +## 交错 OFF/ON 性能对照与独立复算 + +已保留旧版单 block 和 block_dim=32 的 pilot，以及最新 OFF/ON 的阶段试跑 CSV；它们位于 [production_runs](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/production_runs/manifest.json)，采样方法和版本不能混合。不同阶段的 32.6 微秒与 83.7 微秒，或单点标量/矢量比值，都不足以替代严格控制的交错结果，本文暂不将它们写成最终优化结论。 + +固定对照包含 rows=1/17/257/4096/16384 与 N=16/64/128/256 的 20 个形状，每个独立进程同时测两 dtype，即 40 个 shape/dtype 条件。OFF/ON 交错、三轮，实际 120 个独立进程均退出 0、8400 条原始组样本通过核验，block_dim=32、五组、每组五次重复、预热三次。旧/新执行顺序按预先确定的轮次和形状奇偶交替，不作事后挑选。它不是同进程 A/B。 + +完整 [ab_summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/ab_summary.json) 保留固定参数、执行顺序、每进程原始 CSV/log、二进制前后 hash 和设备快照。两个二进制运行前后均未变化，并与前述 OFF/ON 全量构建记录匹配；外部还核对了源码 commit 与七个源码 hash。因此没有将脚本 `--source-id` 的调用方标签本身当成源码证明。 + +每个变体每轮合并 CSV 有 1400 行，共六份：[OFF 第一轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/old_run1.csv)、[第二轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/old_run2.csv)、[第三轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/old_run3.csv)，以及 [ON 第一轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/new_run1.csv)、[第二轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/new_run2.csv)、[第三轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/new_run3.csv)。它们只是各 cell 原始 CSV 的拼接，**不能将合并表再计为新的 8400 个观察值**。`quant_only` 和标量路径均完整保留作控制项，没有按结果筛选。 + +自写 [ascend_ab_benchmark.py](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/ascend_ab_benchmark.py) 的 SHA256 为 `ce49e0bb9f3c03db4a8b747030e92c6e543c23269da6e892d70d9416e3c0342d`，与实机摘要和本地脚本一致。它不编译、不修改源码、不重设环境，只运行已经构建且完整验证过的两份程序。复现时先在相应主机加载配套 CANN 环境，再把 `BINARY_OFF`、`BINARY_ON` 设置为两份 `run_summary.json` 中 `binary.path` 指向的真实文件，使用全新的输出目录： + +```bash +python3 results/ascend/ab_f614762/ascend_ab_benchmark.py \ + --old-binary "$BINARY_OFF" --new-binary "$BINARY_ON" \ + --source-id f614762a69e75524db65b47fbf7d6d01836db438 \ + --block-dim 32 --repeats 5 --warmup 3 --groups 5 \ + --output results/ascend/reproduce_interleaved_ab_01 +``` + +本脚本运行的是各形状基准附带检查，完整正确性来自分开的 production 记录；不把这 120 个基准进程算作新的完整测试套件。也不应在对照过程中并发编译、运行其他 NPU 作业或修改输入/源码。 + +### 全部正负例与控制条件 + +独立复算逐项核对六份合并 CSV 和 120 份 cell CSV，重建合并内容并与原件逐字段相符；120 份进程日志 hash 也一致。每轮使用五组原始毫秒读数的中位数，同轮相同形状、dtype、method 配对。耗时下降百分比定义为 `(OFF_ms-ON_ms)/OFF_ms*100`，“稳定达到 5%”严格要求三轮每一轮都达到该阈值。 + +| 分类 | 120 个向量路径条件 | 160 个控制条件 | +|---|---:|---:| +| 三轮每轮均减少至少 5% 耗时 | 97 | 0 | +| 三轮均更快，但未全部达到 5% | 11 | 105 | +| 三轮均更慢，但未全部达到 5% | 0 | 7 | +| 方向混合或不稳定 | 12 | 48 | +| 三轮每轮均增加至少 5% 耗时 | 0 | 0 | + +这里的控制条件是 scalar transform/split/fused 和 `quant_only`。控制没有三轮都达到 5% 的稳定变化，**不等于控制完全不受编译布局、发射空档或系统状态影响**；单轮仍可有较大变化。不能将向量路径的全部变化都归因于这一条 Muls 指令。 + +所有负例保留。例如 FP16 rows=17、N=16 的 vector_split，三轮降时约为 `+4.518%、+4.571%、-4.507%`；负值表示该轮 ON 更慢。向量配置没有三轮全部更慢的情况，不代表没有单轮退化。三轮 5% 规则是描述性稳定标准，不是统计显著性检验；未计算置信区间。 + +### 建议显式启用的已测范围 + +在 **本机 Ascend910B1/CANN9.0.0、block_dim=32、transform scale=1、FP16/BF16，以及 rows∈{257,4096,16384}、N∈{64,128,256}** 的离散受测范围中，各操作有 18 个 shape/dtype 条件。ON 相对 OFF 的全部三轮耗时下降范围为： + +| VectorGather 操作 | 所有对应形状和三轮的耗时下降 | +|---|---:| +| transform | 40.885385%～79.021993% | +| split | 19.237522%～29.758525% | +| fused | 21.967081%～29.329016% | + +该范围支持在匹配条件下显式尝试 `--vector-scale`，**不能写成任意 rows≥257、任意 dtype/scale 或其他 NPU 都有同样收益**。生产构建默认保持 OFF，未根据有限基准更改全局派发。 + +N≥64 的完整受测范围共 90 个向量条件，其中 79 个三轮每轮至少降时 5%，最差单轮约退化 2.950976%。N=16 的 30 个条件只有 18 个满足该稳定规则，最差单轮约退化 4.506625%；小 rows 和短 N 应按实际输入重新测量。上述百分比是逐配置比较，没有应用工作负载权重，**不能推导未测量的应用级总提速或端到端收益**。 + +### 可复算的毫秒表和分析来源 + +所有派生时间表使用毫秒，保留全部正负例： + +| 表 | 行数 | 内容 | +|---|---:|---| +| [timings_median_ms.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/timings_median_ms.csv) | 1680 | 变体/轮次/形状/精度/方法的五组统计 | +| [off_on_paired_rounds_ms.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/off_on_paired_rounds_ms.csv) | 840 | 每轮 OFF/ON 配对及耗时变化 | +| [off_on_three_round_stability.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/off_on_three_round_stability.csv) | 280 | 三轮逐配置稳定性分类 | +| [within_variant_scalar_vector_ms.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/within_variant_scalar_vector_ms.csv) | 720 | 同一变体、同一 block_dim 内 Scalar/Vector 对照 | +| [within_variant_vector_fusion_ms.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/within_variant_vector_fusion_ms.csv) | 240 | 同一变体、相同条件内 vector split/fused 对照 | + +这些表重复表达同一批 8400 条原始观察的统计与配对，不能把表行数相加为新增测量。变体内部 Scalar/Vector 和融合表也不能与 OFF/ON 开关效果混为一类比较。 + +自写 [analyze_ascend_ab.py](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/analyze_ascend_ab.py) 的 SHA256 为 `11a4c17b4378b18dce74b3bb92062d18afae80005cb86e32a694bdd4da3c4551`，只读原始证据、不运行 NPU。固定审计记录见 [analysis.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/analysis.json) 和 [结论.md](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/结论.md)。可在项目目录复算到尚不存在的新目录： + +```bash +python3 results/ascend/ab_f614762/analysis/analyze_ascend_ab.py \ + --input results/ascend/ab_f614762 \ + --expected-source-id f614762 \ + --output results/ascend/recomputed_ab_analysis +``` + +公开 `analysis.json` 只将派生元数据的 `input_directory` 从私有本地绝对路径改为 `..`（相对于该 JSON 所在目录，指向原始 A/B 目录）；其他 JSON 字段和五份 CSV 字节均未修改。原始/公开 JSON 的双 SHA256、修改字段和范围保存在 [publication_provenance.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/analysis/publication_provenance.json)。复算脚本会记录执行者实际解析的输入路径，因此该路径字段及 JSON hash 可随目录变化；这不改变数值结果、输入 hash 或样本数。 + +最新原始 CSV 同时记录 `kernel_ms` 和 `kernel_us`，它们是同一 ACL timeline event 读数的两种单位，不是新增样本或新增有效位数。旧探索 CSV 如只有微秒，原件保持不变；需要毫秒时只能新增明确标注的派生表，不能改写原始证据。 + +计时区间应明确：NPU event 不包含分配、CPU 参考或主机/设备复制，但可能包含主机发射间的设备空闲，不能冒充隔离测得的纯内核时间或端到端延迟。只读输入复用存在缓存条件，逻辑 I/O 换算 GB/s 不是物理带宽计数器读数；没有独占/固定频率证据时不作相应宣称。 + +## 归档与当前局限 + +- [initial_probes 清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/initial_probes/manifest.json)：25 份原始文件及自写 Add/短拷贝/RNE 探针，Add 256 元素通过、Pad/RNE 各 34 组合通过。重复单点不累计。 +- [division_probe 清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/division_probe/manifest.json)：14 份原始文件及自写两路径探针，明确矢量失败、NPU 标量子路径通过。 +- [production_runs 清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/production_runs/manifest.json)：110 份正式/探索/controller 原件，按版本与开关区分。逐文件 SHA256、源码对应关系和历史范围保留。 +- [ab_f614762 清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/ascend/ab_f614762/manifest.json)：251 份完成的交错对照原件，包含自写脚本、120 个进程数据与设备快照；另收录已独立复核的五份派生表、分析器、结论和单字段路径修改来源说明。 + +所有公开原件都按传输清单校验字节数与 SHA256，目录 `.gitattributes` 使用 `-text` 保留原始字节。不复制私有 CANN 头文件、SDK 生成物、租赁凭据、算力券号、SSH/Jupyter 访问资料。当前交付为原生 C++ API，不宣称已完成 PyTorch GPU 框架扩展、矩阵单元版本、其他昇腾型号适配或最优 block_dim。 + +正式提交及双份公开档案核验尚是单独步骤；设备释放必须在工作完成、原始结果取回并核验后执行。不同项目的验证记录分别维护；PR 与课程登记在项目所有者验收后单独执行。 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_api.cpp b/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_api.cpp new file mode 100644 index 00000000..8c4456af --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_api.cpp @@ -0,0 +1,105 @@ +#include "hadamard_api.h" +#include "aclrtlaunch_hadamard_transform_fp16.h" +#include "aclrtlaunch_hadamard_transform_bf16.h" +#include "aclrtlaunch_hadamard_quantize_fp16.h" +#include "aclrtlaunch_hadamard_quantize_bf16.h" +#include "aclrtlaunch_hadamard_fused_fp16.h" +#include "aclrtlaunch_hadamard_fused_bf16.h" + +#include +#include + +namespace hadamard::ascend { +namespace { + +bool valid_scalars(StorageType storage, Method method, std::uint32_t n, + float scale, std::uint32_t blocks) { + return (storage == StorageType::FP16 || storage == StorageType::BF16) + && (method == Method::ScalarButterfly || method == Method::VectorGather) + && n >= 1 && n <= 256 && (n & (n - 1)) == 0 + && std::isfinite(scale) && scale > 0.0f && blocks >= 1 && blocks <= 32; +} + +bool valid_range(const void* pointer, std::size_t bytes, std::size_t alignment) { + const auto address = reinterpret_cast(pointer); + return pointer && address % alignment == 0 + && bytes <= std::numeric_limits::max() - address; +} + +bool overlap(const void* lhs, std::size_t lhs_bytes, const void* rhs, std::size_t rhs_bytes) { + const auto a = reinterpret_cast(lhs); + const auto b = reinterpret_cast(rhs); + return a < b + rhs_bytes && b < a + lhs_bytes; +} + +bool valid_buffers(const std::uint16_t* input, std::uint16_t* output, + std::uint8_t* packed, float* scales, std::size_t rows, + std::uint32_t n, bool quantize) { + constexpr auto maximum = std::numeric_limits::max(); + const std::size_t row_bytes = std::size_t(n) * sizeof(std::uint16_t); + if (rows > maximum / row_bytes) return false; + const std::size_t input_bytes = rows * row_bytes; + if (!valid_range(input, input_bytes, alignof(std::uint16_t))) return false; + if (!quantize) { + return valid_range(output, input_bytes, alignof(std::uint16_t)) + && (input == output || !overlap(input, input_bytes, output, input_bytes)); + } + if (rows > maximum / sizeof(float)) return false; + const std::size_t packed_bytes = rows * ((n + 1) / 2); + const std::size_t scale_bytes = rows * sizeof(float); + return valid_range(packed, packed_bytes, alignof(std::uint8_t)) + && valid_range(scales, scale_bytes, alignof(float)) + && !overlap(input, input_bytes, packed, packed_bytes) + && !overlap(input, input_bytes, scales, scale_bytes) + && !overlap(packed, packed_bytes, scales, scale_bytes); +} + +std::uint8_t* raw(const void* pointer) { + return reinterpret_cast(const_cast(pointer)); +} + +} // namespace + +aclError transform(const std::uint16_t* input, std::uint16_t* output, + std::size_t rows, std::uint32_t n, float scale, + StorageType storage, Method method, aclrtStream stream, + std::uint32_t block_dim) { + if (!valid_scalars(storage, method, n, scale, block_dim)) return ACL_ERROR_INVALID_PARAM; + if (!rows) return ACL_SUCCESS; + if (!valid_buffers(input, output, nullptr, nullptr, rows, n, false)) return ACL_ERROR_INVALID_PARAM; + const auto count = static_cast(rows); + const auto implementation = static_cast(method); + if (storage == StorageType::FP16) + return ACLRT_LAUNCH_KERNEL(hadamard_transform_fp16)(block_dim, stream, raw(input), raw(output), count, n, scale, implementation); + return ACLRT_LAUNCH_KERNEL(hadamard_transform_bf16)(block_dim, stream, raw(input), raw(output), count, n, scale, implementation); +} + +aclError quantize_int4(const std::uint16_t* input, std::uint8_t* packed, float* scales, + std::size_t rows, std::uint32_t n, + StorageType storage, Method method, aclrtStream stream, + std::uint32_t block_dim) { + if (!valid_scalars(storage, method, n, 1.0f, block_dim)) return ACL_ERROR_INVALID_PARAM; + if (!rows) return ACL_SUCCESS; + if (!valid_buffers(input, nullptr, packed, scales, rows, n, true)) return ACL_ERROR_INVALID_PARAM; + const auto count = static_cast(rows); + // 两种变换方法共用同一量化 kernel，使分步/融合对照只改变变换与融合方式。 + if (storage == StorageType::FP16) + return ACLRT_LAUNCH_KERNEL(hadamard_quantize_fp16)(block_dim, stream, raw(input), packed, raw(scales), count, n); + return ACLRT_LAUNCH_KERNEL(hadamard_quantize_bf16)(block_dim, stream, raw(input), packed, raw(scales), count, n); +} + +aclError transform_int4(const std::uint16_t* input, std::uint8_t* packed, float* scales, + std::size_t rows, std::uint32_t n, float scale, + StorageType storage, Method method, aclrtStream stream, + std::uint32_t block_dim) { + if (!valid_scalars(storage, method, n, scale, block_dim)) return ACL_ERROR_INVALID_PARAM; + if (!rows) return ACL_SUCCESS; + if (!valid_buffers(input, nullptr, packed, scales, rows, n, true)) return ACL_ERROR_INVALID_PARAM; + const auto count = static_cast(rows); + const auto implementation = static_cast(method); + if (storage == StorageType::FP16) + return ACLRT_LAUNCH_KERNEL(hadamard_fused_fp16)(block_dim, stream, raw(input), packed, raw(scales), count, n, scale, implementation); + return ACLRT_LAUNCH_KERNEL(hadamard_fused_bf16)(block_dim, stream, raw(input), packed, raw(scales), count, n, scale, implementation); +} + +} // namespace hadamard::ascend diff --git a/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_api.h b/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_api.h new file mode 100644 index 00000000..203f83a9 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_api.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +namespace hadamard::ascend { + +enum class StorageType : std::uint32_t { FP16 = 0, BF16 = 1 }; +enum class Method : std::uint32_t { ScalarButterfly = 0, VectorGather = 1 }; + +// 当前实现面向 Ascend910B1。两种方法使用同一 block_dim，首版允许 1..32，默认 1。 +// 该参数范围仍须由对应发布版本的实机 grid 验证支撑，不代表性能最优值。 +// 连续 [rows,n]，n 为 1..256 的二次幂；16 位指针承载 FP16/BF16 原始位模式。 +// input/output 至少 2B 对齐，scales 至少 4B 对齐，packed 无额外地址对齐要求。 +// 所有计算均在 NPU 完成，只发射到 caller stream，不分配/复制 host 或 GM 张量，不同步。 +// rows=0 时验证枚举、n、scale、block_dim 后直接返回成功，允许空指针/空 stream。 +// 非空时调用者须保证缓冲区属于当前设备、容量足够、stream 有效且异步执行前不释放。 +// 输入及 FP32 中间值应有限，结果须在目标类型可表示范围；API 不同步扫描设备数据。 +// transform 允许 input==output 完全原位；其余部分重叠及量化缓冲之间的重叠均拒绝。 +// 返回参数/发射状态；完成性及异步执行错误由调用者同步 stream 或事件确认。 +aclError transform(const std::uint16_t* input, std::uint16_t* output, + std::size_t rows, std::uint32_t n, float scale, + StorageType storage, Method method, aclrtStream stream, + std::uint32_t block_dim = 1); + +// 每行 ceil(n/2) 字节，偶数元素位于低 nibble，n=1 的高 nibble 为 0。 +// s=max(abs(x))/7（全零行 s=1），q=clamp(RNE(x/s),-7,7)，scales 为 rows 个 float。 +// NPU 标量 / 遵循本项目数值探针；不使用已发现中点误差的矢量 Div/近似倒数。 +aclError quantize_int4(const std::uint16_t* input, std::uint8_t* packed, float* scales, + std::size_t rows, std::uint32_t n, + StorageType storage, Method method, aclrtStream stream, + std::uint32_t block_dim = 1); + +// 融合路径先 RNE 到公开 FP16/BF16 类型，再读回 FP32 量化，保持分步语义。 +aclError transform_int4(const std::uint16_t* input, std::uint8_t* packed, float* scales, + std::size_t rows, std::uint32_t n, float scale, + StorageType storage, Method method, aclrtStream stream, + std::uint32_t block_dim = 1); + +} // namespace hadamard::ascend diff --git a/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_kernel.cpp b/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_kernel.cpp new file mode 100644 index 00000000..832bd7b3 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/ascend/hadamard_kernel.cpp @@ -0,0 +1,252 @@ +#include "kernel_operator.h" + +#ifndef HADAMARD_ASCEND_VECTOR_SCALE +#define HADAMARD_ASCEND_VECTOR_SCALE 0 +#endif + +// 本项目原生 Ascend C 实现；数学顺序沿用本项目 FWHT，不复制厂商 SDK 实现。 +// 默认两方法共用标量scale；可选宏仅让VectorGather改用Muls，RNE与标量除法量化不变。 +template class HadamardKernel { +public: + __aicore__ inline void Run(GM_ADDR input, GM_ADDR output, GM_ADDR packed_output, + GM_ADDR scale_output, uint64_t rows, uint32_t n, + float transform_scale, uint32_t method) { + n_ = n; + padded_n_ = (n + 7) / 8 * 8; // 每个 FP32 bank 都从 32B 对齐地址开始。 + uint32_t stages = 0; + for (uint32_t stride = 1; stride < n; stride *= 2) ++stages; + input_.SetGlobalBuffer(reinterpret_cast<__gm__ T*>(input), rows * n); + if constexpr (Quantize) { + packed_.SetGlobalBuffer(reinterpret_cast<__gm__ uint8_t*>(packed_output), rows * ((n + 1) / 2)); + scales_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(scale_output), rows); + } else { + output_.SetGlobalBuffer(reinterpret_cast<__gm__ T*>(output), rows * n); + } + + pipe_.InitBuffer(input_queue_, 1, 256 * sizeof(T)); + pipe_.InitBuffer(typed_queue_, 1, 256 * sizeof(T)); + pipe_.InitBuffer(current_buffer_, padded_n_ * sizeof(float)); + pipe_.InitBuffer(next_buffer_, padded_n_ * sizeof(float)); + if constexpr (Transform) { + if (method == 1) { + pipe_.InitBuffer(peer_buffer_, padded_n_ * sizeof(float)); + pipe_.InitBuffer(combined_buffer_, 2 * padded_n_ * sizeof(float)); + const uint32_t index_bytes = stages ? stages * padded_n_ * sizeof(uint32_t) : 32; + pipe_.InitBuffer(partner_offsets_buffer_, index_bytes); + pipe_.InitBuffer(select_offsets_buffer_, index_bytes); + } + } + if constexpr (Quantize) { + pipe_.InitBuffer(packed_queue_, 1, 128); + pipe_.InitBuffer(scale_queue_, 1, 32); + pipe_.InitBuffer(divisor_buffer_, 32); + } + + auto typed_input = input_queue_.template AllocTensor(); + auto typed_output = typed_queue_.template AllocTensor(); + auto current_base = current_buffer_.template Get(); + auto next_base = next_buffer_.template Get(); + for (uint32_t i = 0; i < padded_n_; ++i) { + current_base.SetValue(i, 0.0f); + next_base.SetValue(i, 0.0f); + } + for (uint32_t i = 0; i < 256; ++i) typed_output.SetValue(i, static_cast(0)); + if constexpr (Transform) { + if (method == 1) InitializeOffsets(); + } + AscendC::LocalTensor packed_local; + AscendC::LocalTensor scale_local; + if constexpr (Quantize) { + packed_local = packed_queue_.template AllocTensor(); + scale_local = scale_queue_.template AllocTensor(); + for (uint32_t i = 0; i < 128; ++i) packed_local.SetValue(i, uint8_t(0)); + for (uint32_t i = 0; i < 8; ++i) scale_local.SetValue(i, 0.0f); + auto divisor = divisor_buffer_.template Get(); + divisor.SetValue(0, 7.0f); + } + Fence(); + + const uint32_t row_bytes = n * sizeof(T); + const uint8_t right_pad = ((32 - row_bytes % 32) % 32) / sizeof(T); + const AscendC::DataCopyExtParams typed_copy{1, row_bytes, 0, 0, 0}; + const AscendC::DataCopyPadExtParams input_pad{true, 0, right_pad, static_cast(0)}; + for (uint64_t row = AscendC::GetBlockIdx(); row < rows; row += AscendC::GetBlockNum()) { + auto current = current_base; + auto next = next_base; + AscendC::DataCopyPad(typed_input, input_[row * n], typed_copy, input_pad); + AscendC::PipeBarrier(); + AscendC::Cast(current, typed_input, AscendC::RoundMode::CAST_NONE, n); + AscendC::PipeBarrier(); + if constexpr (Transform) { + if (method == 1) VectorButterfly(current, next); + #if HADAMARD_ASCEND_VECTOR_SCALE + if (method == 1) { + // count重载原位逐元素相乘；不产生跨repeat的源/目标依赖。 + AscendC::Muls(current, current, transform_scale, static_cast(n)); + AscendC::PipeBarrier(); + } else + #endif + { + Fence(); + if (method == 0) ScalarButterfly(current); + for (uint32_t i = 0; i < n; ++i) + current.SetValue(i, current.GetValue(i) * transform_scale); + Fence(); + } + AscendC::Cast(typed_output, current, AscendC::RoundMode::CAST_RINT, n); + AscendC::PipeBarrier(); + if constexpr (Quantize) { + // 融合必须先舍入到公开 dtype，再读取这些值参与量化。 + AscendC::Cast(current, typed_output, AscendC::RoundMode::CAST_NONE, n); + AscendC::PipeBarrier(); + } + } + if constexpr (Quantize) { + Fence(); + QuantizeRow(current, packed_local, scale_local); + Fence(); + const AscendC::DataCopyExtParams byte_copy{1, (n + 1) / 2, 0, 0, 0}; + const AscendC::DataCopyExtParams scale_copy{1, uint32_t(sizeof(float)), 0, 0, 0}; + AscendC::DataCopyPad(packed_[row * ((n + 1) / 2)], packed_local, byte_copy); + AscendC::DataCopyPad(scales_[row], scale_local, scale_copy); + } else { + AscendC::PipeBarrier(); + AscendC::DataCopyPad(output_[row * n], typed_output, typed_copy); + } + // 下一行可能复用 typed_output/packed/scale UB，必须等待前一行搬出完成。 + Fence(); + } + input_queue_.FreeTensor(typed_input); + typed_queue_.FreeTensor(typed_output); + if constexpr (Quantize) { + packed_queue_.FreeTensor(packed_local); + scale_queue_.FreeTensor(scale_local); + } + } + +private: + template __aicore__ inline void Fence() { + auto id = pipe_.template AllocEventID(); + AscendC::SetFlag(id); + AscendC::WaitFlag(id); + pipe_.template ReleaseEventID(id); + } + + __aicore__ inline void InitializeOffsets() { + auto partner = partner_offsets_buffer_.template Get(); + auto select = select_offsets_buffer_.template Get(); + auto peer = peer_buffer_.template Get(); + auto combined = combined_buffer_.template Get(); + for (uint32_t i = 0; i < padded_n_; ++i) { + peer.SetValue(i, 0.0f); + combined.SetValue(i, 0.0f); + combined.SetValue(padded_n_ + i, 0.0f); + } + uint32_t stage = 0; + for (uint32_t stride = 1; stride < n_; stride *= 2, ++stage) { + for (uint32_t i = 0; i < padded_n_; ++i) { + const uint32_t p = i < n_ ? (i ^ stride) * sizeof(float) : 0; + const uint32_t s = i < n_ ? (i + ((i & stride) ? padded_n_ : 0)) * sizeof(float) : 0; + partner.SetValue(stage * padded_n_ + i, p); + select.SetValue(stage * padded_n_ + i, s); + } + } + } + + __aicore__ inline void ScalarButterfly(AscendC::LocalTensor& values) { + for (uint32_t stride = 1; stride < n_; stride *= 2) { + for (uint32_t start = 0; start < n_; start += 2 * stride) { + for (uint32_t i = 0; i < stride; ++i) { + const float a = values.GetValue(start + i); + const float b = values.GetValue(start + i + stride); + values.SetValue(start + i, a + b); + values.SetValue(start + i + stride, a - b); + } + } + } + } + + __aicore__ inline void VectorButterfly(AscendC::LocalTensor& values, + AscendC::LocalTensor& next) { + auto peer = peer_buffer_.template Get(); + auto combined = combined_buffer_.template Get(); + auto partner = partner_offsets_buffer_.template Get(); + auto select = select_offsets_buffer_.template Get(); + uint32_t stage = 0; + for (uint32_t stride = 1; stride < n_; stride *= 2, ++stage) { + AscendC::Gather(peer, values, partner[stage * padded_n_], 0, n_); + AscendC::PipeBarrier(); + AscendC::Add(combined, values, peer, n_); + AscendC::Sub(combined[padded_n_], peer, values, n_); + AscendC::PipeBarrier(); + AscendC::Gather(next, combined, select[stage * padded_n_], 0, n_); + AscendC::PipeBarrier(); + auto previous = values; + values = next; + next = previous; + } + } + + __aicore__ inline int32_t RoundNibble(float value) { + int32_t q = static_cast(value); // 范围约为[-7,7]，先向零截断。 + if (value >= 0.0f) { + const float fraction = value - static_cast(q); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + } else { + const float fraction = static_cast(q) - value; + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) --q; + } + return q < -7 ? -7 : (q > 7 ? 7 : q); + } + + __aicore__ inline void QuantizeRow(const AscendC::LocalTensor& values, + AscendC::LocalTensor& packed, + AscendC::LocalTensor& scale) { + float maximum = 0.0f; + for (uint32_t i = 0; i < n_; ++i) { + const float value = values.GetValue(i); + const float magnitude = value < 0.0f ? -value : value; + if (magnitude > maximum) maximum = magnitude; + } + // 从UB读取运行时分母，保持成功scalar Div探针的形式，不换成矢量Div/倒数乘。 + const auto divisor = divisor_buffer_.template Get(); + const float seven = divisor.GetValue(0); + const float row_scale = maximum == 0.0f ? 1.0f : maximum / seven; + scale.SetValue(0, row_scale); + const float stored_scale = scale.GetValue(0); + for (uint32_t byte = 0; byte < (n_ + 1) / 2; ++byte) { + const int32_t low = RoundNibble(values.GetValue(2 * byte) / stored_scale); + const int32_t high = 2 * byte + 1 < n_ ? RoundNibble(values.GetValue(2 * byte + 1) / stored_scale) : 0; + packed.SetValue(byte, static_cast((low & 15) | ((high & 15) << 4))); + } + } + + uint32_t n_, padded_n_; + AscendC::TPipe pipe_; + AscendC::GlobalTensor input_, output_; + AscendC::GlobalTensor packed_; + AscendC::GlobalTensor scales_; + AscendC::TQue input_queue_; + AscendC::TQue typed_queue_, packed_queue_, scale_queue_; + AscendC::TBuf current_buffer_, next_buffer_, peer_buffer_, combined_buffer_; + AscendC::TBuf partner_offsets_buffer_, select_offsets_buffer_, divisor_buffer_; +}; + +extern "C" __global__ __aicore__ void hadamard_transform_fp16(GM_ADDR x, GM_ADDR y, uint64_t rows, uint32_t n, float scale, uint32_t method) { + HadamardKernel kernel; kernel.Run(x, y, nullptr, nullptr, rows, n, scale, method); +} +extern "C" __global__ __aicore__ void hadamard_transform_bf16(GM_ADDR x, GM_ADDR y, uint64_t rows, uint32_t n, float scale, uint32_t method) { + HadamardKernel kernel; kernel.Run(x, y, nullptr, nullptr, rows, n, scale, method); +} +extern "C" __global__ __aicore__ void hadamard_quantize_fp16(GM_ADDR x, GM_ADDR packed, GM_ADDR scales, uint64_t rows, uint32_t n) { + HadamardKernel kernel; kernel.Run(x, nullptr, packed, scales, rows, n, 1.0f, 0); +} +extern "C" __global__ __aicore__ void hadamard_quantize_bf16(GM_ADDR x, GM_ADDR packed, GM_ADDR scales, uint64_t rows, uint32_t n) { + HadamardKernel kernel; kernel.Run(x, nullptr, packed, scales, rows, n, 1.0f, 0); +} +extern "C" __global__ __aicore__ void hadamard_fused_fp16(GM_ADDR x, GM_ADDR packed, GM_ADDR scales, uint64_t rows, uint32_t n, float scale, uint32_t method) { + HadamardKernel kernel; kernel.Run(x, nullptr, packed, scales, rows, n, scale, method); +} +extern "C" __global__ __aicore__ void hadamard_fused_bf16(GM_ADDR x, GM_ADDR packed, GM_ADDR scales, uint64_t rows, uint32_t n, float scale, uint32_t method) { + HadamardKernel kernel; kernel.Run(x, nullptr, packed, scales, rows, n, scale, method); +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/ascend/run_platform.py b/03_hadamard_tc/a962695448-rgb/platforms/ascend/run_platform.py new file mode 100644 index 00000000..d8609abb --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/ascend/run_platform.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""原生 Ascend C/CANN 构建与 NPU 验证；基准须显式开启，不安装或替换 SDK。""" +import argparse +import csv +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import shutil +import statistics +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = Path(__file__).resolve().parent +BENCHMARK_COLUMNS = ( + "dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs," + "input_working_set_bytes,seed,input_read_only,scale,block_dim,warmup,timer,vector_scale_enabled,kernel_ms" +).split(",") +VECTOR_SCALE_SCOPE = ["vector_transform", "vector_split", "vector_fused"] + + +def digest(path): + result = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + result.update(chunk) + return result.hexdigest() + + +def capture(command, environment): + result = subprocess.run(list(map(str, command)), cwd=ROOT, env=environment, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + return {"command": list(map(str, command)), "returncode": result.returncode, "output": result.stdout} + + +def save(path, report): + path.write_text(json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8") + + +def run(command, log, report, manifest, environment, expected=0): + record = {"command": list(map(str, command)), "log": log.name, "returncode": None, + "expected_returncode": expected} + report["stages"].append(record) + save(manifest, report) + print("RUN", " ".join(record["command"]), flush=True) + start = time.monotonic() + try: + with log.open("x", encoding="utf-8") as output: + result = subprocess.run(record["command"], cwd=ROOT, env=environment, + stdout=output, stderr=subprocess.STDOUT, check=False) + record["returncode"] = result.returncode + finally: + record["wall_seconds"] = time.monotonic() - start + if log.is_file(): + record["sha256"] = digest(log) + save(manifest, report) + if record["returncode"] != expected: + raise RuntimeError("stage failed: " + log.name) + + +def environment_for(cann): + environment = dict(os.environ) + script = cann / "set_env.sh" + setup = {"script": str(script), "used": script.is_file()} + if script.is_file(): + # SDK 自带脚本只在子进程中改变环境。env 的完整结果留在内存，不写入日志。 + result = subprocess.run(["bash", "-c", 'source "$1" >/dev/null && env -0', "ascend-env", str(script)], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if result.returncode: + raise RuntimeError("CANN set_env.sh failed: " + result.stderr.decode(errors="replace")[-3000:]) + environment = {key.decode(): value.decode() for entry in result.stdout.split(b"\0") if b"=" in entry + for key, value in [entry.split(b"=", 1)]} + setup["sha256"] = digest(script) + environment.update(ASCEND_HOME_PATH=str(cann), ASCEND_TOOLKIT_HOME=str(cann), ASCEND_CANN_PACKAGE_PATH=str(cann)) + extra_bins = [str(p) for p in (cann / "bin", cann / "compiler/ccec_compiler/bin", cann / "tools/ccec_compiler/bin") if p.is_dir()] + environment["PATH"] = os.pathsep.join(extra_bins + [environment.get("PATH", "")]) + libraries = [str(p) for p in (cann / "lib64", cann / "aarch64-linux/lib64", Path("/usr/local/Ascend/driver/lib64")) if p.is_dir()] + inherited = environment.get("LD_LIBRARY_PATH", "") + environment["LD_LIBRARY_PATH"] = os.pathsep.join(libraries + ([inherited] if inherited else [])) + return environment, setup + + +def summarize(path, expected_vector_scale=None): + groups = {} + groups_ms = {} + observed_vector_scale = None + with path.open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + if reader.fieldnames != BENCHMARK_COLUMNS: + raise RuntimeError("unexpected benchmark CSV columns; vector_scale_enabled and kernel_ms are required") + for row in reader: + if None in row or any(value is None for value in row.values()): + raise RuntimeError("malformed benchmark CSV row") + if row["timer"] != "acl_timeline_event_ms": + raise RuntimeError("unexpected benchmark timing source") + if row["vector_scale_enabled"] not in ("true", "false"): + raise RuntimeError("invalid benchmark vector_scale_enabled flag") + enabled = row["vector_scale_enabled"] == "true" + if expected_vector_scale is not None and enabled != expected_vector_scale: + raise RuntimeError("benchmark does not match requested vector-scale build") + if observed_vector_scale is not None and enabled != observed_vector_scale: + raise RuntimeError("mixed vector-scale builds in one benchmark CSV") + observed_vector_scale = enabled + microseconds = float(row["kernel_us"]) + milliseconds = float(row["kernel_ms"]) + if not (math.isfinite(microseconds) and math.isfinite(milliseconds) + and microseconds > 0 and milliseconds > 0): + raise RuntimeError("benchmark us/ms must both be finite and positive") + # 两列均以12位有效数字序列化；只容纳序列化误差，不容纳单位或样本错配。 + if not math.isclose(microseconds, milliseconds * 1000.0, rel_tol=2e-11, abs_tol=0.0): + raise RuntimeError("benchmark kernel_us/kernel_ms unit conversion mismatch") + key = tuple(row[field] for field in ("dtype", "batch", "seq", "heads", "dim", "block_dim", "method")) + groups.setdefault(key, []).append(microseconds) + groups_ms.setdefault(key, []).append(milliseconds) + if not groups: + raise RuntimeError("empty benchmark CSV") + series = [] + medians = {key: statistics.median(values) for key, values in groups.items()} + if not all(math.isfinite(value) for value in medians.values()): + raise RuntimeError("nonfinite benchmark median") + for key, values in groups.items(): + values_ms = groups_ms[key] + median_ms = statistics.median(values_ms) + if not math.isfinite(median_ms): + raise RuntimeError("nonfinite benchmark millisecond median") + series.append(dict(zip(("dtype", "batch", "seq", "heads", "dim", "block_dim", "method"), key), + raw_samples_us=values, median_us=statistics.median(values), + minimum_us=min(values), maximum_us=max(values), + raw_samples_ms=values_ms, median_ms=median_ms, + minimum_ms=min(values_ms), maximum_ms=max(values_ms))) + comparisons = [] + for key, value in medians.items(): + if key[-1].startswith("scalar_"): + peer = key[:-1] + (key[-1].replace("scalar_", "vector_"),) + ratio = value / medians[peer] + if not math.isfinite(ratio): + raise RuntimeError("nonfinite benchmark comparison") + comparisons.append({"shape_dtype_blocks": key[:-1], "operation": key[-1][7:], + "scalar_median_us": value, "vector_median_us": medians[peer], + "scalar_over_vector_same_blocks": ratio}) + return {"timer": "ACL_EVENT_TIME_LINE events; aclrtEventElapsedTime milliseconds converted to microseconds", + "unit_relationship": "kernel_ms = kernel_us / 1000; same samples, not additional timing or precision", + "vector_scale_enabled": observed_vector_scale, + "vector_scale_scope": VECTOR_SCALE_SCOPE if observed_vector_scale else [], + "not_timed": ["allocation", "copies", "validation", "event creation/destruction", "warmup"], + "cache_condition": "same seeded read-only inputs reused; warm-cache samples", + "quant_only": "one shared NPU scalar-division quantization implementation; not two different algorithms", + "logical_GBs": "logical tensor I/O estimate, not measured physical memory bandwidth", + "series": series, "comparisons_including_slowdowns": comparisons} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cann-root", type=Path, default=Path("/usr/local/Ascend/cann-9.0.0")) + parser.add_argument("--output", type=Path) + parser.add_argument("--cmake", default="cmake") + parser.add_argument("--build-jobs", type=int, default=1) + parser.add_argument("--quick", action="store_true") + parser.add_argument("--skip-stress", action="store_true", help="明确跳过大M专项，报告不宣称完整验收") + parser.add_argument("--dtype", choices=("both", "fp16", "bf16"), default="both") + parser.add_argument("--block-dim", type=int, default=1) + parser.add_argument("--benchmark", action="store_true", help="验证通过后运行小规模基准矩阵") + parser.add_argument("--pilot-benchmark", action="store_true", help="只用M17/N128先核对真实ACL event时间和耗时") + parser.add_argument("--repeats", "--iterations", type=int, default=5) + parser.add_argument("--groups", type=int, default=5) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--vector-scale", action="store_true", + help="编译启用VectorGather的Muls scale候选；默认关闭，量化仍用NPU标量除法") + args = parser.parse_args() + if not 1 <= args.block_dim <= 32 or not 1 <= args.build_jobs <= 8: + parser.error("block-dim must be 1..32; build-jobs must be 1..8") + if not 1 <= args.repeats <= 10000 or not 1 <= args.groups <= 10000 or not 0 <= args.warmup <= 10000: + parser.error("repeats/groups must be 1..10000; warmup must be 0..10000") + cann = args.cann_root.resolve() + cmake_entry = cann / "aarch64-linux/tikcpp/ascendc_kernel_cmake/ascendc.cmake" + if not cmake_entry.is_file(): + parser.error("missing actual Ascend C CMake entry: " + str(cmake_entry)) + destination = (args.output or ROOT / "results/ascend" / time.strftime("%Y%m%d-%H%M%S", time.gmtime())).resolve() + build = ROOT / "build/ascend" / (destination.name + "-" + hashlib.sha256(str(destination).encode()).hexdigest()[:10]) + if destination.exists() or build.exists(): + parser.error("output/build directory already exists; use a fresh output path") + try: + environment, setup = environment_for(cann) + except (OSError, RuntimeError, UnicodeError) as error: + parser.error(str(error)) + destination.mkdir(parents=True) + files = [SOURCE / name for name in ("CMakeLists.txt", "hadamard_api.h", "hadamard_api.cpp", "hadamard_kernel.cpp", + "validate_and_benchmark.cpp", "run_platform.py")] + [ROOT / "include/reference.hpp"] + manifest = destination / "run_summary.json" + report = {"status": "RUNNING", "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "platform": "ascend", "run_mode": "npu", "soc_target": "Ascend910B1", "host_architecture": platform.machine(), + "cann_root": str(cann), "sdk_environment_setup": setup, "build_directory": str(build), + "quick": args.quick, "skip_stress": args.skip_stress, "requested_dtype": args.dtype, "block_dim": args.block_dim, + "vector_scale_enabled": args.vector_scale, + "vector_scale_scope": VECTOR_SCALE_SCOPE if args.vector_scale else [], + "benchmark_requested": args.benchmark or args.pilot_benchmark, "pilot_benchmark": args.pilot_benchmark, + "source_sha256": {str(path.relative_to(ROOT)): digest(path) for path in files}, + "python_version": sys.version, "cmake_version": capture([args.cmake, "--version"], environment), + "git_head": capture(["git", "rev-parse", "HEAD"], environment), + "git_status": capture(["git", "status", "--short"], environment), "stages": []} + npu_smi = shutil.which("npu-smi", path=environment.get("PATH")) + if npu_smi: + report["device_inventory"] = capture([npu_smi, "info"], environment) + save(manifest, report) + code = 1 + try: + run([args.cmake, "-S", SOURCE, "-B", build, "-DCMAKE_BUILD_TYPE=Release", "-DRUN_MODE=npu", "-DBUILD_VALIDATION=ON", + "-DENABLE_VECTOR_SCALE=" + ("ON" if args.vector_scale else "OFF"), + "-DSOC_VERSION=Ascend910B1", "-DASCEND_CANN_PACKAGE_PATH=" + str(cann)], + destination / "configure.log", report, manifest, environment) + expected_cache = "ENABLE_VECTOR_SCALE:BOOL=" + ("ON" if args.vector_scale else "OFF") + if expected_cache not in (build / "CMakeCache.txt").read_text(encoding="utf-8").splitlines(): + raise RuntimeError("CMake cache does not match requested vector-scale build") + report["vector_scale_cmake_cache"] = expected_cache + save(manifest, report) + run([args.cmake, "--build", build, "--target", "validate_and_benchmark", "-j", str(args.build_jobs)], + destination / "build.log", report, manifest, environment) + binary = build / "validate_and_benchmark" + report["binary"] = {"path": str(binary), "sha256": digest(binary)} + invalid = [[], ["--validate", "--dim", "0"], ["--validate", "--dim", "3"], ["--validate", "--dim", "512"], + ["--validate", "--dtype", "fp32"], ["--validate", "--batch", "-1"], ["--validate", "--seq", "0"], + ["--validate", "--heads", "x"], ["--validate", "--batch", "18446744073709551615", "--seq", "2"], + ["--validate", "--heads", "999999999999999999999999999999"], ["--validate", "--dim"], + ["--validate", "--unknown", "1"], ["--benchmark", "--repeats", "0"], ["--benchmark", "--groups", "10001"], + ["--validate", "--block-dim", "0"], ["--validate", "--block-dim", "33"], ["--benchmark", "--warmup", "-1"]] + for index, case in enumerate(invalid): + run([binary] + case, destination / ("invalid_%02d.log" % index), report, manifest, environment, expected=2) + report["cli_rejection_cases"] = len(invalid) + command = [binary, "--validate", "--block-dim", str(args.block_dim), "--dtype", args.dtype, + "--json", destination / "validation.json"] + if args.quick: + command.append("--quick") + if args.skip_stress: + command.append("--skip-stress") + run(command, destination / "validation.log", report, manifest, environment) + validation = json.loads((destination / "validation.json").read_text(encoding="utf-8")) + report["validation"] = validation + if validation.get("status") != "PASS" or validation.get("execution") != "npu" or validation.get("main_block_dim") != args.block_dim: + raise RuntimeError("validation JSON does not confirm requested NPU execution") + if validation.get("vector_scale_enabled") is not args.vector_scale: + raise RuntimeError("validation JSON does not confirm requested vector-scale build") + if not args.quick and args.dtype == "both": + if validation.get("full_matrix") is not True or (not args.skip_stress and validation.get("full_suite_complete") is not True): + raise RuntimeError("full validation JSON is incomplete") + if args.benchmark or args.pilot_benchmark: + command = [binary, "--benchmark", "--block-dim", str(args.block_dim), "--dtype", args.dtype, + "--repeats", str(args.repeats), "--groups", str(args.groups), "--warmup", str(args.warmup), + "--csv", destination / "benchmark.csv"] + if args.pilot_benchmark: + command += ["--batch", "1", "--seq", "17", "--heads", "1", "--dim", "128"] + run(command, destination / "benchmark.log", report, manifest, environment) + report["benchmark"] = summarize(destination / "benchmark.csv", expected_vector_scale=args.vector_scale) + report["status"] = "PASS" + code = 0 + except KeyboardInterrupt: + report["status"] = "INTERRUPTED" + report["error"] = "caller interrupted; completed artifacts retained" + code = 130 + except (OSError, RuntimeError, ValueError, KeyError) as error: + report["status"] = "FAIL" + report["error"] = str(error) + print("FAIL", error, file=sys.stderr, flush=True) + finally: + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + report["artifacts"] = {path.name: digest(path) for path in destination.iterdir() if path.is_file() and path != manifest} + save(manifest, report) + print(json.dumps({"status": report["status"], "summary": str(manifest)}, ensure_ascii=False), flush=True) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/platforms/ascend/validate_and_benchmark.cpp b/03_hadamard_tc/a962695448-rgb/platforms/ascend/validate_and_benchmark.cpp new file mode 100644 index 00000000..8aadb025 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/ascend/validate_and_benchmark.cpp @@ -0,0 +1,657 @@ +#include "hadamard_api.h" +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef HADAMARD_ASCEND_VECTOR_SCALE +#define HADAMARD_ASCEND_VECTOR_SCALE 0 +#endif +constexpr bool kVectorScaleEnabled = HADAMARD_ASCEND_VECTOR_SCALE != 0; + +namespace api = hadamard::ascend; +#define CHECK(call) do { aclError e_ = (call); if (e_ != ACL_SUCCESS) \ + throw std::runtime_error(std::string(#call) + " returned ACL error " + std::to_string(e_)); } while (0) + +// 主机只处理标准 16 位存储位模式，不使用 SDK half/BF16 构造器或 PyTorch。 +struct FP16Bits { std::uint16_t bits; }; +struct BF16Bits { std::uint16_t bits; }; +static_assert(sizeof(FP16Bits) == 2 && sizeof(BF16Bits) == 2, "16-bit storage required"); +std::uint32_t configured_blocks = 1; + +std::uint32_t float_bits(float x) { std::uint32_t b; std::memcpy(&b, &x, 4); return b; } +float bits_float(std::uint32_t b) { float x; std::memcpy(&x, &b, 4); return x; } +std::uint32_t rshift_rne(std::uint32_t x, unsigned shift) { + const auto high = x >> shift, low = x & ((1u << shift) - 1u), midpoint = 1u << (shift - 1); + return high + (low > midpoint || (low == midpoint && (high & 1u))); +} + +std::uint16_t encode_half(float x) { + const auto b = float_bits(x), fraction = b & 0x7fffffu; + const auto sign = static_cast((b >> 16) & 0x8000u); + const int exponent_bits = (b >> 23) & 255; + const int exponent = exponent_bits - 127; + if (exponent_bits == 255) return sign | (fraction ? 0x7e00u : 0x7c00u); + if (exponent > 15) return sign | 0x7c00u; + if (exponent < -25) return sign; + if (exponent < -14) return sign | rshift_rne(fraction | 0x800000u, -exponent - 1); + return sign | (((exponent + 15) << 10) + rshift_rne(fraction, 13)); +} + +std::uint16_t encode_bfloat(float x) { + const auto b = float_bits(x), high = b >> 16, low = b & 0xffffu; + if ((b & 0x7f800000u) == 0x7f800000u) + return static_cast(high | ((b & 0x7fffffu) ? 0x40u : 0u)); + return static_cast(high + (low > 0x8000u || (low == 0x8000u && (high & 1u)))); +} + +float decode_half(std::uint16_t b) { + const int exponent = (b >> 10) & 31, fraction = b & 1023; + if (exponent == 31) return fraction ? std::numeric_limits::quiet_NaN() + : ((b & 0x8000u) ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + const float x = exponent ? std::ldexp(static_cast(1024 + fraction), exponent - 25) + : std::ldexp(static_cast(fraction), -24); + return b & 0x8000u ? -x : x; +} + +template float read(T); +template<> float read(FP16Bits x) { return decode_half(x.bits); } +template<> float read(BF16Bits x) { return bits_float(static_cast(x.bits) << 16); } +template T rounded(float); +template<> FP16Bits rounded(float x) { return {encode_half(x)}; } +template<> BF16Bits rounded(float x) { return {encode_bfloat(x)}; } +template api::StorageType storage(); +template<> api::StorageType storage() { return api::StorageType::FP16; } +template<> api::StorageType storage() { return api::StorageType::BF16; } + +template aclError launch_transform(const T* x, T* y, std::size_t rows, int n, float scale, + aclrtStream stream, api::Method method, std::uint32_t blocks = configured_blocks) { + return api::transform(reinterpret_cast(x), reinterpret_cast(y), + rows, static_cast(n), scale, storage(), method, stream, blocks); +} +template aclError launch_quantize(const T* x, std::uint8_t* q, float* scales, std::size_t rows, int n, + aclrtStream stream, api::Method method, std::uint32_t blocks = configured_blocks) { + return api::quantize_int4(reinterpret_cast(x), q, scales, rows, + static_cast(n), storage(), method, stream, blocks); +} +template aclError launch_fused(const T* x, std::uint8_t* q, float* scales, std::size_t rows, int n, float scale, + aclrtStream stream, api::Method method, std::uint32_t blocks = configured_blocks) { + return api::transform_int4(reinterpret_cast(x), q, scales, rows, + static_cast(n), scale, storage(), method, stream, blocks); +} + +// 每个区间都有前后哨兵。17 个元素的偏移同时覆盖仅 2 字节对齐的 FP16/BF16 指针。 +template class Guarded { + static constexpr std::size_t guard = 17; + T* raw_ = nullptr; + void* pinned_ = nullptr; + aclrtStream stream_; + std::size_t count_; + std::vector initial_; + std::string name_; +public: + explicit Guarded(std::size_t count, aclrtStream stream, std::string name = "unnamed") + : stream_(stream), count_(count), name_(name) { + if (count > (std::numeric_limits::max() - 63) / sizeof(T) - 2 * guard) + throw std::runtime_error("guarded allocation size overflow"); + const auto total = ((count + 2 * guard) * sizeof(T) + 63) / 64 * 64; + initial_.assign(total, 0xa5); + try { + CHECK(aclrtMallocHost(&pinned_, total)); + CHECK(aclrtMalloc(reinterpret_cast(&raw_), total, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK(aclrtMemsetAsync(raw_, total, 0xa5, total, stream)); + } catch (...) { + if (raw_) aclrtFree(raw_); + if (pinned_) aclrtFreeHost(pinned_); + throw; + } + } + ~Guarded() { + if (stream_) aclrtSynchronizeStream(stream_); + if (raw_) aclrtFree(raw_); + if (pinned_) aclrtFreeHost(pinned_); + } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + T* data() { return raw_ + guard; } + void upload(const std::vector& values, aclrtStream stream) { + if (values.size() != count_) throw std::runtime_error("upload size mismatch"); + std::memcpy(initial_.data() + guard * sizeof(T), values.data(), count_ * sizeof(T)); + std::memcpy(pinned_, initial_.data(), initial_.size()); + CHECK(aclrtMemcpyAsync(raw_, initial_.size(), pinned_, initial_.size(), ACL_MEMCPY_HOST_TO_DEVICE, stream)); + } + std::vector download(aclrtStream stream, bool unchanged = false, const char* phase = "readback") { + CHECK(aclrtMemcpyAsync(pinned_, initial_.size(), raw_, initial_.size(), ACL_MEMCPY_DEVICE_TO_HOST, stream)); + CHECK(aclrtSynchronizeStream(stream)); + const auto* bytes = static_cast(pinned_); + const std::size_t prefix = guard * sizeof(T), end = prefix + count_ * sizeof(T); + for (std::size_t i = 0; i < initial_.size(); ++i) { + if (bytes[i] == initial_[i] || (!unchanged && i >= prefix && i < end)) continue; + const bool is_guard = i < prefix || i >= end; + throw std::runtime_error(std::string(is_guard ? "device buffer guard overwritten" : "read-only input modified") + + " buffer=" + name_ + " phase=" + phase + " region=" + (i < prefix ? "prefix" : (i >= end ? "suffix" : "payload")) + + " byte_from_payload=" + std::to_string(static_cast(i) - static_cast(prefix)) + + " expected=" + std::to_string(static_cast(initial_[i])) + + " actual=" + std::to_string(static_cast(bytes[i])) + + " elements=" + std::to_string(count_) + " element_bytes=" + std::to_string(sizeof(T))); + } + std::vector result(count_); + std::memcpy(result.data(), bytes + prefix, count_ * sizeof(T)); + return result; + } +}; + +struct Options { + bool validate = false, benchmark = false, custom_shape = false, quick = false; + bool skip_stress = false; + std::size_t batch = 1, seq = 1, heads = 1; + int dim = 128, repeats = 5, groups = 5, warmup = 3; + std::uint32_t block_dim = 1; + std::string dtype = "both", csv = "ascend_benchmark.csv", json = "ascend_validation.json"; +}; + +std::size_t positive(const std::string& text, const char* name) { + if (text.empty() || text.find_first_not_of("0123456789") != std::string::npos) + throw std::invalid_argument(std::string(name) + " must be a positive integer"); + std::size_t used = 0; + const auto value = std::stoull(text, &used); + if (!value || value > std::numeric_limits::max() || used != text.size()) + throw std::invalid_argument(std::string(name) + " is outside its supported range"); + return static_cast(value); +} + +std::size_t multiply(std::size_t a, std::size_t b) { + if (b && a > std::numeric_limits::max() / b) + throw std::invalid_argument("shape product overflows size_t"); + return a * b; +} + +std::size_t checked_shape(std::size_t b, std::size_t s, std::size_t h, int n) { + if (n < 1 || n > 256 || !hadamard::power_of_two(n)) + throw std::invalid_argument("dim must be a power of two in [1,256]"); + const auto rows = multiply(multiply(b, s), h); + if (!rows) throw std::invalid_argument("rows must be positive for CLI tensors"); + multiply(multiply(rows, static_cast(n)), sizeof(FP16Bits)); + multiply(rows, sizeof(float)); + return rows; +} + +Options parse(int argc, char** argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string key = argv[i]; + if (key == "--validate") o.validate = true; + else if (key == "--benchmark") o.benchmark = true; + else if (key == "--quick") o.quick = true; + else if (key == "--skip-stress") o.skip_stress = true; + else { + if (i + 1 == argc) throw std::invalid_argument("missing value for " + key); + const std::string value = argv[++i]; + if (key == "--dtype") o.dtype = value; + else if (key == "--csv") o.csv = value; + else if (key == "--json") o.json = value; + else if (key == "--batch" || key == "--seq" || key == "--heads" || key == "--dim") { + const auto v = positive(value, key.c_str()); + o.custom_shape = true; + if (key == "--batch") o.batch = v; + else if (key == "--seq") o.seq = v; + else if (key == "--heads") o.heads = v; + else { + if (v > 256) throw std::invalid_argument("dim is greater than 256"); + o.dim = static_cast(v); + } + } else if (key == "--block-dim") { + const auto v = positive(value, key.c_str()); + if (v > 32) throw std::invalid_argument("block-dim must be in [1,32]"); + o.block_dim = static_cast(v); + } else if (key == "--repeats" || key == "--groups" || key == "--warmup") { + const auto v = (key == "--warmup" && value == "0") ? 0 : positive(value, key.c_str()); + if (v > 10000) throw std::invalid_argument("repeats/groups/warmup exceed 10000"); + if (key == "--repeats") o.repeats = static_cast(v); + else if (key == "--groups") o.groups = static_cast(v); + else o.warmup = static_cast(v); + } else throw std::invalid_argument("unknown argument " + key); + } + } + if (!o.validate && !o.benchmark) throw std::invalid_argument("specify --validate and/or --benchmark"); + if (o.dtype != "both" && o.dtype != "fp16" && o.dtype != "bf16") + throw std::invalid_argument("dtype must be fp16, bf16 or both"); + checked_shape(o.batch, o.seq, o.heads, o.dim); + return o; +} + +template std::vector make_input(std::size_t rows, int n, int pattern, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::normal_distribution normal(0.0f, 0.5f); + std::vector result(rows * n); + for (std::size_t i = 0; i < result.size(); ++i) { + float x = 0; + if (pattern == 0) x = uniform(rng); + else if (pattern == 1) x = normal(rng); + else if (pattern == 2) x = i % n == (i / n + seed) % n ? 8.0f : uniform(rng); + else if (pattern == 4) x = i % n == (i / n) % n ? 1.0f : 0.0f; + result[i] = rounded(x); + } + return result; +} + +struct Summary { + std::size_t cases = 0, elements = 0, exact_transform_elements = 0, contract_checks = 0; + std::size_t grid_cases = 0, grid_elements = 0, large_m_cases = 0, large_m_elements = 0; + double grid_max_rounded_error = 0, grid_max_unrounded_error = 0; + double max_rounded_error = 0, max_unrounded_error = 0; +}; + +template void one_case(aclrtStream stream, std::size_t rows, int n, float scale, + int pattern, unsigned seed, const char* dtype, Summary& summary) { + const auto input = make_input(rows, n, pattern, seed); + const std::size_t size = input.size(), bytes = rows * ((n + 1) / 2); + const std::string context = std::string(dtype) + " rows=" + std::to_string(rows) + " n=" + std::to_string(n) + + " pattern=" + std::to_string(pattern) + " seed=" + std::to_string(seed) + " scale=" + std::to_string(scale) + + " block_dim=" + std::to_string(configured_blocks); + Guarded x(size, stream, context + " input"), baseline(size, stream, context + " scalar"), + optimized(size, stream, context + " vector"), inplace(size, stream, context + " vector-inplace"), + scalar_inplace(size, stream, context + " scalar-inplace"); + Guarded base_split(bytes, stream, context + " scalar-split"), opt_split(bytes, stream, context + " vector-split"), + base_fused(bytes, stream, context + " scalar-fused"), opt_fused(bytes, stream, context + " vector-fused"); + Guarded bs(rows, stream, context + " scalar-split-scales"), os(rows, stream, context + " vector-split-scales"), + bfs(rows, stream, context + " scalar-fused-scales"), ofs(rows, stream, context + " vector-fused-scales"); + x.upload(input, stream); + inplace.upload(input, stream); + scalar_inplace.upload(input, stream); + CHECK(launch_transform(x.data(), baseline.data(), rows, n, scale, stream, api::Method::ScalarButterfly)); + CHECK(launch_transform(x.data(), optimized.data(), rows, n, scale, stream, api::Method::VectorGather)); + CHECK(launch_transform(inplace.data(), inplace.data(), rows, n, scale, stream, api::Method::VectorGather)); + CHECK(launch_transform(scalar_inplace.data(), scalar_inplace.data(), rows, n, scale, stream, api::Method::ScalarButterfly)); + CHECK(launch_quantize(baseline.data(), base_split.data(), bs.data(), rows, n, stream, api::Method::ScalarButterfly)); + CHECK(launch_quantize(optimized.data(), opt_split.data(), os.data(), rows, n, stream, api::Method::VectorGather)); + CHECK(launch_fused(x.data(), base_fused.data(), bfs.data(), rows, n, scale, stream, api::Method::ScalarButterfly)); + CHECK(launch_fused(x.data(), opt_fused.data(), ofs.data(), rows, n, scale, stream, api::Method::VectorGather)); + CHECK(aclrtSynchronizeStream(stream)); + x.download(stream, true); + const auto a = baseline.download(stream), b = optimized.download(stream), ip = inplace.download(stream), sip = scalar_inplace.download(stream); + if (std::memcmp(a.data(), b.data(), size * sizeof(T)) || std::memcmp(b.data(), ip.data(), size * sizeof(T)) + || std::memcmp(a.data(), sip.data(), size * sizeof(T))) + throw std::runtime_error("scalar/vector/in-place transform not bitwise identical: " + context); + std::vector fx(size), actual(size); + for (std::size_t i = 0; i < size; ++i) { fx[i] = read(input[i]); actual[i] = read(b[i]); } + auto fwht = fx; + hadamard::fwht(fwht.data(), rows, n, scale); + for (std::size_t i = 0; i < size; ++i) { + if (rounded(fwht[i]).bits != b[i].bits) + throw std::runtime_error("rounded FP32 CPU FWHT mismatch: " + context + " index=" + std::to_string(i)); + } + // 第二项独立参考使用 O(N^2) FP64 稠密矩阵公式，不共享蝶形代码。 + const auto dense = hadamard::dense_reference(fx, n, static_cast(scale)); + const double tolerance = std::string(dtype) == "fp16" ? 1e-2 : 5e-2; + for (std::size_t i = 0; i < size; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + const double error = std::abs(static_cast(actual[i]) - expected); + if (!std::isfinite(actual[i]) || !(error < tolerance)) + throw std::runtime_error("rounded FP64 dense oracle mismatch: " + context + " index=" + std::to_string(i) + + " error=" + std::to_string(error)); + summary.max_rounded_error = std::max(summary.max_rounded_error, error); + summary.max_unrounded_error = std::max(summary.max_unrounded_error, std::abs(actual[i] - dense[i])); + } + const auto expected_q = hadamard::quantize_int4(actual, n); + if (base_split.download(stream) != expected_q.packed || opt_split.download(stream) != expected_q.packed || + base_fused.download(stream) != expected_q.packed || opt_fused.download(stream) != expected_q.packed || + bs.download(stream) != expected_q.scales || os.download(stream) != expected_q.scales || + bfs.download(stream) != expected_q.scales || ofs.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/scalar/vector split/fused INT4 bytes or scales mismatch: " + context); + ++summary.cases; + summary.elements += size; + summary.exact_transform_elements += size; +} + +template void contract_tests(aclrtStream stream, Summary& summary) { + Guarded input(64, stream, "contract.input"), output(64, stream, "contract.output"); + Guarded packed(32, stream, "contract.packed"); + Guarded scales(8, stream, "contract.scales"); + input.download(stream, true, "initialized"); + output.download(stream, true, "initialized"); + packed.download(stream, true, "initialized"); + scales.download(stream, true, "initialized"); + auto reject = [&](aclError status) { + if (status != ACL_ERROR_INVALID_PARAM) throw std::runtime_error("invalid API input did not return ACL_ERROR_INVALID_PARAM"); + ++summary.contract_checks; + }; + auto success = [&](aclError status) { CHECK(status); ++summary.contract_checks; }; + for (const auto method : {api::Method::ScalarButterfly, api::Method::VectorGather}) { + const char* method_name = method == api::Method::ScalarButterfly ? "scalar_butterfly" : "vector_gather"; + std::cout << "CONTRACT_PROGRESS method=" << method_name << " phase=invalid-parameters" << std::endl; + for (int n : {0, 3, 512}) { + reject(launch_transform(input.data(), output.data(), 1, n, 1, stream, method)); + reject(launch_quantize(input.data(), packed.data(), scales.data(), 1, n, stream, method)); + reject(launch_fused(input.data(), packed.data(), scales.data(), 1, n, 1, stream, method)); + } + for (float scale : {0.0f, -1.0f, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { + reject(launch_transform(input.data(), output.data(), 1, 8, scale, stream, method)); + reject(launch_fused(input.data(), packed.data(), scales.data(), 1, 8, scale, stream, method)); + } + reject(launch_transform(static_cast(nullptr), output.data(), 1, 8, 1, stream, method)); + reject(launch_transform(input.data(), static_cast(nullptr), 1, 8, 1, stream, method)); + reject(launch_transform(input.data(), input.data() + 1, 1, 8, 1, stream, method)); + reject(launch_transform(input.data(), output.data(), std::numeric_limits::max(), 256, 1, stream, method)); + reject(launch_quantize(input.data(), reinterpret_cast(input.data()), scales.data(), 1, 8, stream, method)); + reject(launch_fused(input.data(), packed.data(), nullptr, 1, 8, 1, stream, method)); + auto* odd = reinterpret_cast(reinterpret_cast(input.data()) + 1); + reject(launch_transform(odd, output.data(), 1, 8, 1, stream, method)); + auto* bad_scale = reinterpret_cast(reinterpret_cast(scales.data()) + 2); + reject(launch_quantize(input.data(), packed.data(), bad_scale, 1, 8, stream, method)); + success(launch_transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, method)); + success(launch_quantize(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, method)); + success(launch_fused(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, method)); + success(launch_transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, nullptr, method)); + success(launch_quantize(static_cast(nullptr), nullptr, nullptr, 0, 8, nullptr, method)); + success(launch_fused(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, nullptr, method)); + for (std::uint32_t blocks : {0u, 33u}) { + reject(launch_transform(input.data(), output.data(), 1, 8, 1, stream, method, blocks)); + reject(launch_quantize(input.data(), packed.data(), scales.data(), 1, 8, stream, method, blocks)); + reject(launch_fused(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, method, blocks)); + reject(launch_transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, nullptr, method, blocks)); + } + const auto bad_storage = static_cast(99); + const auto* raw_input = reinterpret_cast(input.data()); + auto* raw_output = reinterpret_cast(output.data()); + reject(api::transform(raw_input, raw_output, 1, 8, 1, bad_storage, method, stream)); + reject(api::quantize_int4(raw_input, packed.data(), scales.data(), 1, 8, bad_storage, method, stream)); + reject(api::transform_int4(raw_input, packed.data(), scales.data(), 1, 8, 1, bad_storage, method, stream)); + reject(api::transform(nullptr, nullptr, 0, 8, 1, bad_storage, method, nullptr)); + input.download(stream, true, "after-invalid-and-zero-rows"); + output.download(stream, true, "after-invalid-and-zero-rows"); + packed.download(stream, false, "after-invalid-and-zero-rows"); + scales.download(stream, false, "after-invalid-and-zero-rows"); + // 正负半整数：预期手写，避免舍入测试仅复用 CPU 参考实现。 + const std::vector ties{7, -7, .5f, 1.5f, 2.5f, -.5f, -1.5f, -2.5f}; + std::vector t(64, rounded(0)); + for (std::size_t i = 0; i < ties.size(); ++i) t[i] = rounded(ties[i]); + std::cout << "CONTRACT_PROGRESS method=" << method_name << " phase=ties-upload" << std::endl; + input.upload(t, stream); + input.download(stream, true, "after-ties-upload"); + std::cout << "CONTRACT_PROGRESS method=" << method_name << " phase=ties-quantize" << std::endl; + CHECK(launch_quantize(input.data(), packed.data(), scales.data(), 1, 8, stream, method)); + const auto q = packed.download(stream, false, "after-ties-quantize"); + const auto s = scales.download(stream, false, "after-ties-quantize"); + const std::array expected{{0x97, 0x20, 0x02, 0xee}}; + if (!std::equal(expected.begin(), expected.end(), q.begin()) || s[0] != 1.0f) + throw std::runtime_error("positive/negative ties-to-even test failed"); + input.download(stream, true, "after-ties-quantize"); + ++summary.contract_checks; + + // 已观测的除法中点回归：1/(FP32(1/7))与0.5/(FP32(1/7))应量化为7、3。 + // 近似除法可能把第二项抬至3.5，错误地产生0x47。预期完全固定，不从actual生成。 + Guarded chain_input(2, stream, "division-chain.input"), + chain_transform(2, stream, "division-chain.transform"); + Guarded chain_split(1, stream, "division-chain.split"), + chain_fused(1, stream, "division-chain.fused"); + Guarded chain_split_scale(1, stream, "division-chain.split_scale"), + chain_fused_scale(1, stream, "division-chain.fused_scale"); + const std::vector chain_values{rounded(0.75f), rounded(0.25f)}; + const std::array expected_transform{{rounded(1.0f), rounded(0.5f)}}; + constexpr std::uint8_t expected_packed = 0x37; + constexpr std::uint32_t expected_scale_bits = 0x3e124925u; + chain_input.upload(chain_values, stream); + std::cout << "CONTRACT_PROGRESS method=" << method_name << " phase=division-chain" << std::endl; + CHECK(launch_transform(chain_input.data(), chain_transform.data(), 1, 2, 1.0f, stream, method)); + CHECK(launch_quantize(chain_transform.data(), chain_split.data(), chain_split_scale.data(), 1, 2, stream, method)); + CHECK(launch_fused(chain_input.data(), chain_fused.data(), chain_fused_scale.data(), 1, 2, 1.0f, stream, method)); + chain_input.download(stream, true, "division-chain-complete"); + const auto chain_y = chain_transform.download(stream, false, "division-chain-complete"); + const auto chain_qs = chain_split.download(stream, false, "division-chain-complete"); + const auto chain_qf = chain_fused.download(stream, false, "division-chain-complete"); + const auto chain_ss = chain_split_scale.download(stream, false, "division-chain-complete"); + const auto chain_sf = chain_fused_scale.download(stream, false, "division-chain-complete"); + if (std::memcmp(chain_y.data(), expected_transform.data(), 2 * sizeof(T)) + || chain_qs[0] != expected_packed || chain_qf[0] != expected_packed + || float_bits(chain_ss[0]) != expected_scale_bits || float_bits(chain_sf[0]) != expected_scale_bits) { + std::cerr << "DIVISION_CHAIN_FAIL method=" << method_name + << " split_packed=" << unsigned(chain_qs[0]) << " fused_packed=" << unsigned(chain_qf[0]) + << " split_scale_bits=" << float_bits(chain_ss[0]) << " fused_scale_bits=" << float_bits(chain_sf[0]) << std::endl; + throw std::runtime_error("fixed transform/quantize/fused division-chain regression failed"); + } + ++summary.contract_checks; // 每个dtype/Method计一个串联回归，不把其中三次API发射算成三个用例。 + std::cout << "DIVISION_CHAIN_PASS dtype=" << (storage() == api::StorageType::FP16 ? "fp16" : "bf16") + << " method=" << method_name + << " input=[0.75,0.25] transform=[1,0.5] scale_bits=0x3e124925 packed=0x37 split_fused_exact=true guards=true contract_increment=1" + << std::endl; + } + const auto invalid = static_cast(-1); + reject(launch_transform(input.data(), output.data(), 1, 8, 1, stream, invalid)); + reject(launch_quantize(input.data(), packed.data(), scales.data(), 1, 8, stream, invalid)); + reject(launch_fused(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, invalid)); + reject(launch_transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, nullptr, invalid)); + CHECK(aclrtSynchronizeStream(stream)); + output.download(stream, true); +} + +template void large_m_check(aclrtStream stream, Summary& summary) { + constexpr std::size_t rows = 262145; + constexpr std::uint32_t blocks = 32; + const auto input = make_input(rows, 1, 0, 1847); + Guarded x(rows, stream, "large-m.input"), scalar(rows, stream, "large-m.scalar"), + vector(rows, stream, "large-m.vector"), inplace(rows, stream, "large-m.inplace"); + x.upload(input, stream); inplace.upload(input, stream); + std::cout << "LARGE_M_START rows=" << rows << " n=1 block_dim=" << blocks << " operations=transform,inplace" << std::endl; + CHECK(launch_transform(x.data(), scalar.data(), rows, 1, 1, stream, api::Method::ScalarButterfly, blocks)); + CHECK(launch_transform(x.data(), vector.data(), rows, 1, 1, stream, api::Method::VectorGather, blocks)); + CHECK(launch_transform(inplace.data(), inplace.data(), rows, 1, 1, stream, api::Method::VectorGather, blocks)); + const auto a = scalar.download(stream), b = vector.download(stream), c = inplace.download(stream); + if (std::memcmp(input.data(), a.data(), rows * sizeof(T)) || std::memcmp(input.data(), b.data(), rows * sizeof(T)) + || std::memcmp(input.data(), c.data(), rows * sizeof(T))) throw std::runtime_error("large-M N1 identity mismatch"); + x.download(stream, true); + summary.large_m_cases = 1; + summary.large_m_elements = rows; + std::cout << "LARGE_M_PASS rows=" << rows << " n=1 block_dim=" << blocks << std::endl; +} + +template Summary validate(aclrtStream stream, const char* dtype, const Options& options) { + Summary result; + const std::vector dims = options.custom_shape ? std::vector{options.dim} + : (options.quick ? std::vector{1, 64, 256} : std::vector{1, 2, 4, 8, 16, 32, 64, 128, 256}); + const std::vector rows = options.custom_shape + ? std::vector{checked_shape(options.batch, options.seq, options.heads, options.dim)} + : (options.quick ? std::vector{3} : std::vector{1, 3, 17, 257}); + for (int n : dims) { + for (auto r : rows) { + for (int normalized = 0; normalized < (n == 1 ? 1 : 2); ++normalized) { + const float scale = normalized ? 1.0f / std::sqrt(static_cast(n)) : 1.0f; + for (int pattern = 0; pattern < 5; ++pattern) { + // 零值/脉冲无随机性，只计一次；N=1 的两种 scale 相同，也只计一次。 + const int seeds = pattern < 3 && !options.quick ? 3 : 1; + for (int seed = 0; seed < seeds; ++seed) + one_case(stream, r, n, scale, pattern, 123 + 7919 * seed, dtype, result); + } + } + } + std::cout << "VALIDATION_PROGRESS dtype=" << dtype << " n=" << n << " cases=" << result.cases << std::endl; + } + if (!options.custom_shape) { + const auto original_blocks = configured_blocks; + Summary grid; + for (std::uint32_t blocks = 1; blocks <= 32; ++blocks) { + if (options.quick && blocks != original_blocks) continue; + configured_blocks = blocks; + for (int n : {1, 256}) one_case(stream, 33, n, 1.0f, 0, 1741, dtype, grid); + std::cout << "GRID_PROGRESS dtype=" << dtype << " rows=33 block_dim=" << blocks << std::endl; + } + configured_blocks = original_blocks; + result.grid_cases = grid.cases; + result.grid_elements = grid.elements; + result.grid_max_rounded_error = grid.max_rounded_error; + result.grid_max_unrounded_error = grid.max_unrounded_error; + } + if (!options.quick && !options.custom_shape && !options.skip_stress) { + large_m_check(stream, result); + } + contract_tests(stream, result); + std::cout << "VALIDATION_PASS dtype=" << dtype << " cases=" << result.cases << " elements=" << result.elements + << " max_rounded_error=" << std::setprecision(12) << result.max_rounded_error + << " max_unrounded_error=" << result.max_unrounded_error << " contract_checks=" << result.contract_checks << std::endl; + return result; +} + +template void benchmark(aclrtStream stream, const char* dtype, const Options& o, std::ofstream& csv) { + struct Shape { std::size_t b, s, h; int n; }; + std::vector shapes; + if (o.custom_shape) shapes.push_back({o.batch, o.seq, o.heads, o.dim}); + else for (int n : {64, 128, 256}) for (auto rows : {1, 17, 257}) + shapes.push_back({1, static_cast(rows), 1, n}); + const std::array names{{"scalar_transform", "vector_transform", "scalar_split", + "vector_split", "scalar_fused", "vector_fused", "quant_only"}}; + for (const auto shape : shapes) { + const auto rows = checked_shape(shape.b, shape.s, shape.h, shape.n), count = rows * shape.n; + const auto input = make_input(rows, shape.n, 0, 2909); + Guarded x(count, stream), y(count, stream); + Guarded q(rows * ((shape.n + 1) / 2), stream); + Guarded s(rows, stream); + x.upload(input, stream); + const float scale = 1.0f; + // quant_only 的输入在计时前生成；两方法共享同一量化实现，只报告一条量化基准。 + CHECK(launch_transform(x.data(), y.data(), rows, shape.n, scale, stream, api::Method::ScalarButterfly)); + auto launch = [&](int which) { + const auto method = which % 2 ? api::Method::VectorGather : api::Method::ScalarButterfly; + if (which < 4) CHECK(launch_transform(x.data(), y.data(), rows, shape.n, scale, stream, method)); + if (which >= 2 && which < 4) CHECK(launch_quantize(y.data(), q.data(), s.data(), rows, shape.n, stream, method)); + if (which >= 4 && which < 6) CHECK(launch_fused(x.data(), q.data(), s.data(), rows, shape.n, scale, stream, method)); + if (which == 6) CHECK(launch_quantize(y.data(), q.data(), s.data(), rows, shape.n, stream, api::Method::ScalarButterfly)); + }; + for (int which = 0; which < 7; ++which) for (int i = 0; i < o.warmup; ++i) launch(which); + CHECK(aclrtSynchronizeStream(stream)); + for (int group = 0; group < o.groups; ++group) { + // 各组轮换方法顺序；两端事件之间无分配、CPU 参考或主机设备复制。 + for (int order = 0; order < 7; ++order) { + const int which = (order + group) % 7; + aclrtEvent begin = nullptr, end = nullptr; + // 每个样本新建时间线 event，避免假设旧 event 的 reset/reuse 语义。 + CHECK(aclrtCreateEventWithFlag(&begin, ACL_EVENT_TIME_LINE)); + try { CHECK(aclrtCreateEventWithFlag(&end, ACL_EVENT_TIME_LINE)); } + catch (...) { aclrtDestroyEvent(begin); throw; } + float elapsed = 0; + try { + CHECK(aclrtRecordEvent(begin, stream)); + for (int i = 0; i < o.repeats; ++i) launch(which); + CHECK(aclrtRecordEvent(end, stream)); + CHECK(aclrtSynchronizeStream(stream)); + CHECK(aclrtEventElapsedTime(&elapsed, begin, end)); + } catch (...) { + aclrtSynchronizeStream(stream); aclrtDestroyEvent(begin); aclrtDestroyEvent(end); throw; + } + CHECK(aclrtDestroyEvent(begin)); CHECK(aclrtDestroyEvent(end)); + const double us = static_cast(elapsed) * 1000.0 / o.repeats; + if (!(us > 0) || !std::isfinite(us)) throw std::runtime_error("invalid event timing"); + const std::size_t logical_bytes = which < 2 ? count * sizeof(T) * 2 + : (which < 4 ? count * sizeof(T) * 3 : count * sizeof(T)) + rows * ((shape.n + 1) / 2) + rows * sizeof(float); + csv << dtype << ',' << shape.b << ',' << shape.s << ',' << shape.h << ',' << shape.n << ',' << rows + << ',' << names[which] << ',' << group << ',' << order << ',' << o.repeats << ',' << std::setprecision(12) << us + << ',' << logical_bytes << ',' << logical_bytes / us / 1000.0 << ',' << count * sizeof(T) + << ",2909,true,1," << configured_blocks << ',' << o.warmup << ",acl_timeline_event_ms," + << (kVectorScaleEnabled ? "true" : "false") << ',' << us / 1000.0 << '\n'; + csv.flush(); + } + } + x.download(stream, true); + const auto output = y.download(stream); + const auto packed = q.download(stream); + const auto scales = s.download(stream); + std::vector actual(count); + for (std::size_t i = 0; i < count; ++i) actual[i] = read(output[i]); + const auto quantized = hadamard::quantize_int4(actual, shape.n); + if (packed != quantized.packed || scales != quantized.scales) + throw std::runtime_error("benchmark-size fused INT4 differs from CPU quantization"); + for (std::size_t row : {std::size_t(0), rows / 2, rows - 1}) { + std::vector sample(shape.n); + for (int i = 0; i < shape.n; ++i) sample[i] = read(input[row * shape.n + i]); + const auto expected = hadamard::dense_reference(sample, shape.n, scale); + for (int i = 0; i < shape.n; ++i) { + const double error = std::abs(actual[row * shape.n + i] - read(rounded(static_cast(expected[i])))); + if (!(error < (std::string(dtype) == "fp16" ? .01 : .05))) + throw std::runtime_error("benchmark-size sampled dense oracle mismatch"); + } + } + std::cout << "BENCHMARK_PROGRESS dtype=" << dtype << " rows=" << rows << " n=" << shape.n << std::endl; + } +} + +void write_summary(std::ostream& f, const char* dtype, const Summary& s) { + f << '"' << dtype << "\":{\"cases\":" << s.cases << ",\"elements\":" << s.elements + << ",\"exact_scalar_vector_elements\":" << s.exact_transform_elements + << ",\"fp32_fwht_output_bits_exact\":true" + << ",\"max_abs_error_rounded_fp64\":" << std::setprecision(15) << s.max_rounded_error + << ",\"max_abs_error_unrounded_fp64\":" << s.max_unrounded_error + << ",\"api_contract_checks\":" << s.contract_checks + << ",\"grid_cases\":" << s.grid_cases << ",\"grid_elements\":" << s.grid_elements + << ",\"grid_max_abs_error_rounded_fp64\":" << s.grid_max_rounded_error + << ",\"grid_max_abs_error_unrounded_fp64\":" << s.grid_max_unrounded_error + << ",\"large_m_cases\":" << s.large_m_cases << ",\"large_m_elements\":" << s.large_m_elements << '}'; +} + +int main(int argc, char** argv) { + Options options; + try { options = parse(argc, argv); } + catch (const std::exception& e) { std::cerr << "INVALID_ARGUMENT " << e.what() << '\n'; return 2; } + aclrtStream stream = nullptr; + bool initialized = false, device_set = false; + try { + configured_blocks = options.block_dim; + CHECK(aclInit(nullptr)); initialized = true; + CHECK(aclrtSetDevice(0)); device_set = true; + CHECK(aclrtCreateStream(&stream)); + const char* soc = aclrtGetSocName(); + if (!soc || std::string(soc) != "Ascend910B1") throw std::runtime_error("this build requires observed SOC Ascend910B1"); + std::cout << "DEVICE soc=" << soc << " execution=npu block_dim=" << configured_blocks + << " vector_scale_enabled=" << (kVectorScaleEnabled ? "true" : "false") << std::endl; + Summary fp16, bf16; + if (options.validate) { + if (options.dtype != "bf16") fp16 = validate(stream, "fp16", options); + if (options.dtype != "fp16") bf16 = validate(stream, "bf16", options); + std::ofstream json(options.json); + if (!json) throw std::runtime_error("cannot create validation JSON " + options.json); + json << "{\"status\":\"PASS\",\"full_matrix\":" << ((!options.quick && !options.custom_shape && options.dtype == "both") ? "true" : "false") + << ",\"full_suite_complete\":" << ((!options.quick && !options.custom_shape && !options.skip_stress && options.dtype == "both") ? "true" : "false") + << ",\"execution\":\"npu\",\"soc\":\"Ascend910B1\",\"main_block_dim\":" << configured_blocks + << ",\"methods\":[\"scalar_butterfly\",\"vector_gather\"]" + << ",\"vector_scale_enabled\":" << (kVectorScaleEnabled ? "true" : "false") + << ",\"oracle\":\"FP32 CPU FWHT exact output bits plus all-element dtype-rounded FP64 dense\",\"fp16_tolerance_strict\":0.01,\"bf16_tolerance_strict\":0.05," + << "\"large_m_definition\":\"262145 rows, N1, both transforms and vector in-place, block_dim32; no large-M INT4 claim; does not exercise indices above 2^32\"," + << "\"large_m_skipped\":" << ((options.quick || options.custom_shape || options.skip_stress) ? "true" : "false") << ',' + << "\"warmup_not_counted\":true,\"dtypes\":{"; + bool comma = false; + if (options.dtype != "bf16") { write_summary(json, "fp16", fp16); comma = true; } + if (options.dtype != "fp16") { if (comma) json << ','; write_summary(json, "bf16", bf16); } + json << "}}\n"; + if (!json) throw std::runtime_error("failed writing validation JSON"); + } + if (options.benchmark) { + std::ofstream csv(options.csv); + if (!csv) throw std::runtime_error("cannot create benchmark CSV " + options.csv); + csv << "dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale,block_dim,warmup,timer,vector_scale_enabled,kernel_ms\n"; + if (options.dtype != "bf16") benchmark(stream, "fp16", options, csv); + if (options.dtype != "fp16") benchmark(stream, "bf16", options, csv); + if (!csv) throw std::runtime_error("failed writing benchmark CSV"); + } + CHECK(aclrtDestroyStream(stream)); stream = nullptr; + CHECK(aclrtResetDevice(0)); device_set = false; + CHECK(aclFinalize()); initialized = false; + std::cout << "PASS requested validation/benchmark operations completed" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "FAIL " << e.what() << std::endl; + if (stream) { aclrtSynchronizeStream(stream); aclrtDestroyStream(stream); } + if (device_set) aclrtResetDevice(0); + if (initialized) aclFinalize(); + return 1; + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/biren/README.md b/03_hadamard_tc/a962695448-rgb/platforms/biren/README.md new file mode 100644 index 00000000..7d7f29f4 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/biren/README.md @@ -0,0 +1,283 @@ +# 壁仞 106M 原生 SUPA 后端 + +本目录是 Hadamard 项目的独立壁仞移植。原始 Warp32、可选均衡打包和可选小批量发射三个固定版本，分别在壁仞 106M 完成原生 SUPA 构建、1504 组变换验证、180 项 API 契约检查、14 项 CLI 拒绝检查和 12 项独立主机 BF16 舍入自检。相同矩阵跨版本不累加成新覆盖。小批量版本另完成固定留出矩阵的交错独立进程复测。 + +**正确性通过不等于某个方法在所有形状上更快：原始 Warp32 有明显退化，均衡打包不适合全局启用，小批量选项也只针对受测边界。**下文保留全部版本、负例、探索轮次和选择理由，API 默认仍为共享内存 `Optimized`。 + +壁仞当前设备为 warp32，天数和沐曦已经验证的 Warp64 路径不能直接套用。实现、测试、版本和原始结果均需独立归档，不继承其他平台的通过数量或速度比。 + +## 当前实机环境与资源边界 + +| 项目 | 已观察到的状态 | +|---|---| +| 设备 | 壁仞 106M，界面标称显存 32GB | +| 设备 API 可见显存 | 34,091,302,912 字节；保留该原始数值，不把标称容量和可见容量混为一列 | +| 执行资源 | 64 个 SM、warpSize=32、每 block 最大 1024 线程、共享内存 32 KiB | +| SDK | SUPA SDK `1.10.0.1.rc1` | +| 编译器 | `brcc 1.10.0`，Clang `16.0.1` | +| 设备管理工具 | BRSMI `1.10.2`、Driver `1.10.1`、SUPA `1.10` | +| 运行库 | 原生 `libsupa-runtime.so` | +| CUDA 兼容层 | 当前环境未找到 SUDA，不以 CUDA API 兼容层为前提 | +| Python 框架 | 当前 PyTorch 为 `2.8+cpu`，不能据此宣称已提供壁仞 GPU PyTorch 扩展 | +| 虚拟化显示 | SVI 显示 Disabled；该显示本身不能证明没有其他任务共享资源或已经获得独占整卡 | + +SDK、驱动、设备工具和框架版本分别记录，不将其合并为一个推测版本。后续基准还需要记录采样时的设备状态、共享/独占证据及工具权限；不能只凭 SM 数、显存容量或 SVI 状态宣称独占与固定时钟。 + +## 原生 SUPA 路线 + +本平台使用真实 `brcc` 编译 `.su` 源文件，包含已安装 SDK 的 `supa.h`、`device/supa_fp16.h`、`device/supa_bf16.h`，链接本机 `libsupa-runtime.so`。项目源文件只引用厂商接口，**不将私人 SDK 头文件实现复制进公开仓库**。使用者需要单独安装相应 SDK。 + +运行时使用 SUPA 的 `suError_t`、`suStream_t`、`suEvent_t` 和 `su*` 函数；FP16/BF16 存储类型为 `float16`、`bfloat16`。这不是给 CUDA 函数名简单加宏，也不是将 CPU 结果作为设备输出。 + +迁移时已经核对的接口差异包括： + +- 设备分配使用 `suMallocDevice`，不能推测成 `suMalloc`。 +- `suMemcpyAsync` 的参数顺序为 `(dst, src, bytes, stream, kind)`，stream 位于方向枚举之前；不能沿用 CUDA 顺序。 +- 使用 SUPA 自己的 stream/event 句柄和错误枚举，不与 CUDA 句柄混传。显式检查发射返回值和后续 stream 同步的执行错误。 +- 初始化、上传、kernel、回读放在同一个显式 stream，不能假设 SUPA 与 CUDA 有相同的默认 stream 隐式依赖规则。 +- 编译器使用真实 SUPA 宏 `__SUPACC__`，设备分支由 `__SUPA_ARCH__` 区分；不伪造 `__CUDACC__`、`__ILUVATAR__` 或 `__MACACC__`。 + +不能将天数的 `-x ivcore`、沐曦的 cucc/cu-bridge 或 NVIDIA 的 `-arch=sm_89` 命令照搬到本平台。最终复现命令应以独立 runner 的真实 brcc 及 include/lib 路径为准。 + +## BF16 最近偶数舍入：先保留失败，再修正语义 + +最初直接使用原生 BF16 转换时，实机样例输入 FP32 位模式 `0x3f818000`：独立最近偶数舍入参考期望 BF16 `0x3f82`，设备却给出 `0x3f81`，探针明确失败。该输入恰好位于两个 BF16 表示值中点，低候选尾位为奇数，因此按最近偶数规则应选高候选。 + +这条记录证明**该设备/SDK 的这一路原生转换不能在项目里直接当成已经满足 RNE 的操作**。不能把参考答案改成 `0x3f81`、放宽阈值或隐藏 BF16 来绕过失败，也不把一个反例泛化为所有未来 SDK 的行为。 + +修正使用本项目自行实现的位级 RNE helper：先从 FP32 位模式计算最近偶数的 BF16 表示，再得到一个 BF16 可精确表示的 FP32 值，最后转为 SUPA 的 `bfloat16` 存储。合法位转换使用固定大小的 `memcpy`，不通过违反别名规则的指针强转读取。此实现遵循格式定义，不复制厂商 SDK 的转换代码。 + +输入生成、设备变换输出、独立参考输出及融合量化的中间舍入必须统一到同一 RNE 契约；尤其不能用原生 BF16 主机构造器的不同舍入行为生成“参考”，否则设备和参考同时犯错也可能误判通过。最终仍须通过独立 FP64 数值参考与位模式/边界检查，而不是只比较两个共享同一路 helper 的实现。 + +原失败与修正探针两个独立源文件版本及编译/运行日志已保留在 [initial](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/initial/transfer_manifest.json)：[原始失败](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/initial/smoke_run.log)、[修正通过](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/initial/smoke_rne_run.log)。单独的探针不代表完整 Hadamard；后文提供随后独立运行的完整结果。这些结果仍**不证明硬件原生 BF16 算术加速**，实现使用 BF16 存储/转换与 FP32 内部加减。 + +## 最小实机探针的准确范围 + +- FP16、BF16 各 257 个样例，覆盖随机正负数、正负零、舍入中点和部分边界值。修正后的 BF16 存储位模式、随后转换为 FP32 的简单算术与独立宿主机参考一致。 +- 设备缓冲区前后各 17 个元素的哨兵、输入保持和仅 2 字节对齐的半精度存储检查通过，覆盖不满线程块的尾部。 +- 两个完整的 32-lane warp，对 XOR 距离 `1、2、4、8、16` 独立检查，共 320 个观察值通过。每个 warp 使用可区分的数据，防止跨 warp 取错值被相同输入掩盖。 + +这里的 shuffle 通过结果仅覆盖已运行的完整 warp/整数交换。它不等于浮点蝶形、多寄存器交互、尾行、跨 CTA 网格复用或融合量化已经通过。SUPA 接口名称带有 `_sync` 时，也不能未经核验就套用 CUDA 对活跃线程 mask 的保证；完整 warp 参与、无效逻辑元素填零和分支收敛仍需在实现中保证。 + +## 完整 API、复现入口与待验收项 + +独立接口见 [hadamard_api.h](hadamard_api.h)，实现见 [hadamard_api.su](hadamard_api.su)，命名空间为 `hadamard::biren`。声明提供 `transform`、`quantize_int4`、`transform_int4` 的 `float16`/`bfloat16` 重载，返回 `suError_t`，使用 `suStream_t`。方法为 `Baseline`、共享内存 `Optimized`、条件编译的 `Warp32`，默认仍为 `Optimized`；固定版本的三种方法均有完整正确性证据，性能结论则分别列出。 + +接口为原生 C++ 异步调用：连续 FP16/BF16 `[rows,n]`，N 为 1～256 的 2 次幂，内部 FP32；连续 `[batch,seq,heads,head_dim]` 在检查乘法溢出后展平成 rows，不接受 stride 参数。scale 必须有限且为正；输入应有限，变换输出应在存储类型的有限范围内。 + +INT4 目标语义为 `[-7,7]`，每行 `s=max(abs(x))/7`，全零行取 1，最近偶数舍入，偶数元素在低四位，N=1 的空高四位为零；融合路径先按 FP16/BF16 存储类型舍入再量化。 + +调用方提供 `rows*n` 个输入/变换输出元素、`rows*ceil(n/2)` 个 packed 字节及 `rows` 个 float scales。半精度指针要求 2 字节对齐、scales 要求 float 对齐；transform 只允许完全原位或互不重叠，量化各缓冲区互不重叠。API 检查参数、对齐、大小溢出和重叠，但不查询真实分配容量或设备归属。 + +所有发射都使用调用方 stream，不分配、不复制、不等待。调用方检查发射错误，并在使用回读结果前同步相应 stream。合法 `rows=0` 只检查 N、scale、方法后返回，不发射内核。 + +Warp32 需要真实 SUPA 编译环境和 `-DHADAMARD_BIREN_WARP32`，调用方还须在初始化时经 `suGetDeviceProperties` 确认当前设备 `warpSize==32`；该查询不在每次 API 发射路径内进行。未编译支持的非空 Warp32 调用返回 `suErrorNotSupported`，不静默回退。其他同为 warp32 的设备仍不能直接继承本平台语义或性能结论。 + +### 固定版本复现命令 + +runner 位于 [run_platform.py](run_platform.py)，原生验证程序为 [validate_and_benchmark.su](validate_and_benchmark.su)。复现下面原始 Warp32 记录时，先固定 `2cbaf41` 源码；从项目 `03_hadamard_tc/a962695448-rgb` 目录执行，结果目录必须是新目录。后续可选实验不自动继承本轮性能结果： + +```bash +# 快速工具链/小矩阵检查，不代表完整验收。 +python3 platforms/biren/run_platform.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 --warp32 --quick --no-benchmark \ + --output results/biren/warp32_quick_01 + +# 完整矩阵及九路径基准。 +python3 platforms/biren/run_platform.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 --warp32 --repeats 100 --groups 5 \ + --output results/biren/warp32_full_01 + +# 如仅复测共享路径，省略 --warp32 并使用新目录。 +python3 platforms/biren/run_platform.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 --no-benchmark \ + --output results/biren/shared_full_01 +``` + +`--sdk-root` 默认 `/usr/local/birensupa/sdk/latest`，`--compiler` 默认该 SDK 下的 `brcc/bin/brcc`；runner 根据实际解析的 SDK 目录读取 `supa/include` 与 `supa/lib`。编译参数为 `-x supa -std=c++17 -O2`、项目/SDK include 路径、两个 `.su` 文件以及 `-lsupa-runtime` 和相应 rpath；`--warp32` 额外定义项目专用宏。这里的 `--warp32` 是 runner 的构建选项，不是生成二进制的运行参数。 + +runner 仅为子进程设置 `SUPA_PATH=SDK/supa`、`BIREN_HOME=SDK`、工具搜索路径及实际存在的 SUPA/brcc 库目录，不修改用户系统环境或替换框架。设备库搜索需要前者指向 `supa` 子目录，不能误指整个 SDK 根目录。缺失编译器、构建失败、检查失败都应以非零退出码结束；结果目录已存在则拒绝覆盖。 + +本次 Warp32 完整模式实际完成 1504 组变换、180 项 API 契约检查、14 项 CLI 拒绝检查，另外有 12 项独立写出的主机 BF16 RNE 边界检查；九路径基准每轮 1350 条组样本。主机检查、探针观察值和相同矩阵重复运行不叠加为 GPU 覆盖。 + +输出包括构建日志、CLI 拒绝日志、验证 JSON/日志、可选基准 CSV/日志及 `run_summary.json`。摘要固定实际源文件 SHA256、Git HEAD/状态、SDK/编译器、命令和退出码；其中算法来源提交与本次实际壁仞测试提交是不同字段，不混为同一版本。 + +## 原始 Warp32 版本的完整实机证据 + +实际测试提交为 `2cbaf41d9f71a657ca2cca027a3083d9d497322d`，工作树干净，五个源文件 SHA256 已与该提交的 Git 对象逐一核对。原生 brcc 构建退出 0、构建日志为空；完整 runner 为 `PASS`，`quick=false`、`full_matrix=true`、`warp32_enabled=true`。源版本、命令、SDK 环境、原始日志与各阶段退出码见 [run_summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/run_summary.json)。 + +| 项目 | 当前状态 | 原始证据 | +|---|---|---| +| 原生构建与探针 | 原始 BF16 反例保留，RNE 修正探针通过 | [初始清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/initial/transfer_manifest.json) | +| 三种方法、BF16 RNE 与独立参考 | 1504 组变换及 12 项主机格式检查通过 | [validation.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/validation.json) | +| API、内存与 INT4 | 180 项 API、14 项 CLI，字节与 scale 精确一致 | [完整日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/validation.log) 及各 `invalid_*.log` | +| 同机三轮基准 | 4050 条原始组样本，三进程退出 0 | [repeat_runs.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/repeat_runs.json) | +| 复现与归档 | 固定源码、环境及逐文件 SHA256 | [公开清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/manifest.json) | + +完整矩阵覆盖 N=1～256 的全部 2 次幂、普通行数 1/3/17/257、scale=1 和 `1/sqrt(N)`，N=1 不重复计算相同 scale。输入为均匀 `[-1,1)`、标准差 0.5 的正态分布、幅度 8 离群值、全零和单位脉冲；随机分布使用种子 123/8042/15961，确定性输入只计一次。另有 N=1/2、rows=65537 和 262145 的网格复用/尾行检查。 + +| 精度 | 实际变换组合 | 检查元素 | API 契约检查 | 已舍入参考最大绝对误差 | 未舍入参考最大绝对误差 | +|---|---:|---:|---:|---:|---:| +| FP16 | 752 | 4,105,264 | 90 | 0.00390625 | 0.015620231628418 | +| BF16 | 752 | 4,105,264 | 90 | 0.000003814697265625 | 0.124984741210938 | + +每 dtype 的所有元素在基线、共享优化、Warp32 及对应原位路径上逐字节一致；CPU 对实际设备输出量化、各方法分步与融合量化的全部 packed bytes 和 float scales 精确一致。262145 行 N=1/2 两例合计 786,435 元素，已经包含在表内，不重复加总。 + +完整验证沿用项目已舍入参考：独立 FP64 稠密 Hadamard 结果经 FP32 转换后，按输出 dtype 的 RNE 舍入再比较，FP16 绝对误差严格小于 `1e-2`、BF16 严格小于 `5e-2`。**阈值判定仅针对已舍入列；未舍入误差包含最终存储舍入，不能也说它低于该阈值。**主机 BF16 参考使用独立高/低位比较及手写边界，不调用设备 helper 或将原生不同舍入当成预期。 + +### SDK 路径错误的失败记录 + +最初的 `8040b60` quick 构建因设备库搜索路径错误而失败，brcc 报无法找到 `libsupadevice`；[原始构建日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/diagnostics/runs/biren-api-quick-8040b60/build.log) 和 [运行摘要](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/diagnostics/runs/biren-api-quick-8040b60/run_summary.json) 均保留。修正为 `SUPA_PATH=SDK/supa`，使编译器找到该 SDK 的设备库，随后 `2cbaf41` 成功构建并完成全部运行。没有使用关闭设备库链接的参数绕开问题，先前失败也不计为通过。 + +### 同机三轮性能：保留 Warp32 退化 + +三轮使用同一二进制 SHA256 `aa7e015ef5ce8a991c3431ef1c299a1529da4d093028163029a5dd4aaf641901`。两 dtype × N=64/128/256 × rows=1/17/257/4096/16384，共 30 个 shape/dtype 条件；基线、共享、Warp32 各 transform/split/fused 三种操作，共九路径。每路径先预热 10 次，五组、每组重复 100 次，组间轮换方法顺序，每轮 1350 条组样本。 + +同轮相同 shape/dtype/scale/操作先取五组中位数再匹配比较：共享优化版有 **64/90** 组每轮比基线减少至少 5% 耗时，Warp32 有 **81/90** 组。Warp32 相对基线有六组稳定退化，全部是 rows=17 的 transform；相对共享版有 29/90 组更慢，只有 49/90 组每轮达到至少 5% 降时。因此默认仍保留共享 `Optimized`，不能因使用 warp shuffle 就声称更快。 + +| 操作 | 基线 / 原始 Warp32 | 共享优化版 / 原始 Warp32 | +|---|---:|---:| +| transform | 0.87596～1.95562 倍 | 0.85528～1.84059 倍 | +| split | 1.08715～2.07062 倍 | 0.83266～1.69125 倍 | +| fused | 1.02949～2.99272 倍 | 0.51151～1.76663 倍 | + +比值低于 1 表示 Warp32 更慢。最明显的已记录退化是 FP16 `[16,64,16,256]` fused：Warp32 约 1416～1418 微秒，共享版约 725 微秒，慢约 95.32%～95.50%。负例与其他结果完整保存在三轮 [第一轮 CSV](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/benchmark.csv)、[第二轮 CSV](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/benchmark_run2.csv)、[第三轮 CSV](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/benchmark_run3.csv) 及 [中性事件描述分析](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/analysis/analysis.json)。 + +Warp32 内部分步/融合比为约 1.600～2.089 倍，融合降时约 37.52%～52.13%。**内部融合有效不代表胜过共享实现**；不能用这个比值隐藏上面的跨方法退化。所有范围只覆盖当前 N=64/128/256 的实测性能矩阵，不外推 N=1～32。 + +### 属性诊断与后续实验边界 + +独立属性程序及日志保存在 [diagnostics](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/diagnostics/manifest.json)，它查询 kernel 元数据，不执行性能测量。旧版及后续 balanced-pack 实验的 N=256 Warp fused 均报告 `numRegs=72`、`localSizeBytes=0`，共享版报告 48 个寄存器。**不能据此说已发生寄存器 spill，也不能把寄存器差异当成性能退化的已证实因果。** + +下面分别报告可选 balanced-pack 和小批量发射实验。它们具有独立源码提交、编译开关和原始记录；不能将某一路的局部改善扩展为其他路径或其他版本的收益。 + +## 可选均衡打包实验：保留局部改善与负例 + +固定提交 `c01ac87fed92dc5d9bdf8bcf27f3dc9243a2818f` 的 `--balanced-pack` 仅改动 Warp32 的 INT4 打包路径，影响 split/fused，不改变 transform。完整运行为 PASS：1504 组、180 API、14 CLI、12 主机 BF16 格式检查；具体源码和二进制 hash 见 [run_summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/balanced_c01ac87/run_summary.json)，实际计数及误差见 [validation.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/balanced_c01ac87/validation.json)。 + +本实验保存了 **五轮共 6750 条原始组样本**。第 2/3 轮与属性程序的 CPU 编译共同启动，在查看这些结果前已记录排除决定，主统计选用第一次 `benchmark.csv` 和新的 `benchmark_run4.csv`、`benchmark_run5.csv`，共 4050 条。原始第 2/3 轮完整保留，[timing_selection_note.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/balanced_c01ac87/timing_selection_note.json) 记录提前决定的原因；不能说本实验总共只有 4050 条，也不能只发布挑出的快样本。 + +同轮形状/精度/scale 匹配后，主三轮的基线/候选比为 split 约 **1.104～2.180 倍**、fused **1.020～2.784 倍**；共享优化版/候选则分别为 **0.837～1.653 倍**、**0.548～1.740 倍**。低于 1 表示候选更慢，不能据融合有效就宣布全面胜过共享版。 + +与旧版不同时间段的阶段中位数对照，FP16 rows=16384、N=256 fused 从约 1417.44 降至 1323.65 微秒，减少 6.62%；该条件的基线和共享控制路径分别变化约 +0.11% 和 -0.03%。但是候选仍比本轮共享版慢约 82.3%～82.5%。FP16 rows=257、N=128 fused 阶段中位下降约 16.74%，按共享控制归一后约 14.80%。 + +负例包括 BF16 rows=257、N=64 的阶段耗时增加约 8.63%，以及 FP16 rows=16384、N=128 增加约 7.18%。60 个量化操作比较中，42 个阶段中位更快、18 个更慢，其中七个退化超过 3%。**旧新版本不是同一时间段的同进程 A/B，这些是观察描述，不能当成已排除时间漂移的因果估计。**transform 未改，不能把它的波动计为改动收益。 + +结论是保留该可选实验供复现，不建议全局开启。全部五轮 CSV/日志、主样本选择和派生分析都在 [balanced_c01ac87 清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/balanced_c01ac87/manifest.json)；[分析](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/balanced_c01ac87/analysis/analysis.json) 明确引用 1/4/5。属性记录中旧新版本 N=256 Warp fused 都是 72 个寄存器、local=0，不能据此声称修复了 spill。 + +## 可选小批量发射与留出验证 + +固定提交 `8f75553a074d79b850294377d5aea6381e93da19` 的 `--small-batch-warp` 只在 `rows<=64` 时把 Warp32 发射改为每 CTA 一行、32 线程；超过阈值保持原发射。它可影响 Warp32 的 transform/split/fused，基线和共享实现不变。本次没有同时开启 balanced-pack。 + +完整矩阵独立通过 1504 组变换、180 API、14 CLI 和 12 主机 BF16 格式检查，三轮 4050 条组样本均成功结束。每 dtype 752 组、4,105,264 个元素，与基线、共享、原位和 INT4 参考一致；误差定义与前文保持一致。原始记录见 [validation.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/small_8f75553/validation.json)、[运行摘要](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/small_8f75553/run_summary.json) 和 [分析](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/small_8f75553/analysis/analysis.json)。 + +为检查阈值附近而不是只看已有调优点，另固定 `rows={32,63,64,65}`、`N={64,128,256}`、两 dtype，共 24 个 shape/dtype 条件。rows=65 是不触发新逻辑的阈值外控制点。每条件执行两种二进制的 22 个输入参数组合，共 **528 个不同的输入参数组合、48 个验证进程**，全部通过；重复执行的 API/主机自检不另算新增覆盖。 + +性能复测为三轮，每个条件交错运行旧/新两个独立进程，按预先固定的轮次/条件奇偶规则轮换顺序，共 **144 个独立基准进程**。这不是同进程 A/B。每个条件仍为九方法、五组、100 次重复，六份旧/新每轮合并 CSV 各有 1080 行，共 **6480 条不同的原始组样本**；合并 CSV 只是底层 raw CSV 的拼接，不算第二份新观察。 + +[holdout_summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/holdout_summary.json) 记录 PASS、固定矩阵、交错顺序、全部命令/退出码和来源文件。旧二进制 SHA256 与 `2cbaf41` 的构建摘要匹配，新二进制与 `8f75553` 匹配，运行前后 hash 未变；因此不仅依赖脚本调用方给出的源码标签。实验脚本本身不编译、不调参、不改变矩阵，调用方负责避免并发编译或 GPU 工作。 + +原始与分变体分析已归档：[旧版分析](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/analysis_old/analysis.json)、[新版分析](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/analysis_new/analysis.json)。独立复核核对了 391 份原始文件的 hash、旧新源码/二进制链及所有进程退出码；跨变体按相同轮次、形状、精度和操作直接配对，不将各自对 baseline 的比值当成旧新收益。 + +### 小批量留出结果与明确推荐范围 + +受新发射逻辑影响的 rows=32/63/64、三种 N、两 dtype、三操作，共 **54 个操作条件**。它们相对旧 Warp32 在三轮中全部更快，其中 **52/54** 每轮均减少至少 5% 耗时。最弱的 FP16 rows=63、N=64 fused 三轮只减少约 **3.946%～4.072%**；最强的 FP16 rows=63、N=256 fused 减少约 **53.120%～53.138%**。因此不能将 54/54“更快”写成 54/54“至少 5%”。 + +rows=65 不触发新发射逻辑，是阈值外控制。控制条件没有稳定达到每轮 5% 的变化，观察到的最差变化约为 **-1.541%**，不计作新方案功劳。这是交错独立进程的有限测量，不是已经排除所有系统波动的同进程因果实验。 + +与共享 `Optimized` 比较时，受影响的 54 个条件中仅 **47/54** 三轮更快，只有 **23/54** 每轮减少至少 5%。其余七个负例全部在 N=256，最差仍比共享版慢约 **9.825%**。小批量方案改善了旧 Warp32，不意味着它成为所有条件的最快方法。 + +**目前最明确的推荐范围是已测 rows=32/63/64、N=64、FP16/BF16 的 fused 路径**：相对同轮共享实现每轮耗时减少约 **10.09%～12.68%**。在本机与已验证 SDK 条件下，可显式编译小批量选项并选择 `Method::Warp32` 使用该路径；不据此改全局默认，也不扩展到未测 N/rows/设备。 + +主矩阵另外保留非交错阶段对照：rows=17 的六个 transform 条件，旧版三轮均比基线慢、新版三轮均比基线快，旧新阶段描述的耗时减少约 **7.773%～21.645%**。这不是前述留出实验的交错配对，不能直接作为同进程因果结论。rows=1 没有稳定 5% 收益，部分观察退化约 **2.44%**，所以不能宣传“所有小批量都更快”。 + +大输入 rows=16384、N=256 fused 不触发小批量逻辑，仍明显慢于共享版：FP16 慢约 **95.5%～95.7%**，BF16 慢约 **84.3%～84.6%**。这里继续选择共享实现更符合已取得的证据；balanced-pack 也不建议全局开启。 + +### 复现实验开关与固定留出脚本 + +两个可选开关必须与 `--warp32` 一起使用。下面的性能实验分别构建；两开关同时开启的组合已额外完成正确性检查，但没有组合性能测量，不能把任一单独实验的速度比套给组合： + +```bash +# 固定 c01ac87 源码，复现均衡打包实验。 +python3 platforms/biren/run_platform.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 \ + --warp32 --balanced-pack --output results/biren/reproduce_balanced_01 + +# 固定 8f75553 源码，复现小批量发射实验。 +python3 platforms/biren/run_platform.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 \ + --warp32 --small-batch-warp --output results/biren/reproduce_small_01 +``` + +自写留出脚本 [biren_small_batch_holdout.py](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/biren_small_batch_holdout.py) 的 SHA256 为 `614db3a3b32fc98708b02122a89d1dd42d4341e9308c68cc7d98f1c5b31fe9cc`，与实机摘要匹配。先分别用固定源码构建旧版和小批量版可执行文件，并保存构建来源；下例中的两个路径必须对应这两次构建的真实文件： + +```bash +python3 results/biren/holdout_8f75553/biren_small_batch_holdout.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 \ + --old-binary results/biren/reproduce_original_01/validate_and_benchmark \ + --new-binary results/biren/reproduce_small_01/validate_and_benchmark \ + --old-source-id 2cbaf41d9f71a657ca2cca027a3083d9d497322d \ + --new-source-id 8f75553a074d79b850294377d5aea6381e93da19 \ + --output results/biren/reproduce_holdout_01 +``` + +`--old-source-id`/`--new-source-id` 只是调用方标签，脚本不会凭标签验证源码。复现者必须另行核对二进制 hash、构建记录和源码提交。新结果目录不得已存在，脚本保留所有进程的原始 JSON/CSV/log，不搜索表现最好的参数。 + +## 最终编译组合检查：正确性与性能分别记录 + +同一固定源码 `8f75553a074d79b850294377d5aea6381e93da19` 另外执行两种构建配置，测试时工作树干净，两套各五个源文件 hash 均与该提交匹配： + +| 构建配置 | 实际验证范围 | API 与专门检查 | 性能测量 | +|---|---|---|---| +| `--warp32 --balanced-pack --small-batch-warp` | 完整 1504 组通过，`quick=false`、`full_matrix=true` | 180 API、14 CLI、12 主机 BF16 RNE 检查通过 | 未测量 | +| 不启用上述三个开关 | **仅 quick 50 组通过**，`quick=true`、`full_matrix=false` | 122 API、12 项未编入 Warp32 路径检查、12 主机 BF16 RNE 检查通过 | 未测量 | + +组合构建的二进制 SHA256 为 `776b7784f2420808c63932cbe871bfcc002e92e23c373622ef579dd46418aa27`，原始记录见 [组合验证](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/contract_variants_8f75553/combined/validation.json) 和 [组合运行摘要](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/contract_variants_8f75553/combined/run_summary.json)。默认无特性开关的二进制 SHA256 为 `eb022b93b6da249096e1e59069e7a300d29c782cea8edaa186a334ce7a4b43a9`，记录见 [默认 quick 验证](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/contract_variants_8f75553/default/validation.json) 和 [默认运行摘要](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/contract_variants_8f75553/default/run_summary.json)。默认构建也执行了 runner 的 CLI 拒绝检查；它与其他构建的重复检查不累计为新覆盖。 + +这确认了组合路径在现有完整矩阵中的正确性，以及默认构建的快速检查；**不能称默认无开关构建已独立跑过全量，也不能声称组合性能优于单独选项**。相同输入和重复 API/主机检查不叠加成更多独立用例，本次没有新增计时样本。 + +复现时固定相同源码，使用新的结果目录： + +```bash +python3 platforms/biren/run_platform.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 \ + --warp32 --balanced-pack --small-batch-warp --no-benchmark \ + --output results/biren/reproduce_combined_contract_01 + +python3 platforms/biren/run_platform.py \ + --sdk-root /usr/local/birensupa/sdk/1.10.0.1.rc1 --quick --no-benchmark \ + --output results/biren/reproduce_default_quick_01 +``` + +[final_jobs_and_source.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/contract_variants_8f75553/final_jobs_and_source.json) 保存采集时已经结束的作业退出码与源码状态，包括早期保留的失败，不能解释为“历史所有作业均成功”或实例释放回执。此批 39 份原始文件通过传输 hash 核验，自写脚本与两套构建/验证日志均收录在 [独立清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/contract_variants_8f75553/manifest.json)。 + +本次性能使用原生 SUPA event，区间排除分配、主机/设备复制、预热和验证；只读输入重复使用可能受缓存影响。事件区间可能含主机发射造成的空闲，不是端到端延迟或隔离测得的纯内核时长。逻辑读写量换算的 GB/s 不是实测物理显存带宽。SVI Disabled 不能证明没有共享干扰，少数重复运行也不是性能保证。 + +[原始版本清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/manifest.json) 保留 29 份完整运行原始文件和七份初始环境/探针的传输校验；[均衡打包](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/balanced_c01ac87/manifest.json)、[小批量版](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/small_8f75553/manifest.json)、[固定留出](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/manifest.json) 分别核验 39、29、391 份传输原始文件，并保留派生分析和自写脚本。诊断另有独立清单。全部目录 `.gitattributes` 使用 `-text`，逐字节保存日志和 CSV；测试源码 hash 对应固定 Git 提交，不能由后续工作树修改覆盖。 + +## 毫秒单位导出与有效数字 + +题目要求以毫秒汇报。原始 CSV 和本文中的微秒读数保留不变，五个分析目录分别增加 `method_summary_ms.csv`： + +- [原始 Warp32 毫秒表](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/2cbaf41/analysis/method_summary_ms.csv) +- [均衡打包毫秒表](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/balanced_c01ac87/analysis/method_summary_ms.csv) +- [小批量版本毫秒表](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/small_8f75553/analysis/method_summary_ms.csv) +- [留出旧版毫秒表](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/analysis_old/method_summary_ms.csv) +- [留出新版毫秒表](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/holdout_8f75553/analysis_new/method_summary_ms.csv) + +转换仅将所有以 `_us` 结尾的时间字段按十进制除以 1000，并将字段名改为 `_ms`；其他字段、列顺序和数据行顺序全部保留。**这是纯单位换算，不增加测量有效位数、精度或原始样本数。**例如 1417.44 微秒等于 1.41744 毫秒，不代表新增了一次更精确测量；百分比、比值、CV、样本计数等不换算。 + +自写 [convert_summary_ms.py](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/biren/diagnostics/convert_summary_ms.py) 使用十进制表示直接变更指数，不经过二进制浮点，也不覆盖原表或既有导出。可在项目目录执行： + +```bash +python3 results/biren/diagnostics/convert_summary_ms.py \ + results/biren/2cbaf41/analysis/method_summary.csv \ + results/biren/balanced_c01ac87/analysis/method_summary.csv \ + results/biren/small_8f75553/analysis/method_summary.csv \ + results/biren/holdout_8f75553/analysis_old/method_summary.csv \ + results/biren/holdout_8f75553/analysis_new/method_summary.csv +``` + +仓库已包含导出结果，脚本遇到已有 `_ms.csv` 会拒绝覆盖；重现转换时应在独立临时副本中使用尚无该导出的目录。导出表和脚本均纳入对应目录的 SHA256 清单。 + +公开交付只包括项目自行编写的源代码、复现入口、环境说明及脱离私人访问凭据的实验结果。私人 SDK 头文件、实例访问地址和租赁凭据不进入仓库。本平台按固定源码和原始日志核对实现与测试结果；PR 和课程登记在项目所有者验收具体代码及证据后单独执行。 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/biren/hadamard_api.h b/03_hadamard_tc/a962695448-rgb/platforms/biren/hadamard_api.h new file mode 100644 index 00000000..1e8d545b --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/biren/hadamard_api.h @@ -0,0 +1,55 @@ +// 算法来源：本项目天数后端固定提交 a387db3332c6f9b01f128dd681848260c9691281。 +// 壁仞原生 SUPA 后端独立验证；未复制任何厂商 SDK 实现。 +#pragma once + +#include +#include +#include +#include +#include + +namespace hadamard::biren { + +enum class Method { Baseline, Optimized, Warp32 }; + +// Warp32 为壁仞 32-lane warp 专用候选，须使用 +// -DHADAMARD_BIREN_WARP32 且由原生 SUPA (__SUPACC__) 编译；其他构建的 +// 非空操作返回 suErrorNotSupported。调用方必须预先确认当前设备 warpSize=32； +// API 不在发射路径中查询设备属性。rows=0 仍遵循下方的无操作约定。 + +// 连续设备内存：[rows, n]，n 为 1..256 的二次幂。scale 必须有限且为正数。 +// 支持 FP16/BF16 存储，内部 FP32；输入应有限，变换后的值应在输出类型范围内。 +// BF16 每次设备输出使用本项目显式最近偶数舍入，避免旧 SUPA 原生 cast 的截断。 +// 所有操作仅在调用方的 stream 上发射，不分配内存、不复制、不等待。 +// rows=0 时只检查 n、scale、method，允许空指针，不发射内核。 +// 输入和变换输出只要求 2 字节对齐；scales 要求 float 对齐。 +// transform 允许 input==output 原位变换，其余重叠均返回 suErrorInvalidValue。 +// 不检查分配容量/设备归属；调用方须提供当前设备有效且足够大的设备缓冲区。 +// 返回参数/发射错误；异步执行错误由调用方同步 stream 时检查。 +suError_t transform(const float16* input, float16* output, std::size_t rows, + int n, float scale, suStream_t stream, + Method method = Method::Optimized); +suError_t transform(const bfloat16* input, bfloat16* output, + std::size_t rows, int n, float scale, suStream_t stream, + Method method = Method::Optimized); + +// 每行 ceil(n/2) 字节；偶数元素在低 4 位，奇数在高 4 位，n=1 的高位为 0。 +// q=clamp(round_to_nearest_even(x/s), -7, 7)，s=max(abs(x))/7；全零行 s=1。 +// scales 为 rows 个 float。所有输入/输出缓冲区必须互不重叠。 +suError_t quantize_int4(const float16* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, suStream_t stream, + Method method = Method::Optimized); +suError_t quantize_int4(const bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, + suStream_t stream, Method method = Method::Optimized); + +// 融合路径先以最近偶数规则舍入到公开的 FP16/BF16 输出类型，再计算 INT4， +// 因而与 transform 后调用 quantize_int4 的字节和 scales 语义一致。 +suError_t transform_int4(const float16* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, + suStream_t stream, Method method = Method::Optimized); +suError_t transform_int4(const bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale, + suStream_t stream, Method method = Method::Optimized); + +} // namespace hadamard::biren diff --git a/03_hadamard_tc/a962695448-rgb/platforms/biren/hadamard_api.su b/03_hadamard_tc/a962695448-rgb/platforms/biren/hadamard_api.su new file mode 100644 index 00000000..6a4b0cdf --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/biren/hadamard_api.su @@ -0,0 +1,409 @@ +// 算法来源：本项目天数后端固定提交 a387db3332c6f9b01f128dd681848260c9691281。 +// 壁仞原生 SUPA 后端独立验证；未复制任何厂商 SDK 实现。 +#include "hadamard_api.h" + +#include +#include + +namespace hadamard::biren { +namespace { + +template __device__ float read_value(T value); +template<> __device__ float read_value(float16 value) { return static_cast(value); } +template<> __device__ float read_value(bfloat16 value) { + return static_cast(value); +} + +// 本项目的显式 BF16 RNE：先舍入成 BF16 可精确表示的 FP32，再使用原生存储类型。 +// SUPA 1.10 的原生 BF16 cast 会截断，因此所有 BF16 输出都必须经此路径。 +__device__ __forceinline__ bfloat16 biren_bf16_rne(float value) { + std::uint32_t bits; + __builtin_memcpy(&bits, &value, sizeof(bits)); + const std::uint32_t magnitude = bits & 0x7fffffffu; + if (magnitude > 0x7f800000u) { + bits = (bits & 0xffff0000u) | 0x00400000u; + } else if (magnitude < 0x7f800000u) { + bits = (bits + 0x7fffu + ((bits >> 16) & 1u)) & 0xffff0000u; + } + float exact_bf16_value; + __builtin_memcpy(&exact_bf16_value, &bits, sizeof(bits)); + return static_cast(exact_bf16_value); +} + +template __device__ T store_value(float value); +template<> __device__ float16 store_value(float value) { return static_cast(value); } +template<> __device__ bfloat16 store_value(float value) { + return biren_bf16_rne(value); +} + +__device__ std::uint8_t quantize_pair(const float* values, int pair, int n, + float row_scale) { + std::uint8_t byte = 0; + for (int k = 0; k < 2 && pair * 2 + k < n; ++k) { + const float x = values[pair * 2 + k] / row_scale; + // 明确实现最近偶数舍入，不依赖编译器默认取整模式。 + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + byte |= static_cast((q & 15) << (4 * k)); + } + return byte; +} + +// 沿用固定来源版本的基线算法：一元素一线程、每层两次屏障、线程 0 串行归约。 +// 与 optimized 独立保留蝶形计算，供同一运行环境下公平比较。 +template +__global__ void baseline_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float row_scale; + const int i = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + if (i < n) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + float next = 0; + if (i < n) { + const float a = values[i], b = values[i ^ stride]; + next = (i & stride) ? b - a : a + b; + } + __syncthreads(); + if (i < n) values[i] = next; + __syncthreads(); + } + if (i < n) values[i] = read_value(store_value(values[i] * scale)); + __syncthreads(); + } + if constexpr (!Quantize) { + if (i < n) output[offset + i] = store_value(values[i]); + } else { + if (i == 0) { + float magnitude = 0; + for (int j = 0; j < n; ++j) + magnitude = fmaxf(magnitude, fabsf(values[j])); + row_scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + if (i < bytes) packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + // 多行复用同一 block 时，防止下行加载覆盖仍在打包/写出的上一行。 + if (rows - row > gridDim.x) __syncthreads(); + } +} + +// 每个线程独占一个蝶形的两个输入/输出，因此单层没有线程间读写冲突， +// 只需在本层结束后同步。此算法不假定 warp 宽度，不使用 warp 级隐式同步。 +template +__global__ void optimized_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float maxima[128]; + __shared__ float row_scale; + const int tid = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + for (int i = tid; i < n; i += blockDim.x) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + if (tid < n / 2) { + const int low = ((tid & ~(stride - 1)) << 1) | (tid & (stride - 1)); + const float a = values[low], b = values[low + stride]; + values[low] = a + b; + values[low + stride] = a - b; + } + __syncthreads(); + } + } + if constexpr (!Quantize) { + for (int i = tid; i < n; i += blockDim.x) + output[offset + i] = store_value(values[i] * scale); + } else { + float magnitude = 0; + for (int i = tid; i < n; i += blockDim.x) { + float value = values[i]; + if constexpr (Transform) { + value = read_value(store_value(value * scale)); + values[i] = value; + } + magnitude = fmaxf(magnitude, fabsf(value)); + } + maxima[tid] = magnitude; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 1; stride /= 2) { + if (tid < stride) maxima[tid] = fmaxf(maxima[tid], maxima[tid + stride]); + __syncthreads(); + } + if (tid == 0) { + // 最后两个值只由线程 0 消费，与 scale 写入合并，少一次屏障。 + const float maximum = fmaxf(maxima[0], maxima[1]); + row_scale = maximum == 0 ? 1.0f : maximum / 7.0f; + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + for (int i = tid; i < bytes; i += blockDim.x) + packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + if (rows - row > gridDim.x) __syncthreads(); + } +} + +#if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + +// 仅在壁仞 SUPA 构建中启用；width=32 已由壁仞实机独立探针验证。 +// SDK 的 mask 参数标为无效，因此必须保证完整 warp 参与，不能依赖 mask 排除 lane。 +__device__ int warp32_quantized_nibble(float value, float scale) { + const float x = value / scale; + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + return q & 15; +} + +template +__global__ void warp32_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float scale) { + static_assert(RowsPerBlock == 1 || RowsPerBlock == 4, "supported row counts per block are 1 and 4"); + constexpr int width = 32; + constexpr int rows_per_block = RowsPerBlock; + constexpr int registers = N > width ? N / width : 1; + const int lane = threadIdx.x % width; + const int warp = threadIdx.x / width; + const std::size_t first_row = static_cast(blockIdx.x) * rows_per_block + warp; + const std::size_t row_stride = static_cast(gridDim.x) * rows_per_block; + // 同一 warp 的 32 个 lane 具有相同 row 与循环次数。尾行只跳过完整 warp， + // 所有参与行均有完整的 32 个 lane 执行每次 shuffle。 + for (std::size_t row = first_row; row < rows; row += row_stride) { + const std::size_t offset = row * N; + float values[registers]; + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + values[r] = i < N ? read_value(input[offset + i]) : 0.0f; + } + if constexpr (Transform) { + #pragma unroll + for (int stride = 1; stride < (N < width ? N : width); stride *= 2) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const float current = values[r]; + const float peer = __shfl_xor_sync(0xffffffffu, current, stride, width); + values[r] = (lane & stride) ? peer - current : current + peer; + } + } + // stride=32/64/128 对应同一 lane 内寄存器编号的第 0/1/2 位。 + // 顺序严格延续前五层 shuffle，不改变浮点加减的关联顺序。 + #pragma unroll + for (int distance = 1; distance < registers; distance *= 2) { + #pragma unroll + for (int base = 0; base < registers; base += distance * 2) { + #pragma unroll + for (int i = 0; i < distance; ++i) { + const float a = values[base + i]; + const float b = values[base + i + distance]; + values[base + i] = a + b; + values[base + i + distance] = a - b; + } + } + } + } + if constexpr (!Quantize) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + if (i < N) output[offset + i] = store_value(values[r] * scale); + } + } else { + float magnitude = 0.0f; + #pragma unroll + for (int r = 0; r < registers; ++r) { + if constexpr (Transform) values[r] = read_value(store_value(values[r] * scale)); + magnitude = fmaxf(magnitude, fabsf(values[r])); + } + #pragma unroll + for (int stride = width / 2; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor_sync(0xffffffffu, magnitude, stride, width)); + const float row_scale = magnitude == 0.0f ? 1.0f : magnitude / 7.0f; + if (lane == 0) scales[row] = row_scale; + #pragma unroll + for (int r = 0; r < registers; ++r) { + #if defined(HADAMARD_BIREN_BALANCED_PACK) + // 消融候选：每个 lane 仅量化自己的元素，再交换整数 nibble。 + // 完整 warp 在条件写入之前参与 shuffle，不依赖 SDK 的 mask 参数筛选。 + const int nibble = warp32_quantized_nibble(values[r], row_scale); + const int peer = __shfl_xor_sync(0xffffffffu, nibble, 1, width); + const int i = lane + r * width; + if ((lane & 1) == 0 && i < N) { + const int high = i + 1 < N ? peer : 0; + packed[row * ((N + 1) / 2) + i / 2] + = static_cast(nibble | (high << 4)); + } + #else + // shuffle 在分支之前：奇数 lane 同样必须提供其相邻元素。 + const float peer = __shfl_xor_sync(0xffffffffu, values[r], 1, width); + const int i = lane + r * width; + if ((lane & 1) == 0 && i < N) { + const int low = warp32_quantized_nibble(values[r], row_scale); + const int high = i + 1 < N ? warp32_quantized_nibble(peer, row_scale) : 0; + packed[row * ((N + 1) / 2) + i / 2] + = static_cast(low | (high << 4)); + } + #endif + } + } + } +} + +template +suError_t enqueue_warp32_shape(unsigned int blocks, int rows_per_block, + suStream_t stream, void** arguments) { + #if defined(HADAMARD_BIREN_SMALL_BATCH) + if (rows_per_block == 1) { + return suLaunchKernelSingleDevice( + reinterpret_cast(warp32_kernel), + dim3(blocks), dim3(32), 0, stream, arguments); + } + #else + (void)rows_per_block; + #endif + return suLaunchKernelSingleDevice( + reinterpret_cast(warp32_kernel), + dim3(blocks), dim3(128), 0, stream, arguments); +} + +template +suError_t launch_warp32(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, suStream_t stream) { + // 待验证的小批量候选：当前 106M 报告 64 个 SM，增加可调度 block 数。 + // 阈值按设备规模提出，不针对特定测量行数；仍需在 32/63/64/65 等边界外测。 + #if defined(HADAMARD_BIREN_SMALL_BATCH) + const int rows_per_block = rows <= 64 ? 1 : 4; + #else + const int rows_per_block = 4; + #endif + // rows <= SIZE_MAX/4 已由 launch 校验；加数最多 3，不会溢出。 + const std::size_t requested = (rows + rows_per_block - 1) / rows_per_block; + const unsigned int blocks = static_cast(requested < 65535 ? requested : 65535); + void* arguments[] = {&input, &output, &packed, &scales, &rows, &scale}; + suError_t status = suSuccess; + #define LAUNCH_WARP32(N) case N: \ + status = enqueue_warp32_shape( \ + blocks, rows_per_block, stream, arguments); \ + break + switch (n) { + LAUNCH_WARP32(1); + LAUNCH_WARP32(2); + LAUNCH_WARP32(4); + LAUNCH_WARP32(8); + LAUNCH_WARP32(16); + LAUNCH_WARP32(32); + LAUNCH_WARP32(64); + LAUNCH_WARP32(128); + LAUNCH_WARP32(256); + default: return suErrorInvalidValue; + } + #undef LAUNCH_WARP32 + return status == suSuccess ? suGetLastError() : status; +} + +#endif // HADAMARD_BIREN_WARP32 + +bool valid_range(const void* pointer, std::size_t bytes, std::size_t alignment) { + const auto address = reinterpret_cast(pointer); + return pointer != nullptr && address % alignment == 0 + && bytes <= std::numeric_limits::max() - address; +} + +bool overlaps(const void* left, std::size_t left_size, + const void* right, std::size_t right_size) { + // valid_range 已确保两个区间端点的加法不会溢出。 + const auto a = reinterpret_cast(left); + const auto b = reinterpret_cast(right); + return a < b + right_size && b < a + left_size; +} + +template +suError_t launch(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, suStream_t stream, Method method) { + if (n < 1 || n > 256 || (n & (n - 1)) != 0 + || !std::isfinite(scale) || scale <= 0 + || (method != Method::Baseline && method != Method::Optimized && method != Method::Warp32)) + return suErrorInvalidValue; + if (rows == 0) return suSuccess; + + constexpr auto maximum = std::numeric_limits::max(); + const std::size_t row_bytes = static_cast(n) * sizeof(T); + if (rows > maximum / row_bytes || rows > maximum / sizeof(float)) + return suErrorInvalidValue; + const std::size_t input_bytes = rows * row_bytes; + if (!valid_range(input, input_bytes, alignof(T))) return suErrorInvalidValue; + if constexpr (Quantize) { + const std::size_t packed_bytes = rows * static_cast((n + 1) / 2); + const std::size_t scale_bytes = rows * sizeof(float); + if (!valid_range(packed, packed_bytes, alignof(std::uint8_t)) + || !valid_range(scales, scale_bytes, alignof(float)) + || overlaps(input, input_bytes, packed, packed_bytes) + || overlaps(input, input_bytes, scales, scale_bytes) + || overlaps(packed, packed_bytes, scales, scale_bytes)) return suErrorInvalidValue; + } else { + if (!valid_range(output, input_bytes, alignof(T)) + || (input != output && overlaps(input, input_bytes, output, input_bytes))) + return suErrorInvalidValue; + } + + if (method == Method::Warp32) { + #if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + return launch_warp32(input, output, packed, scales, rows, n, scale, stream); + #else + return suErrorNotSupported; + #endif + } + + // 保守上限兼容不同设备的 grid.x 限制，超过上限由 block 顺序处理多行。 + const unsigned int blocks = static_cast(rows < 65535 ? rows : 65535); + void* arguments[] = {&input, &output, &packed, &scales, &rows, &n, &scale}; + suError_t status = suSuccess; + if (method == Method::Baseline) { + const int threads = n < 64 ? 64 : n; + status = suLaunchKernelSingleDevice( + reinterpret_cast(baseline_kernel), + dim3(blocks), dim3(threads), 0, stream, arguments); + } else { + const int threads = n / 2 < 64 ? 64 : n / 2; + status = suLaunchKernelSingleDevice( + reinterpret_cast(optimized_kernel), + dim3(blocks), dim3(threads), 0, stream, arguments); + } + return status == suSuccess ? suGetLastError() : status; +} + +} // namespace + +#define DEFINE_TYPED_API(T) \ +suError_t transform(const T* input, T* output, std::size_t rows, int n, float scale, \ + suStream_t stream, Method method) { \ + return launch(input, output, nullptr, nullptr, rows, n, scale, stream, method); \ +} \ +suError_t quantize_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, suStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, 1.0f, stream, method); \ +} \ +suError_t transform_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, float scale, suStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, scale, stream, method); \ +} + +DEFINE_TYPED_API(float16) +DEFINE_TYPED_API(bfloat16) +#undef DEFINE_TYPED_API + +} // namespace hadamard::biren diff --git a/03_hadamard_tc/a962695448-rgb/platforms/biren/run_platform.py b/03_hadamard_tc/a962695448-rgb/platforms/biren/run_platform.py new file mode 100644 index 00000000..386e6e59 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/biren/run_platform.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""在真实 SUPA/壁仞设备构建、核查并测量；不安装或替换驱动与框架。""" +import argparse +import csv +import hashlib +import json +import os +from pathlib import Path +import statistics +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[2] +PLATFORM = Path(__file__).resolve().parent +SOURCE_COMMIT = "1681a85ec7b832e56e672f0589a472cc8f91af95" + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def capture(command, env=None): + result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False, env=env) + return {"command": command, "returncode": result.returncode, "output": result.stdout} + + +def run(command, log, stages, expected=0, env=None): + print("RUN", " ".join(map(str, command)), flush=True) + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + completed = subprocess.run(list(map(str, command)), cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, env=env) + stages.append({"command": list(map(str, command)), "log": log.name, + "returncode": completed.returncode, "expected_returncode": expected, + "wall_seconds": time.monotonic() - started, "sha256": sha256(log)}) + print("EXIT", completed.returncode, "LOG", log, flush=True) + if completed.returncode != expected: + print(log.read_text(encoding="utf-8", errors="replace")[-16000:], file=sys.stderr) + raise RuntimeError("stage failed: " + log.name) + + +def summarize_benchmark(path): + samples = {} + with path.open(newline="", encoding="utf-8") as stream: + for row in csv.DictReader(stream): + key = tuple(row[k] for k in ("dtype", "batch", "seq", "heads", "dim", "method")) + samples.setdefault(key, []).append(float(row["kernel_us"])) + medians = {} + summary = [] + for key, values in samples.items(): + medians[key] = statistics.median(values) + summary.append(dict(zip(("dtype", "batch", "seq", "heads", "dim", "method"), key), + samples=values, median_us=statistics.median(values), + minimum_us=min(values), maximum_us=max(values))) + comparisons = [] + for key, value in medians.items(): + if not key[-1].startswith("baseline_"): + continue + peer = key[:-1] + (key[-1].replace("baseline_", "optimized_"),) + comparisons.append({"shape_dtype": key[:-1], "operation": key[-1][9:], + "baseline_median_us": value, "optimized_median_us": medians[peer], + "baseline_over_optimized": value / medians[peer]}) + warp_peer = key[:-1] + (key[-1].replace("baseline_", "warp32_"),) + if warp_peer in medians: + comparisons.append({"shape_dtype": key[:-1], "operation": key[-1][9:], "candidate": "warp32", + "baseline_median_us": value, "optimized_median_us": medians[peer], + "warp32_median_us": medians[warp_peer], "baseline_over_warp32": value / medians[warp_peer], + "optimized_over_warp32": medians[peer] / medians[warp_peer]}) + return {"metric": "native SUPA events; warmup excluded; no allocation/copy in interval", + "working_set": "same seeded read-only input reused; warm-cache timing", + "logical_GBs_note": "logical tensor I/O estimate, not measured physical memory bandwidth", + "statistics": summary, "comparisons_including_slowdowns": comparisons} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sdk-root", type=Path, default=Path("/usr/local/birensupa/sdk/latest")) + parser.add_argument("--compiler", type=Path) + parser.add_argument("--output", type=Path, help="必须是新的结果目录，防止覆盖既有证据") + parser.add_argument("--quick", action="store_true", help="仅运行快速调试矩阵，不代表完整验收") + parser.add_argument("--no-benchmark", action="store_true") + parser.add_argument("--warp32", action="store_true", help="显式启用真实 warp32 壁仞设备的 SUPA 路径") + parser.add_argument("--balanced-pack", action="store_true", help="Warp32 INT4 打包消融候选；须同时使用 --warp32，变换路径不变") + parser.add_argument("--small-batch-warp", action="store_true", help="小批量 Warp32 发射消融候选；须同时使用 --warp32，仅影响 rows<=64") + parser.add_argument("--repeats", type=int, default=100) + parser.add_argument("--groups", type=int, default=5) + args = parser.parse_args() + if args.balanced_pack and not args.warp32: + parser.error("--balanced-pack requires --warp32") + if args.small_batch_warp and not args.warp32: + parser.error("--small-batch-warp requires --warp32") + if args.repeats < 1 or args.repeats > 10000 or args.groups < 1 or args.groups > 10000: + parser.error("repeats/groups must be between 1 and 10000") + sdk = args.sdk_root.resolve() + supa = sdk / "supa" + brcc = sdk / "brcc" + compiler = (args.compiler or brcc / "bin/brcc").absolute() + if not compiler.is_file(): + parser.error("SUPA brcc compiler is missing: " + str(compiler)) + # 仅配置本次子进程环境；不改系统环境、驱动或用户现有开发环境。 + runtime_env = dict(os.environ, SUPA_PATH=str(supa), BIREN_HOME=str(sdk)) + runtime_env["PATH"] = os.pathsep.join([str(brcc / "bin"), str(supa / "bin"), os.environ.get("PATH", "")]) + library_paths = [str(p) for p in (supa / "lib", brcc / "lib") if p.is_dir()] + inherited_libraries = os.environ.get("LD_LIBRARY_PATH", "") + runtime_env["LD_LIBRARY_PATH"] = os.pathsep.join(library_paths + ([inherited_libraries] if inherited_libraries else [])) + destination = (args.output or ROOT / "results/biren" / time.strftime("%Y%m%d-%H%M%S", time.gmtime())).resolve() + if destination.exists(): + parser.error("output directory already exists; select a fresh path") + destination.mkdir(parents=True) + binary = destination / "validate_and_benchmark" + sources = [PLATFORM / "hadamard_api.h", PLATFORM / "hadamard_api.su", + PLATFORM / "validate_and_benchmark.su", PLATFORM / "run_platform.py", ROOT / "include/reference.hpp"] + report = {"status": "RUNNING", "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "platform": "biren", "adapted_from": {"platform": "metax", "commit": SOURCE_COMMIT}, + "quick": args.quick, "warp32_enabled": args.warp32, + "balanced_pack_enabled": args.balanced_pack, + "balanced_pack_scope": ["warp32_split", "warp32_fused"] if args.balanced_pack else [], + "small_batch_warp_enabled": args.small_batch_warp, + "small_batch_warp_scope": {"methods": ["warp32_transform", "warp32_split", "warp32_fused"], + "max_rows": 64, "rows_per_block": 1, "threads_per_block": 32} + if args.small_batch_warp else {}, + "sdk_environment": {key: runtime_env[key] for key in ("SUPA_PATH", "BIREN_HOME")}, + "source_sha256": {str(p.relative_to(ROOT)): sha256(p) for p in sources}, + "git_head": capture(["git", "rev-parse", "HEAD"]), + "git_status": capture(["git", "status", "--short"]), + "compiler": capture([str(compiler), "--version"], env=runtime_env), + "python_version": sys.version, "stages": []} + manifest = destination / "run_summary.json" + manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + try: + command = [compiler, "-x", "supa", "-std=c++17", "-O2", "-I" + str(ROOT / "include"), + "-I" + str(supa / "include")] + if args.warp32: + command.append("-DHADAMARD_BIREN_WARP32") + if args.balanced_pack: + command.append("-DHADAMARD_BIREN_BALANCED_PACK") + if args.small_batch_warp: + command.append("-DHADAMARD_BIREN_SMALL_BATCH") + command += [PLATFORM / "hadamard_api.su", PLATFORM / "validate_and_benchmark.su", + "-L" + str(supa / "lib"), "-lsupa-runtime", "-Wl,-rpath," + str(supa / "lib"), "-o", binary] + run(command, destination / "build.log", report["stages"], env=runtime_env) + report["binary_sha256"] = sha256(binary) + # CLI 验证在创建设备上下文之前执行。错误输入必须明确返回 2。 + invalid = [[], ["--validate", "--dim", "0"], ["--validate", "--dim", "3"], + ["--validate", "--dim", "512"], ["--validate", "--dtype", "fp32"], + ["--validate", "--batch", "-1"], ["--validate", "--seq", "0"], + ["--validate", "--heads", "x"], ["--validate", "--batch", "18446744073709551615", "--seq", "2"], + ["--validate", "--heads", "999999999999999999999999999999"], + ["--validate", "--dim"], ["--validate", "--unknown", "1"], + ["--benchmark", "--repeats", "0"], ["--benchmark", "--groups", "10001"]] + for i, case in enumerate(invalid): + run([binary] + case, destination / ("invalid_%02d.log" % i), report["stages"], expected=2, env=runtime_env) + report["cli_rejection_cases"] = len(invalid) + command = [binary, "--validate", "--json", destination / "validation.json"] + if args.quick: + command.append("--quick") + run(command, destination / "validation.log", report["stages"], env=runtime_env) + report["validation"] = json.loads((destination / "validation.json").read_text(encoding="utf-8")) + if report["validation"]["status"] != "PASS" or (not args.quick and not report["validation"]["full_matrix"]): + raise RuntimeError("validation JSON does not confirm requested matrix") + if report["validation"]["warp32_enabled"] != args.warp32: + raise RuntimeError("validation JSON does not match requested Warp32 build") + if report["validation"]["balanced_pack_enabled"] != args.balanced_pack: + raise RuntimeError("validation JSON does not match requested balanced-pack build") + if report["validation"]["small_batch_warp_enabled"] != args.small_batch_warp: + raise RuntimeError("validation JSON does not match requested small-batch-warp build") + if not args.no_benchmark: + command = [binary, "--benchmark", "--csv", destination / "benchmark.csv", + "--groups", str(args.groups), "--repeats", str(args.repeats)] + if args.quick: + command += ["--batch", "1", "--seq", "17", "--heads", "1", "--dim", "128"] + run(command, destination / "benchmark.log", report["stages"], env=runtime_env) + report["benchmark"] = summarize_benchmark(destination / "benchmark.csv") + report["status"] = "PASS" + except (OSError, RuntimeError, ValueError, KeyError) as error: + report["status"] = "FAIL" + report["error"] = str(error) + print("FAIL", error, file=sys.stderr) + finally: + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + report["artifacts"] = {p.name: sha256(p) for p in destination.iterdir() if p.is_file() and p != manifest} + manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "quick": args.quick, "summary": str(manifest)}, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/platforms/biren/validate_and_benchmark.su b/03_hadamard_tc/a962695448-rgb/platforms/biren/validate_and_benchmark.su new file mode 100644 index 00000000..e4a976b8 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/biren/validate_and_benchmark.su @@ -0,0 +1,573 @@ +// 验证矩阵来源：1681a85ec7b832e56e672f0589a472cc8f91af95 的已验收沐曦 runner。 +// 壁仞必须独立使用 SUPA 编译运行；不复用其他平台的通过结论或性能数据。 +#include "hadamard_api.h" +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace api = hadamard::biren; +#define CHECK(call) do { suError_t e_ = (call); if (e_ != suSuccess) \ + throw std::runtime_error(std::string(#call) + ": " + suGetErrorString(e_)); } while (0) + +template float read(T); +template<> float read(float16 x) { return static_cast(x); } +template<> float read(bfloat16 x) { return static_cast(x); } + +// SUPA 1.10 的主机 bfloat16(float) 构造器会截断，不能作为本项目的 RNE oracle。 +// 此主机实现按高/低 16 位和奇偶位显式比较；不调用 API 中的 GPU store helper。 +bfloat16 host_bf16_rne(float value) { + static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32-bit representation"); + static_assert(sizeof(bfloat16) == sizeof(std::uint16_t), "bfloat16 storage must be 16 bits"); + std::uint32_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + std::uint16_t high = static_cast(bits >> 16); + const std::uint32_t low = bits & 0xffffu; + if ((bits & 0x7f800000u) == 0x7f800000u) { + if (bits & 0x007fffffu) high = static_cast(high | 0x0040u); + } else if (low > 0x8000u || (low == 0x8000u && (high & 1u))) { + high = static_cast(high + 1u); + } + bfloat16 result{}; + std::memcpy(&result, &high, sizeof(high)); + return result; +} + +std::size_t verify_host_bf16_rne() { + struct Case { std::uint32_t input; std::uint16_t expected; }; + // 独立写出的格式边界：正负零、中点上下、奇偶中点与最小次正规区间。 + const std::array cases{{ + {0x00000000u, 0x0000u}, {0x80000000u, 0x8000u}, + {0x3f807fffu, 0x3f80u}, {0x3f808000u, 0x3f80u}, + {0x3f808001u, 0x3f81u}, {0x3f818000u, 0x3f82u}, + {0xbf807fffu, 0xbf80u}, {0xbf808000u, 0xbf80u}, + {0xbf808001u, 0xbf81u}, {0xbf818000u, 0xbf82u}, + {0x00008000u, 0x0000u}, {0x00018000u, 0x0002u}}}; + for (const auto test : cases) { + float value = 0; + std::memcpy(&value, &test.input, sizeof(value)); + const auto converted = host_bf16_rne(value); + std::uint16_t actual = 0; + std::memcpy(&actual, &converted, sizeof(actual)); + if (actual != test.expected) throw std::runtime_error("independent host BF16 RNE boundary check failed"); + } + return cases.size(); +} +template T rounded(float); +template<> float16 rounded(float x) { return float16(x); } +template<> bfloat16 rounded(float x) { return host_bf16_rne(x); } + +std::vector active_methods() { + std::vector methods{api::Method::Baseline, api::Method::Optimized}; +#if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + methods.push_back(api::Method::Warp32); +#endif + return methods; +} + +const char* method_name(api::Method method) { + if (method == api::Method::Baseline) return "baseline"; + if (method == api::Method::Optimized) return "optimized"; + return "warp32"; +} + +// 每个区间都有前后哨兵。17 个元素的偏移同时覆盖仅 2 字节对齐的 FP16/BF16 指针。 +template class Guarded { + static constexpr std::size_t guard = 17; + T* raw_ = nullptr; + std::size_t count_; + std::vector initial_; + std::string name_; +public: + explicit Guarded(std::size_t count, suStream_t stream, const char* name = "unnamed") + : count_(count), initial_((count + 2 * guard) * sizeof(T), 0xa5), name_(name) { + CHECK(suMallocDevice(reinterpret_cast(&raw_), initial_.size())); + // 初始化、上传、kernel、回读使用同一个非阻塞 stream，不依赖跨 stream 排序。 + // 按字节 memset 也避免异步初始化读取随后被 upload 更新的主机 shadow。 + CHECK(suMemsetAsync(raw_, 0xa5, initial_.size(), stream)); + } + ~Guarded() { if (raw_) suFree(raw_); } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + T* data() { return raw_ + guard; } + void upload(const std::vector& values, suStream_t stream) { + if (values.size() != count_) throw std::runtime_error("upload size mismatch"); + std::memcpy(initial_.data() + guard * sizeof(T), values.data(), count_ * sizeof(T)); + CHECK(suMemcpyAsync(raw_, initial_.data(), initial_.size(), stream, suMemcpyHostToDevice)); + } + std::vector download(suStream_t stream, bool unchanged = false, const char* phase = "readback") { + std::vector bytes(initial_.size()); + CHECK(suMemcpyAsync(bytes.data(), raw_, bytes.size(), stream, suMemcpyDeviceToHost)); + CHECK(suStreamSynchronize(stream)); + const std::size_t prefix = guard * sizeof(T), end = prefix + count_ * sizeof(T); + for (std::size_t i = 0; i < bytes.size(); ++i) { + if (bytes[i] == initial_[i] || (!unchanged && i >= prefix && i < end)) continue; + const bool is_guard = i < prefix || i >= end; + throw std::runtime_error(std::string(is_guard ? "device buffer guard overwritten" : "read-only input modified") + + " buffer=" + name_ + " phase=" + phase + " region=" + (i < prefix ? "prefix" : (i >= end ? "suffix" : "payload")) + + " byte_from_payload=" + std::to_string(static_cast(i) - static_cast(prefix)) + + " expected=" + std::to_string(static_cast(initial_[i])) + + " actual=" + std::to_string(static_cast(bytes[i])) + + " elements=" + std::to_string(count_) + " element_bytes=" + std::to_string(sizeof(T))); + } + std::vector result(count_); + std::memcpy(result.data(), bytes.data() + prefix, count_ * sizeof(T)); + return result; + } +}; + +struct Options { + bool validate = false, benchmark = false, custom_shape = false, quick = false; + std::size_t batch = 1, seq = 1, heads = 1; + int dim = 128, repeats = 100, groups = 5; + std::string dtype = "both", csv = "biren_benchmark.csv", json = "biren_validation.json"; +}; + +std::size_t positive(const std::string& text, const char* name) { + if (text.empty() || text.find_first_not_of("0123456789") != std::string::npos) + throw std::invalid_argument(std::string(name) + " must be a positive integer"); + std::size_t used = 0; + const auto value = std::stoull(text, &used); + if (!value || value > std::numeric_limits::max() || used != text.size()) + throw std::invalid_argument(std::string(name) + " is outside its supported range"); + return static_cast(value); +} + +std::size_t multiply(std::size_t a, std::size_t b) { + if (b && a > std::numeric_limits::max() / b) + throw std::invalid_argument("shape product overflows size_t"); + return a * b; +} + +std::size_t checked_shape(std::size_t b, std::size_t s, std::size_t h, int n) { + if (n < 1 || n > 256 || !hadamard::power_of_two(n)) + throw std::invalid_argument("dim must be a power of two in [1,256]"); + const auto rows = multiply(multiply(b, s), h); + if (!rows || rows > static_cast(std::numeric_limits::max())) + throw std::invalid_argument("rows outside supported grid range"); + multiply(multiply(rows, static_cast(n)), sizeof(float16)); + return rows; +} + +Options parse(int argc, char** argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string key = argv[i]; + if (key == "--validate") o.validate = true; + else if (key == "--benchmark") o.benchmark = true; + else if (key == "--quick") o.quick = true; + else { + if (i + 1 == argc) throw std::invalid_argument("missing value for " + key); + const std::string value = argv[++i]; + if (key == "--dtype") o.dtype = value; + else if (key == "--csv") o.csv = value; + else if (key == "--json") o.json = value; + else if (key == "--batch" || key == "--seq" || key == "--heads" || key == "--dim") { + const auto v = positive(value, key.c_str()); + o.custom_shape = true; + if (key == "--batch") o.batch = v; + else if (key == "--seq") o.seq = v; + else if (key == "--heads") o.heads = v; + else { + if (v > 256) throw std::invalid_argument("dim is greater than 256"); + o.dim = static_cast(v); + } + } else if (key == "--repeats" || key == "--groups") { + const auto v = positive(value, key.c_str()); + if (v > 10000) throw std::invalid_argument("repeats/groups exceed 10000"); + if (key == "--repeats") o.repeats = static_cast(v); + else o.groups = static_cast(v); + } else throw std::invalid_argument("unknown argument " + key); + } + } + if (!o.validate && !o.benchmark) throw std::invalid_argument("specify --validate and/or --benchmark"); + if (o.dtype != "both" && o.dtype != "fp16" && o.dtype != "bf16") + throw std::invalid_argument("dtype must be fp16, bf16 or both"); + checked_shape(o.batch, o.seq, o.heads, o.dim); + return o; +} + +template std::vector make_input(std::size_t rows, int n, int pattern, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::normal_distribution normal(0.0f, 0.5f); + std::vector result(rows * n); + for (std::size_t i = 0; i < result.size(); ++i) { + float x = 0; + if (pattern == 0) x = uniform(rng); + else if (pattern == 1) x = normal(rng); + else if (pattern == 2) x = i % n == (i / n + seed) % n ? 8.0f : uniform(rng); + else if (pattern == 4) x = i % n == (i / n) % n ? 1.0f : 0.0f; + result[i] = rounded(x); + } + return result; +} + +struct Summary { + std::size_t cases = 0, elements = 0, exact_transform_elements = 0, contract_checks = 0; + std::size_t warp32_cases = 0, exact_warp32_elements = 0; + std::size_t warp32_grid_stride_cases = 0, warp32_grid_stride_elements = 0, unsupported_warp32_checks = 0; + double max_rounded_error = 0, max_unrounded_error = 0; +}; + +template void one_case(suStream_t stream, std::size_t rows, int n, float scale, + int pattern, unsigned seed, const char* dtype, Summary& summary) { + const auto input = make_input(rows, n, pattern, seed); + const std::size_t size = input.size(), bytes = rows * ((n + 1) / 2); + Guarded x(size, stream), baseline(size, stream), optimized(size, stream), inplace(size, stream); + Guarded base_split(bytes, stream), opt_split(bytes, stream), base_fused(bytes, stream), opt_fused(bytes, stream); + Guarded bs(rows, stream), os(rows, stream), bfs(rows, stream), ofs(rows, stream); + x.upload(input, stream); + inplace.upload(input, stream); + CHECK(api::transform(x.data(), baseline.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform(x.data(), optimized.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::transform(inplace.data(), inplace.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::quantize_int4(baseline.data(), base_split.data(), bs.data(), rows, n, stream, api::Method::Baseline)); + CHECK(api::quantize_int4(optimized.data(), opt_split.data(), os.data(), rows, n, stream, api::Method::Optimized)); + CHECK(api::transform_int4(x.data(), base_fused.data(), bfs.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform_int4(x.data(), opt_fused.data(), ofs.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(suStreamSynchronize(stream)); + x.download(stream, true); + const auto a = baseline.download(stream), b = optimized.download(stream), ip = inplace.download(stream); + const std::string context = std::string(dtype) + " rows=" + std::to_string(rows) + " n=" + std::to_string(n) + + " pattern=" + std::to_string(pattern) + " seed=" + std::to_string(seed) + " scale=" + std::to_string(scale); + if (std::memcmp(a.data(), b.data(), size * sizeof(T)) || std::memcmp(b.data(), ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/optimized/in-place transform not bitwise identical: " + context); + std::vector fx(size), actual(size); + for (std::size_t i = 0; i < size; ++i) { fx[i] = read(input[i]); actual[i] = read(b[i]); } + // 全部元素使用 O(N^2) FP64 稠密矩阵公式，不调用任何 FWHT 作为正确性 oracle。 + const auto dense = hadamard::dense_reference(fx, n, static_cast(scale)); + const double tolerance = std::string(dtype) == "fp16" ? 1e-2 : 5e-2; + for (std::size_t i = 0; i < size; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + const double error = std::abs(static_cast(actual[i]) - expected); + if (!std::isfinite(actual[i]) || !(error < tolerance)) + throw std::runtime_error("rounded FP64 dense oracle mismatch: " + context + " index=" + std::to_string(i) + + " error=" + std::to_string(error)); + summary.max_rounded_error = std::max(summary.max_rounded_error, error); + summary.max_unrounded_error = std::max(summary.max_unrounded_error, std::abs(actual[i] - dense[i])); + } + const auto expected_q = hadamard::quantize_int4(actual, n); + if (base_split.download(stream) != expected_q.packed || opt_split.download(stream) != expected_q.packed || + base_fused.download(stream) != expected_q.packed || opt_fused.download(stream) != expected_q.packed || + bs.download(stream) != expected_q.scales || os.download(stream) != expected_q.scales || + bfs.download(stream) != expected_q.scales || ofs.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/optimized split/fused INT4 bytes or scales mismatch: " + context); +#if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + Guarded warp_output(size, stream, "warp32.output"), warp_inplace(size, stream, "warp32.inplace"); + Guarded warp_split(bytes, stream, "warp32.split"), warp_fused(bytes, stream, "warp32.fused"); + Guarded warp_split_scales(rows, stream, "warp32.split-scales"), warp_fused_scales(rows, stream, "warp32.fused-scales"); + warp_inplace.upload(input, stream); + CHECK(api::transform(x.data(), warp_output.data(), rows, n, scale, stream, api::Method::Warp32)); + CHECK(api::transform(warp_inplace.data(), warp_inplace.data(), rows, n, scale, stream, api::Method::Warp32)); + CHECK(api::quantize_int4(warp_output.data(), warp_split.data(), warp_split_scales.data(), rows, n, stream, api::Method::Warp32)); + CHECK(api::transform_int4(x.data(), warp_fused.data(), warp_fused_scales.data(), rows, n, scale, stream, api::Method::Warp32)); + const auto warp = warp_output.download(stream), warp_ip = warp_inplace.download(stream); + if (std::memcmp(a.data(), warp.data(), size * sizeof(T)) || std::memcmp(a.data(), warp_ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/Warp32/in-place transform not bitwise identical: " + context); + if (warp_split.download(stream) != expected_q.packed || warp_fused.download(stream) != expected_q.packed || + warp_split_scales.download(stream) != expected_q.scales || warp_fused_scales.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/Warp32 split/fused INT4 bytes or scales mismatch: " + context); + x.download(stream, true, "after-warp32"); + ++summary.warp32_cases; + summary.exact_warp32_elements += size; +#endif + ++summary.cases; + summary.elements += size; + summary.exact_transform_elements += size; +} + +template void contract_tests(suStream_t stream, Summary& summary) { + Guarded input(64, stream, "contract.input"), output(64, stream, "contract.output"); + Guarded packed(32, stream, "contract.packed"); + Guarded scales(8, stream, "contract.scales"); + input.download(stream, true, "initialized"); + output.download(stream, true, "initialized"); + packed.download(stream, true, "initialized"); + scales.download(stream, true, "initialized"); + auto reject = [&](suError_t status) { + if (status != suErrorInvalidValue) throw std::runtime_error("invalid API input did not return suErrorInvalidValue"); + ++summary.contract_checks; + }; + auto success = [&](suError_t status) { CHECK(status); ++summary.contract_checks; }; + for (const auto method : active_methods()) { + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=invalid-parameters" << std::endl; + for (int n : {0, 3, 512}) { + reject(api::transform(input.data(), output.data(), 1, n, 1, stream, method)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, n, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, n, 1, stream, method)); + } + for (float scale : {0.0f, -1.0f, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { + reject(api::transform(input.data(), output.data(), 1, 8, scale, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, scale, stream, method)); + } + reject(api::transform(static_cast(nullptr), output.data(), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), static_cast(nullptr), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), input.data() + 1, 1, 8, 1, stream, method)); + reject(api::transform(input.data(), output.data(), std::numeric_limits::max(), 256, 1, stream, method)); + reject(api::quantize_int4(input.data(), reinterpret_cast(input.data()), scales.data(), 1, 8, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), nullptr, 1, 8, 1, stream, method)); + auto* odd = reinterpret_cast(reinterpret_cast(input.data()) + 1); + reject(api::transform(odd, output.data(), 1, 8, 1, stream, method)); + auto* bad_scale = reinterpret_cast(reinterpret_cast(scales.data()) + 2); + reject(api::quantize_int4(input.data(), packed.data(), bad_scale, 1, 8, stream, method)); + success(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, method)); + success(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, method)); + success(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, method)); + input.download(stream, true, "after-invalid-and-zero-rows"); + output.download(stream, true, "after-invalid-and-zero-rows"); + packed.download(stream, false, "after-invalid-and-zero-rows"); + scales.download(stream, false, "after-invalid-and-zero-rows"); + // 正负半整数：预期手写，避免舍入测试仅复用 CPU 参考实现。 + const std::vector ties{7, -7, .5f, 1.5f, 2.5f, -.5f, -1.5f, -2.5f}; + std::vector t(64, rounded(0)); + for (std::size_t i = 0; i < ties.size(); ++i) t[i] = rounded(ties[i]); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-upload" << std::endl; + input.upload(t, stream); + input.download(stream, true, "after-ties-upload"); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-quantize" << std::endl; + CHECK(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, method)); + const auto q = packed.download(stream, false, "after-ties-quantize"); + const auto s = scales.download(stream, false, "after-ties-quantize"); + const std::array expected{{0x97, 0x20, 0x02, 0xee}}; + if (!std::equal(expected.begin(), expected.end(), q.begin()) || s[0] != 1.0f) + throw std::runtime_error("positive/negative ties-to-even test failed"); + input.download(stream, true, "after-ties-quantize"); + ++summary.contract_checks; + } + const auto invalid = static_cast(-1); + reject(api::transform(input.data(), output.data(), 1, 8, 1, stream, invalid)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, invalid)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, invalid)); +#if !defined(HADAMARD_BIREN_WARP32) || !defined(__SUPACC__) + auto unsupported = [&](suError_t status) { + if (status != suErrorNotSupported) throw std::runtime_error("uncompiled Warp32 path did not return suErrorNotSupported"); + ++summary.unsupported_warp32_checks; + }; + unsupported(api::transform(input.data(), output.data(), 1, 8, 1, stream, api::Method::Warp32)); + unsupported(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, api::Method::Warp32)); + unsupported(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, api::Method::Warp32)); + CHECK(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, api::Method::Warp32)); + CHECK(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, api::Method::Warp32)); + CHECK(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, api::Method::Warp32)); + summary.unsupported_warp32_checks += 3; +#endif + CHECK(suStreamSynchronize(stream)); + output.download(stream, true); +} + + +template Summary validate(suStream_t stream, const char* dtype, const Options& options) { + Summary result; + const std::vector dims = options.custom_shape ? std::vector{options.dim} + : (options.quick ? std::vector{1, 64, 256} : std::vector{1, 2, 4, 8, 16, 32, 64, 128, 256}); + const std::vector rows = options.custom_shape + ? std::vector{checked_shape(options.batch, options.seq, options.heads, options.dim)} + : (options.quick ? std::vector{3} : std::vector{1, 3, 17, 257}); + for (int n : dims) { + for (auto r : rows) { + for (int normalized = 0; normalized < (n == 1 ? 1 : 2); ++normalized) { + const float scale = normalized ? 1.0f / std::sqrt(static_cast(n)) : 1.0f; + for (int pattern = 0; pattern < 5; ++pattern) { + // 零值/脉冲无随机性，只计一次；N=1 的两种 scale 相同，也只计一次。 + const int seeds = pattern < 3 && !options.quick ? 3 : 1; + for (int seed = 0; seed < seeds; ++seed) + one_case(stream, r, n, scale, pattern, 123 + 7919 * seed, dtype, result); + } + } + } + std::cout << "VALIDATION_PROGRESS dtype=" << dtype << " n=" << n << " cases=" << result.cases << std::endl; + } + if (!options.quick && !options.custom_shape) { + // 超过 65535 个 block 的网格上限，让同一个 block 必须处理下一行。 + for (int n : {1, 2}) one_case(stream, 65537, n, 1.0f, 0, 1847, dtype, result); +#if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + // 每个 Warp32 CTA 处理 4 行，262145 同时覆盖网格复用和不满 CTA 的尾行。 + for (int n : {1, 2}) { + one_case(stream, 262145, n, 1.0f, 0, 4541, dtype, result); + ++result.warp32_grid_stride_cases; + result.warp32_grid_stride_elements += 262145u * n; + } +#endif + } + contract_tests(stream, result); + std::cout << "VALIDATION_PASS dtype=" << dtype << " cases=" << result.cases << " elements=" << result.elements + << " max_rounded_error=" << std::setprecision(12) << result.max_rounded_error + << " max_unrounded_error=" << result.max_unrounded_error << " contract_checks=" << result.contract_checks + << " warp32_cases=" << result.warp32_cases << " exact_warp32_elements=" << result.exact_warp32_elements + << " warp32_grid_stride_cases=" << result.warp32_grid_stride_cases + << " unsupported_warp32_checks=" << result.unsupported_warp32_checks << std::endl; + return result; +} + +template void benchmark(suStream_t stream, const char* dtype, const Options& o, std::ofstream& csv) { + struct Shape { std::size_t b, s, h; int n; }; + std::vector shapes; + if (o.custom_shape) shapes.push_back({o.batch, o.seq, o.heads, o.dim}); + else for (int n : {64, 128, 256}) for (auto rows : {1, 17, 257, 4096, 16384}) + shapes.push_back({rows >= 4096 ? static_cast(rows / 1024) : 1, + rows >= 4096 ? 64 : static_cast(rows), rows >= 4096 ? 16u : 1u, n}); + struct Configuration { api::Method method; int operation; const char* name; }; + // 无 Warp32 宏时保留原来的 6 路顺序；启用后追加 3 路，独立输出新的原始样本。 + std::vector configurations{ + {api::Method::Baseline, 0, "baseline_transform"}, {api::Method::Optimized, 0, "optimized_transform"}, + {api::Method::Baseline, 1, "baseline_split"}, {api::Method::Optimized, 1, "optimized_split"}, + {api::Method::Baseline, 2, "baseline_fused"}, {api::Method::Optimized, 2, "optimized_fused"}}; +#if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + configurations.push_back({api::Method::Warp32, 0, "warp32_transform"}); + configurations.push_back({api::Method::Warp32, 1, "warp32_split"}); + configurations.push_back({api::Method::Warp32, 2, "warp32_fused"}); +#endif + const int configuration_count = static_cast(configurations.size()); + for (const auto shape : shapes) { + const auto rows = checked_shape(shape.b, shape.s, shape.h, shape.n), count = rows * shape.n; + const auto input = make_input(rows, shape.n, 0, 2909); + Guarded x(count, stream), y(count, stream); + Guarded q(rows * ((shape.n + 1) / 2), stream); + Guarded s(rows, stream); + x.upload(input, stream); + const float scale = 1.0f; + auto launch = [&](int which) { + const auto cfg = configurations[which]; + if (cfg.operation < 2) CHECK(api::transform(x.data(), y.data(), rows, shape.n, scale, stream, cfg.method)); + if (cfg.operation == 1) CHECK(api::quantize_int4(y.data(), q.data(), s.data(), rows, shape.n, stream, cfg.method)); + if (cfg.operation == 2) CHECK(api::transform_int4(x.data(), q.data(), s.data(), rows, shape.n, scale, stream, cfg.method)); + }; + for (int which = 0; which < configuration_count; ++which) for (int i = 0; i < 10; ++i) launch(which); + CHECK(suStreamSynchronize(stream)); + suEvent_t begin, end; + CHECK(suEventCreate(&begin)); CHECK(suEventCreate(&end)); + for (int group = 0; group < o.groups; ++group) { + // 各组轮换方法顺序；两端事件之间无分配、CPU 参考或主机设备复制。 + for (int order = 0; order < configuration_count; ++order) { + const int which = (order + group) % configuration_count; + const auto cfg = configurations[which]; + CHECK(suEventRecord(begin, stream)); + for (int i = 0; i < o.repeats; ++i) launch(which); + CHECK(suEventRecord(end, stream)); + CHECK(suEventSynchronize(end)); + float elapsed = 0; + CHECK(suEventElapsedTime(&elapsed, begin, end)); + const double us = static_cast(elapsed) * 1000.0 / o.repeats; + if (!(us > 0) || !std::isfinite(us)) throw std::runtime_error("invalid event timing"); + const std::size_t logical_bytes = cfg.operation == 0 ? count * sizeof(T) * 2 + : (cfg.operation == 1 ? count * sizeof(T) * 3 : count * sizeof(T)) + rows * ((shape.n + 1) / 2) + rows * sizeof(float); + csv << dtype << ',' << shape.b << ',' << shape.s << ',' << shape.h << ',' << shape.n << ',' << rows + << ',' << cfg.name << ',' << group << ',' << order << ',' << o.repeats << ',' << std::setprecision(12) << us + << ',' << logical_bytes << ',' << logical_bytes / us / 1000.0 << ',' << count * sizeof(T) + << ",2909,true,1\n"; + csv.flush(); + } + } + CHECK(suEventDestroy(begin)); CHECK(suEventDestroy(end)); + x.download(stream, true); + const auto output = y.download(stream); + const auto packed = q.download(stream); + const auto scales = s.download(stream); + std::vector actual(count); + for (std::size_t i = 0; i < count; ++i) actual[i] = read(output[i]); + const auto quantized = hadamard::quantize_int4(actual, shape.n); + if (packed != quantized.packed || scales != quantized.scales) + throw std::runtime_error("benchmark-size fused INT4 differs from CPU quantization"); + for (std::size_t row : {std::size_t(0), rows / 2, rows - 1}) { + std::vector sample(shape.n); + for (int i = 0; i < shape.n; ++i) sample[i] = read(input[row * shape.n + i]); + const auto expected = hadamard::dense_reference(sample, shape.n, scale); + for (int i = 0; i < shape.n; ++i) { + const double error = std::abs(actual[row * shape.n + i] - read(rounded(static_cast(expected[i])))); + if (!(error < (std::string(dtype) == "fp16" ? .01 : .05))) + throw std::runtime_error("benchmark-size sampled dense oracle mismatch"); + } + } + std::cout << "BENCHMARK_PROGRESS dtype=" << dtype << " rows=" << rows << " n=" << shape.n << std::endl; + } +} + +void write_summary(std::ostream& f, const char* dtype, const Summary& s) { + f << '"' << dtype << "\":{\"cases\":" << s.cases << ",\"elements\":" << s.elements + << ",\"exact_baseline_optimized_elements\":" << s.exact_transform_elements + << ",\"warp32_cases\":" << s.warp32_cases << ",\"exact_baseline_warp32_elements\":" << s.exact_warp32_elements + << ",\"warp32_grid_stride_cases\":" << s.warp32_grid_stride_cases + << ",\"warp32_grid_stride_elements\":" << s.warp32_grid_stride_elements + << ",\"unsupported_warp32_checks\":" << s.unsupported_warp32_checks + << ",\"max_abs_error_rounded_fp64\":" << std::setprecision(15) << s.max_rounded_error + << ",\"max_abs_error_unrounded_fp64\":" << s.max_unrounded_error + << ",\"api_contract_checks\":" << s.contract_checks << '}'; +} + +int main(int argc, char** argv) { + Options options; + try { options = parse(argc, argv); } + catch (const std::exception& e) { std::cerr << "INVALID_ARGUMENT " << e.what() << '\n'; return 2; } + try { + const auto host_rne_checks = verify_host_bf16_rne(); + std::cout << "HOST_BF16_RNE_PASS checks=" << host_rne_checks << std::endl; + CHECK(suSetDevice(0)); + suDeviceProp prop{}; + CHECK(suGetDeviceProperties(&prop, 0)); +#if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + if (prop.warpSize != 32) throw std::runtime_error("Warp32 build requires a real device reporting warpSize=32"); +#endif + int runtime = 0, driver = 0; + CHECK(suRuntimeGetVersion(&runtime)); CHECK(suDriverGetVersion(&driver)); + std::cout << "DEVICE name=" << prop.name << " warp=" << prop.warpSize << " runtime=" << runtime << " driver=" << driver << std::endl; + suStream_t stream; + CHECK(suStreamCreateWithFlags(&stream, suStreamNonBlocking)); + Summary fp16, bf16; + if (options.validate) { + if (options.dtype != "bf16") fp16 = validate(stream, "fp16", options); + if (options.dtype != "fp16") bf16 = validate(stream, "bf16", options); + std::ofstream json(options.json); + if (!json) throw std::runtime_error("cannot create validation JSON " + options.json); + json << "{\"status\":\"PASS\",\"full_matrix\":" << ((!options.quick && !options.custom_shape && options.dtype == "both") ? "true" : "false") + << ",\"host_bf16_rne_checks\":" << host_rne_checks +#if defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + << ",\"warp32_enabled\":true,\"methods\":[\"baseline\",\"optimized\",\"warp32\"]" +#else + << ",\"warp32_enabled\":false,\"methods\":[\"baseline\",\"optimized\"]" +#endif +#if defined(HADAMARD_BIREN_BALANCED_PACK) && defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + << ",\"balanced_pack_enabled\":true" +#else + << ",\"balanced_pack_enabled\":false" +#endif +#if defined(HADAMARD_BIREN_SMALL_BATCH) && defined(HADAMARD_BIREN_WARP32) && defined(__SUPACC__) + << ",\"small_batch_warp_enabled\":true,\"small_batch_warp_max_rows\":64" +#else + << ",\"small_batch_warp_enabled\":false" +#endif + << ",\"oracle\":\"all-element FP64 dense, rounded to output dtype\",\"fp16_tolerance_strict\":0.01,\"bf16_tolerance_strict\":0.05," + << "\"warmup_not_counted\":true,\"dtypes\":{"; + bool comma = false; + if (options.dtype != "bf16") { write_summary(json, "fp16", fp16); comma = true; } + if (options.dtype != "fp16") { if (comma) json << ','; write_summary(json, "bf16", bf16); } + json << "}}\n"; + if (!json) throw std::runtime_error("failed writing validation JSON"); + } + if (options.benchmark) { + std::ofstream csv(options.csv); + if (!csv) throw std::runtime_error("cannot create benchmark CSV " + options.csv); + csv << "dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale\n"; + if (options.dtype != "bf16") benchmark(stream, "fp16", options, csv); + if (options.dtype != "fp16") benchmark(stream, "bf16", options, csv); + if (!csv) throw std::runtime_error("failed writing benchmark CSV"); + } + CHECK(suStreamDestroy(stream)); + std::cout << "PASS requested validation/benchmark operations completed" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "FAIL " << e.what() << std::endl; + return 1; + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/README.md b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/README.md new file mode 100644 index 00000000..e93fd785 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/README.md @@ -0,0 +1,296 @@ +# 天数智芯 COREX 后端 + +此目录提供天数智芯 MR-V100 的原生 COREX C++ Hadamard 与融合 INT4 后端。最新实测源码 **`a387db3332c6f9b01f128dd681848260c9691281`** 已完成包含 Warp64 的 **1504 组变换验证、180 项 API 契约检查、14 项 CLI 拒绝检查和三轮 4050 条基准样本**。推荐在实测支持条件下显式选择 `Method::Warp64`；API 默认仍为共享内存 `Method::Optimized`。 + +下方保留初版、共享内存版和 Warp64 版的独立版本与原始证据，测试数量不跨版本累加。支持范围是当前 MR-V100/COREX 组合，未宣称所有国产设备、原生 BF16 硬件算术、矩阵单元加速或所有可能输入上的全局最优性能。 + +## 首轮原型：构建与运行 + +在项目的 `03_hadamard_tc/a962695448-rgb` 目录执行，使用与驱动匹配的完整 COREX 开发环境： + +```bash +mkdir -p build/iluvatar +/usr/local/corex/bin/clang++ -x ivcore -std=c++17 -O2 \ + -Iinclude platforms/iluvatar/shared_baseline.cu \ + -L/usr/local/corex/lib -Wl,-rpath,/usr/local/corex/lib -lcudart \ + -o build/iluvatar/shared_baseline +./build/iluvatar/shared_baseline +``` + +若供应商镜像使用其他 SDK 路径，调整路径。不要照搬 NVIDIA 的 `-arch=sm_89`；如需要指定天数目标架构，应使用该 SDK 和实机确认过的编译参数。 + +## 首轮原型：算法与验证范围 + +- 每个 block 处理一行，线程数为 `max(64, N)`，`N` 为 1～256 的 2 次幂。 +- 全部蝶形加减使用 FP32；共享内存上每层先读取、全 block 同步、写回、再次同步。所有线程参加同步，不依赖 32 或 64 的 warp 宽度，也不依赖 WMMA。 +- 变换后先按输出 FP16/BF16 舍入，再量化。每行对称 INT4 为 `[-7,7]`，采用最近偶数舍入，偶数元素放低四位；零行 scale 为 1，`N=1` 的空高四位为 0。 +- 所有传输和 kernel 使用同一个显式非阻塞 stream，回传后同步，再在 CPU 上检查。 +- 每种精度 162 组变换用例：9 种 N、3 种行数（1/17/257）、2 种 scale（1 和 `1/sqrt(N)`）、零值/脉冲/确定性随机输入。`N=1` 的两种 scale 数值相同，但两条参数组合均执行。 +- 每组检查所有元素的 CPU FWHT 误差，并对首行/中间行/末行使用独立 FP64 稠密 Hadamard oracle；阈值严格小于 FP16 `1e-2`、BF16 `5e-2`。 +- 对真实 GPU 变换输出运行 CPU 量化，检查全部打包字节和 scale 与 GPU 分步、融合路径完全一致；另有每种精度一组正负半整数最近偶数舍入测试。 +- 随机输入在 `[-0.25,0.25)`；这只是初步移植范围，不等同于主线更广分布、离群值、多随机种子的完整验收。 + +输出包含真实设备名称、warp 宽度、运行时版本、驱动版本、每种精度用例数、元素数、最大绝对误差；失败以非零退出码退出。主程序始终尝试两种精度，不通过静默跳过 BF16 制造通过结果。 + +## 首轮原型：已取得的 MR-V100 实机证据 + +2026-09-05 在真实天数智芯 MR-V100 32GB 上完成了首轮 COREX 原生编译与运行，编译和验证进程均以退出码 0 结束。设备报告 `warp=64`、`runtime=10020`、`driver=10020`；后两个数值是兼容运行时 API 返回值，不代表使用 NVIDIA CUDA 10.2 编译器。实机编译器为 `/usr/local/corex/bin/clang++` 18.1.8，使用 `-x ivcore -std=c++17 -O2 -Iinclude -lcudart`。 + +| 精度 | 变换用例 | 检查元素 | 最大绝对误差 | 半整数舍入 | CPU / 分步 / 融合 INT4 | +|---|---:|---:|---:|---|---| +| FP16 | 162 | 843,150 | 0.00385761261 | 通过 | 全部字节与 scale 一致 | +| BF16 | 162 | 843,150 | 0.0301055908 | 通过 | 全部字节与 scale 一致 | + +此处误差是 GPU 输出与**未舍入到 FP16/BF16 的 FP32 CPU FWHT，以及抽样 FP64 稠密 oracle**的最大绝对差。这个定义包含最终存储精度的舍入误差，不能直接与其他实验中采用已舍入参考输出的最大误差横向比较。 + +通过结果证明本程序的 BF16 存取/转换与 FP32 累加路径在该实机和 SDK 上可用；不证明硬件原生 BF16 算术，也不意味着国产平台适配已全部完成。 + +首轮源码的汇总 `printf` 触发 COREX 格式检查警告。最终版改用主机 C++ 输出流，算法和用例保持不变，并重新在 MR-V100 编译运行：编译无警告、退出码 0，324 组变换及两组舍入检查再次通过。最终源码 SHA256 为 `d8b00f6e60feaf0dd450ca66255e05bd7e0e01a060d911a9fc7c61779023f180`。 + +原始验证输出、空的成功编译日志、设备环境与哈希清单保存在 [results/iluvatar](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/manifest.json)。重复运行同一批组合不计为新增独立用例。 + +## C++ API 与共享内存优化实现 + +本节描述上述 `b326789` 实机版本的 `Baseline` 和 `Optimized` 两种方法；后续增加的方法必须单独验证，不能继承本轮结果。 + +| 文件 | 职责 | +|---|---| +| [hadamard_api.h](hadamard_api.h) | FP16/BF16 变换、INT4 量化及融合变换量化的公开接口和内存契约 | +| [hadamard_api.cu](hadamard_api.cu) | 基线、共享内存优化版，以及显式编译启用的 Warp64 实现 | +| [validate_and_benchmark.cu](validate_and_benchmark.cu) | 独立 FP64 稠密参考、接口检查、内存边界检查及设备事件计时 | +| [run_platform.py](run_platform.py) | COREX 构建、CLI 拒绝测试、正确性验证、基准汇总及文件哈希记录 | + +`Baseline` 保留每个元素对应一个线程、每层两次同步和线程 0 串行最大值归约。`Optimized` 让每个线程独占一个蝶形的两个输入/输出，每层仅在结束时同步，并用并行最大值归约计算量化 scale。两者均不依赖 NVIDIA warp32 或天数 warp64 的隐式同步，不包含 WMMA。`Optimized` 是实现名称；是否更快由同机测量决定，不能由名称推断。 + +API 使用 `hadamard::iluvatar` 命名空间，提供 `__half` 和 `__nv_bfloat16` 重载： + +- `transform`：Hadamard 变换，允许完全原位操作 `input == output`；不允许其他部分重叠。 +- `quantize_int4`：量化已经舍入到 FP16/BF16 的变换输出。 +- `transform_int4`：融合变换与量化，变换后先舍入到对应存储类型，保持与分步路径相同的量化语义。 + +输入为连续设备内存 `[rows, n]`，`n` 是 1～256 的 2 次幂。连续 `[batch, seq, heads, head_dim]` 按 `rows=batch*seq*heads`、`n=head_dim` 传入；乘法溢出由调用方先检查。API 不接受 stride 参数，也不会为非连续视图自动整理内存。`scale` 必须有限且为正；输入应有限，变换结果应在输出存储类型的有限范围内。 + +调用方负责分配当前设备上容量足够的缓冲区：输入/变换输出各 `rows*n` 个对应 dtype，INT4 输出 `rows*ceil(n/2)` 字节，scale 输出 `rows` 个 float。输入和变换输出要求 2 字节对齐，scale 要求 float 对齐；量化接口的各缓冲区互不重叠。API 检查形状、scale、方法、对齐、整数/地址范围溢出及缓冲区重叠，**不查询真实分配容量或设备归属**。 + +全部操作在调用方 stream 上异步发射，不分配、不复制、不等待。返回值用于检查参数/发射错误，执行错误由调用方同步 stream 时检查。`rows=0` 合法：检查 `n`、`scale`、方法后返回，不检查指针或发射内核；CLI 为便于实际测试，只接受正形状。 + +调用示例（缓冲区及 stream 已由调用方创建）： + +```cpp +#include "platforms/iluvatar/hadamard_api.h" + +cudaError_t transform_then_quantize( + const __half* input, __half* output, std::uint8_t* packed, + float* scales, std::size_t rows, cudaStream_t stream) { + namespace api = hadamard::iluvatar; + auto status = api::transform(input, output, rows, 128, 1.0f, stream, + api::Method::Optimized); + if (status != cudaSuccess) return status; + return api::quantize_int4(output, packed, scales, rows, 128, stream, + api::Method::Optimized); +} +``` + +只需要量化输出时，可用 `transform_int4(input, packed, scales, rows, 128, 1.0f, stream, method)` 替换两次调用。不要在前一调用结束前释放缓冲区；读取主机结果前还需安排复制及 stream 同步。这是原生 C++ API，当前不提供天数 PyTorch Python 扩展。 + +## 新版构建与可复现运行 + +从项目 `03_hadamard_tc/a962695448-rgb` 目录执行。runner 只需要 Python 标准库和已配套安装的 COREX SDK，不安装或替换框架、驱动。结果目录必须尚不存在，避免覆盖先前证据。 + +下面不带 `--warp64` 的命令重现共享内存路径；MR-V100 的推荐 Warp64 命令见后文专节。选择测试版本时同时固定相应源码 commit，不将不同版本样本混成一次实验。 + +```bash +# 快速检查工具链和小矩阵；即使通过，也不代表完整验收。 +python3 platforms/iluvatar/run_platform.py \ + --quick --no-benchmark --output results/iluvatar/api_quick_01 + +# 正式矩阵 + 默认基准：不要加 --quick。 +python3 platforms/iluvatar/run_platform.py \ + --repeats 100 --groups 5 --output results/iluvatar/api_full_01 +``` + +默认编译器是 `/usr/local/corex/bin/clang++`，SDK 根目录是 `/usr/local/corex`。可通过 `--corex-root /实际/SDK/路径` 或 `--compiler /实际/编译器/路径` 指定；二者应属于匹配的安装。runner 使用 `-x ivcore -std=c++17 -O2`，链接配套 `libcudart`；不使用 NVIDIA `-arch=sm_89`，也不启用 fast-math。 + +需要仅构建可执行程序时： + +```bash +mkdir -p build/iluvatar +/usr/local/corex/bin/clang++ -x ivcore -std=c++17 -O2 -Iinclude \ + platforms/iluvatar/hadamard_api.cu \ + platforms/iluvatar/validate_and_benchmark.cu \ + -L/usr/local/corex/lib -Wl,-rpath,/usr/local/corex/lib -lcudart \ + -o build/iluvatar/validate_and_benchmark + +# 直接运行默认完整矩阵。 +./build/iluvatar/validate_and_benchmark --validate \ + --json build/iluvatar/validation.json + +# 定位一个连续四维形状；这是定向验证，full_matrix 为 false。 +./build/iluvatar/validate_and_benchmark --validate --dtype bf16 \ + --batch 2 --seq 17 --heads 3 --dim 128 \ + --json build/iluvatar/validation_custom.json + +# 已构建程序的独立基准进程；每轮使用新的文件名。 +./build/iluvatar/validate_and_benchmark --benchmark --groups 5 --repeats 100 \ + --csv build/iluvatar/benchmark_run02.csv +``` + +直接执行二进制会按指定路径写文件，应使用新文件名；只有 runner 对整个结果目录实施“不覆盖既有目录”的检查。二进制参数错误返回 2，设备/正确性失败返回 1；runner 只有请求的所有阶段及对应退出码检查成功才返回 0。快速模式和 `--no-benchmark` 下的成功都不能当作完整性能验收。 + +runner 的主要输出： + +| 文件 | 内容 | +|---|---| +| `build.log`、`invalid_*.log` | 编译原始输出、CLI 拒绝测试原始输出 | +| `validation.log`、`validation.json` | 设备信息、进度、实际用例数、误差与 API 契约检查数；JSON 标明 `full_matrix` | +| `benchmark.log`、`benchmark.csv` | 基准进度和每组未经筛选的事件计时样本 | +| `run_summary.json` | 源文件 SHA256、Git HEAD/状态、编译器信息、各阶段命令/退出码、结果摘要及产物哈希 | +| `validate_and_benchmark` | 本轮编译的本机可执行文件；其 SHA256 单独记录 | + +`run_summary.json` 中的 Git HEAD 是执行时读取值；若源码尚未提交，必须同时依据 `git_status` 和源文件 SHA256 追溯，不能仅将 HEAD 当成实际测试代码。发布日志前核查访问地址等私有信息；包含私有信息的原件与本机二进制可单独保存，公开副本使用自己的哈希。 + +## 共享内存 API：完整验证矩阵 + +以下矩阵已由固定版本 `b326789` 完成，实际计数与误差见后文原始结果： + +- FP16/BF16 分别遍历 `N={1,2,4,8,16,32,64,128,256}`、`rows={1,3,17,257}`，scale 为 1 和 `1/sqrt(N)`；N=1 的重复 scale 只计算一次。 +- 输入包括均匀分布 `[-1,1)`、标准差 0.5 的正态分布、带幅度 8 离群值的均匀分布、全零和单位脉冲。前三种使用种子 `123、8042、15961`；零值/脉冲没有随机性，只计一次。 +- 全量模式另外执行 `rows=65537`、N=1/2 的用例，覆盖 block 循环处理多行的同步和网格上限路径。每 dtype 实际完成 750 组合、共 1500，不将重复基准发射计为新增正确性用例。 +- 所有元素与独立 FP64 稠密 Hadamard 矩阵结果比较。验收参考先经 FP32 转换，再舍入到输出 dtype，与主线已舍入参考约定一致；FP16 绝对误差严格小于 `1e-2`，BF16 严格小于 `5e-2`。同时保留相对未舍入 FP64 结果的最大误差，区分存储精度误差；不与首轮原型的误差列直接混用。 +- 基线、优化和优化原位变换逐字节一致；两种方法的分步/融合 INT4 对照 CPU 对实际设备变换结果的量化，packed bytes 和 scale 必须完全一致。分配前后哨兵及输入副本用于检查边界写越界和输入被改动。 +- API 契约覆盖非法 N/scale/方法、空指针、部分重叠、不对齐、大小溢出、零行及手写正负半整数舍入预期。runner 另执行 14 个 CLI 拒绝用例，要求实际返回 2。 + +基准默认覆盖两 dtype、N=64/128/256、rows=1/17/257/4096/16384。大形状以 batch/seq/heads 的乘积表示。每形状比较基线/优化的 transform、split、fused 六条路径，各先预热 10 次；默认 5 组，每组重复 100 次，组间轮换方法顺序。统计使用全部组样本，报告中位数、最小值、最大值以及包含退化的基线/优化比值。 + +计时使用同一显式 stream 的兼容设备事件，区间不含分配、主机/设备复制或 CPU 参考；输入只读重复使用，属于缓存可复用的测量。**`kernel_us` 是该发射序列的事件区间均值，不是端到端延迟，也不是隔离测得的单个内核执行时间；主机发射之间可能存在设备空闲间隔。`logical_GBs` 是逻辑张量读写量估计，不是硬件测得的物理显存带宽。**每个大基准形状另外核验量化一致性及首/中/末行稠密参考。 + +## 共享内存 API：MR-V100 实测结果 + +2026-09-05，COREX `clang++ 18.1.8 / 4.4.0` 构建实际提交 `b3267893a53e45d2e7f35dc2d6e2583c638f4112`，运行时工作树干净，五个参与构建/验证的源文件 SHA256 已与该提交的 Git 对象核对。构建日志为空、退出码 0；完整 runner 退出码 0、`quick=false`、`full_matrix=true`。环境及完整命令见 [run_summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/run_summary.json)。 + +| 精度 | 实际变换组合 | 检查元素 | 对已舍入 FP64 参考的最大绝对误差 | 对未舍入 FP64 参考的最大绝对误差 | API 契约检查 | +|---|---:|---:|---:|---:|---:| +| FP16 | 750 | 3,318,829 | 0.00390625 | 0.015620231628418 | 61 | +| BF16 | 750 | 3,318,829 | 0.000003814697265625 | 0.124984741210938 | 61 | + +两 dtype 的基线、共享内存优化版和优化版原位变换均逐字节一致，全部分步/融合 INT4 packed bytes 与 scale 对照 CPU 对实际设备输出的量化精确一致。两种误差均保留在 [validation.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/validation.json)；**阈值判定使用已舍入参考，未舍入列包含输出存储的量化误差，不能将其也描述为低于同一阈值。**这与前文首轮原型的未舍入误差定义不同。 + +三次独立基准进程使用相同二进制 SHA256 `86e38be3a739548104616831a489799901d1bdba9dc9b26d825b60c030fdb93b`，均退出 0。每轮 900 条原始组样本，合计 2700 条；共有 30 个 shape/dtype 条件、六种方法和 90 组“优化方法对匹配基线”的比较。三轮分别保存为 [第一轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/benchmark.csv)、[第二轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/benchmark_run2.csv)、[第三轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/benchmark_run3.csv)，重复进程命令与退出码见 [repeat_runs.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/repeat_runs.json)。 + +按每轮五组的中位数比较，耗时下降定义为 `100*(1-优化耗时/基线耗时)`： + +- 60/90 组比较在**每一轮**都减少至少 5% 的耗时；66/90 组每轮均更快。 +- 24/90 组至少一轮退化，其中 22 组每轮均退化；4 组每轮退化超过 3%。因此共享内存优化版没有在所有输入上胜出。 +- 这四组均为 N=256 的纯变换：FP16、BF16 各自的 rows=1/17。三轮退化范围分别约为 FP16 rows=1 的 4.89%～5.11%、rows=17 的 3.24%～4.28%，BF16 rows=1 的 4.76%～4.92%、rows=17 的 3.22%～4.27%。 + +全部样本、跨轮波动及退化比较见 [analysis.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/analysis/analysis.json) 和 [method_summary.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/analysis/method_summary.csv)。组内/跨轮标准差仅描述这几次运行的波动，不是置信区间，也不证明任意共享负载下的性能。 + +第二、三轮前后保存了 `ixsmi` 快照，报告 MR-V100、32GB、驱动/IX-ML 4.4.0；快照可见时钟信息，不能据此宣称独占或固定时钟。实际物理计算配额与共享状态仍未得到供应商确认。本轮不包含硬件性能计数器、端到端测量或 Warp64 结果。 + +### 验证脚手架初始化缺陷与修复边界 + +早期 quick 验证首先报哨兵被改写；随后增加阶段诊断，发现契约测试的 `initialized` 阶段、尚未执行本段算子时，输入 payload 已不符合初始化模式。原始失败分别保留在 [首次 quick 日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/prior_failures/iluvatar-api-quick-e508f98/validation.log) 和 [诊断日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/prior_failures/iluvatar-api-diag-e508f98/validation.log)，对应目录有完整 runner 摘要及源文件 hash。诊断版有未提交的插桩，必须按其摘要里的 `source_sha256` 和工作区状态追溯，不能只认 HEAD。 + +`b326789` 将脚手架缓冲区初始化改为调用方显式 stream 上的 `cudaMemsetAsync`，让初始化、上传、kernel、回读具备同一 stream 顺序，并避免初始化的异步 H2D 读取随后被上传逻辑更新的主机 shadow。同时增加初始化、非法参数、上传和量化各阶段的缓冲区检查。实际设备算法、误差阈值和测试矩阵没有通过降低要求来规避失败；修复后完整矩阵和契约检查通过。 + +这证明记录版本修复后通过了现有测试，也说明早期报错不能直接归因于 Hadamard 蝶形内核；**不将此记录扩展成已经证明某个 COREX 驱动缺陷或排除了所有潜在竞态。**失败运行不计入通过数量。 + +### 公开归档 + +[公开清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/shared_b326789/manifest.json) 逐文件记录字节数和 SHA256，包含原始成功日志、三轮 CSV、分析结果和前述失败证据。29 份成功运行原始文件先经传输清单校验，复制后保持字节不变；目录内 `.gitattributes` 将证据标记为 `-text`，避免 Git 换行转换破坏哈希。二进制和租赁凭据未纳入公开材料。当前工作树后续变化不改变本轮固定提交的证据范围。 + +首轮原型证据继续保留在前文及 [初版清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/manifest.json)。后续方法应新增独立记录，保留本轮历史与负例。 + +## Warp64：显式使用与复现 + +`a387db3` 在两种共享内存方法之外增加 `Method::Warp64`。每 CTA 256 个线程处理四行，一行对应完整的 64-lane warp；N≤256 时每个 lane 保存 1/2/4 个 FP32 寄存器槽。行内蝶形、最大值归约和相邻 INT4 元素交换通过经该设备验证的 64-lane shuffle 实现；尾行按整个 warp 屏蔽，网格复用时保持同一行的完整参与。仍先将变换输出舍入到 FP16/BF16，再量化，与公开 API 契约一致。 + +此路径要求同时满足： + +1. 用真实 COREX 编译器生成天数设备代码，编译环境定义 `__ILUVATAR__`，并显式传入 `-DHADAMARD_ILUVATAR_WARP64`。不要手动伪造 `__ILUVATAR__`。 +2. 实际运行设备报告 `prop.warpSize == 64`。runner 会检查；独立 API 调用方必须在初始化阶段预查，因为 API 不在每次发射时查询设备。 +3. 使用与当前设备匹配的驱动/SDK，以及前文规定的连续 FP16/BF16 设备缓冲区、合法 N、正有限 scale、正确容量/对齐及 stream。 + +完整复现命令： + +```bash +python3 platforms/iluvatar/run_platform.py --warp64 \ + --repeats 100 --groups 5 --output results/iluvatar/warp64_full_01 + +# --warp64 是编译选择，不是二进制运行参数。 +# 已生成的该二进制再次运行基准，仍包含九条方法路径。 +./results/iluvatar/warp64_full_01/validate_and_benchmark --benchmark \ + --groups 5 --repeats 100 --csv results/iluvatar/warp64_full_01/benchmark_run2.csv +./results/iluvatar/warp64_full_01/validate_and_benchmark --benchmark \ + --groups 5 --repeats 100 --csv results/iluvatar/warp64_full_01/benchmark_run3.csv +``` + +手动构建时，在前文 COREX 编译命令中额外加入 `-DHADAMARD_ILUVATAR_WARP64`。不启用它时，基线和共享内存方法仍可用；`Warp64` 的非空调用返回 `cudaErrorNotSupported`，不会静默切换算法。合法 `rows=0` 继续遵循无操作约定。 + +独立应用可在确定当前设备后调用一次以下预检查，并检查返回值；设备改变后重新检查： + +```cpp +cudaError_t require_warp64_device() { + int device = 0; + auto status = cudaGetDevice(&device); + if (status != cudaSuccess) return status; + cudaDeviceProp prop{}; + status = cudaGetDeviceProperties(&prop, device); + if (status != cudaSuccess) return status; + return prop.warpSize == 64 ? cudaSuccess : cudaErrorNotSupported; +} +``` + +预检查成功且已按要求编译后，在前文调用示例中显式选择 `api::Method::Warp64`。例如，仅需要量化输出时调用 `api::transform_int4(input, packed, scales, rows, 128, 1.0f, stream, api::Method::Warp64)`；变换或分步量化也显式传入同一方法。检查发射返回值，并在回读前同步相应 stream。**仅观察到 warpSize=64 不足以证明其他芯片/SDK 具备相同 shuffle 语义；其他平台仍需完整重新验证。** + +## Warp64：MR-V100 完整结果 + +实测提交 `a387db3332c6f9b01f128dd681848260c9691281` 工作树干净，COREX 18.1.8/4.4.0 编译无输出且退出 0。完整 runner 记录 `status=PASS`、`quick=false`、`warp64_enabled=true`、`full_matrix=true`。源文件 hash 与固定提交逐一吻合；构建命令、阶段退出码和原始输出见 [run_summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/run_summary.json)。 + +| 精度 | 实际变换组合 | 检查元素 | 基线/共享版/Warp64 变换一致性 | API 契约检查 | 对已舍入 FP64 参考的最大误差 | +|---|---:|---:|---|---:|---:| +| FP16 | 752 | 4,105,264 | 全部逐字节一致 | 90 | 0.00390625 | +| BF16 | 752 | 4,105,264 | 全部逐字节一致 | 90 | 0.000003814697265625 | + +这 1504 组是在共享版矩阵上加入每 dtype 两个 `rows=262145, N=1/2` 的网格复用/尾行用例，所有三种方法均对照，原位变换、输入保持、哨兵、分步/融合 INT4 的 bytes 和 scale 检查全部通过。每 dtype 新增网格用例覆盖 786,435 个元素，已包含在上表，不能重复相加。14 项 CLI 拒绝检查来自本次完整 runner，不叠加 quick 重跑。 + +所有元素的独立 FP64 稠密参考定义和误差阈值与共享版相同；未舍入最大误差仍分别为 FP16 `0.015620231628418`、BF16 `0.124984741210938`，不声称未舍入误差也低于已舍入阈值。实测原件见 [validation.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/validation.json) 和 [validation.log](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/validation.log)。 + +### 同机三轮性能 + +三次独立进程使用二进制 SHA256 `9496d38d25617c2eb69b0d46973e26ac2a10d59c80fba75e73767b13a5c69dac`，均退出 0。每轮 1350 条原始样本、共 4050 条：两 dtype × N=64/128/256 × rows=1/17/257/4096/16384 × 九方法 × 五组。每组 100 次重复，统计每轮的组中位数；测量边界与前文设备事件说明相同。 + +**Warp64 的 90/90 组匹配比较在每一轮都比基线降低至少 5% 耗时**；在本次全部形状、操作和三轮中，相对基线及共享内存优化版均未出现退化。以下范围覆盖本次全部对应形状及三轮，定义为“对照中位耗时 / Warp64 中位耗时”；不使用旧六路径实验中的耗时作分母： + +| 操作 | 基线 / Warp64 | 共享内存优化版 / Warp64 | +|---|---:|---:| +| transform | 1.1907～3.3867 倍 | 1.2063～3.3927 倍 | +| split | 1.3044～5.9580 倍 | 1.2328～3.3154 倍 | +| fused | 1.5245～7.3599 倍 | 1.3253～2.9125 倍 | + +最小基线降时出现在 BF16 rows=257、N=64 的 transform，三轮约为 16.015%、16.171%、16.216%。这些结论仅覆盖本次 N=64/128/256 的性能矩阵；N=1～32 虽有正确性覆盖，不能据此外推相同加速范围。 + +只比较 Warp64 自身的分步与融合路径，在同一轮、相同 shape/dtype/scale 的五组中位数上，`split/fused` 为 **1.2915970～1.8199551 倍**，对应融合耗时减少 **22.5765%～45.0536%**。30/30 个形状条件每轮均降低至少 5%，未观察到负收益；最小收益出现在 FP16 `[16,64,16,128]`，三轮降时约为 22.6001%、22.5765%、22.6252%。这组比值只使用当前九路径实验，不混用旧六路径的分步时间。 + +在相同九路径运行里，共享内存优化版自身仍有 22/90 组每轮比基线慢，其中四组每轮超过 3%；其余符合每轮至少 5% 降时的有 62 组。分析总数 152 是 **共享版 62 + Warp64 90**，不是 152 组 Warp64 收益。旧六路径实验的共享版 60 组与本轮 62 组分别保留，不能混成同一统计。 + +全部原始样本见 [第一轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/benchmark.csv)、[第二轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/benchmark_run2.csv)、[第三轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/benchmark_run3.csv)；独立进程与相同二进制的记录见 [repeat_runs.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/repeat_runs.json)。完整分析见 [analysis.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/analysis/analysis.json) 与 [method_summary.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/analysis/method_summary.csv)，逐文件字节/hash 清单见 [manifest.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/iluvatar/warp64_a387db3/manifest.json)。目录 `-text` 规则保留原始字节；不发布租赁凭据或本机二进制。 + +## 本次交付范围与局限 + +- 已验证：MR-V100 32GB、IX-ML/驱动 4.4.0、COREX 18.1.8；连续 FP16/BF16、N=1～256 的 2 次幂、显式 stream、原位变换、INT4 分步与融合、完整 runner 和三轮同机事件基准。共享内存与 Warp64 原始结果各自固定到源码版本。 +- `Method::Warp64` 通过显式 opt-in 使用；生产默认仍为 `Method::Optimized`，没有将有限性能矩阵自动扩展成所有 N 和所有设备的默认策略。 +- 不提供天数 PyTorch Python 扩展；不承诺非连续张量、任意 N、越出输出类型有限范围的输入或跨设备指针。调用方负责分配容量、设备归属及 stream 生命周期。 +- 未证明物理整卡独占、固定频率、跨租户干扰控制，也未取得硬件性能计数器；`ixsmi` 快照只记录对应时刻的设备状态。没有端到端复制/分配计时，不能将事件耗时当成端到端收益。 +- BF16 使用存储/转换与 FP32 累加，不宣称原生 BF16 算术或矩阵单元加速。其他天数型号、其他 SDK、NVIDIA 和其他国产芯片必须分别验证。 +- 后续工作按固定输入集复测、逐形状选择和证据归档推进；PR 与课程登记在项目所有者验收后单独执行。 + +## 后续待完成 + +1. 已保存并复测当前基线；后续每个优化版本继续单独保留源码、SDK、日志和哈希。 +2. 若继续扩大性能矩阵或调整默认派发，为未测 N/shape 增加对照并重新回归，不宣称当前统计覆盖所有输入。 +3. 当前 MR-V100 用户可在编译与设备预检查满足要求后显式使用已验证的 Warp64 路径；其他环境先保留共享实现并单独取证。 +4. 补充独占/共享配额和工具权限；若报告端到端性能，另行实现并明确包含的分配/复制/同步边界。 +5. 发布时核对最终用户分支提交、源码与原始证据的公开 hash。框架扩展和原生 BF16 算术支持属于额外能力，保持上述边界。 + +国产适配结果不能替代九齿项目要求的 A100 验证。 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/analyze_runs.py b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/analyze_runs.py new file mode 100644 index 00000000..700e7ba2 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/analyze_runs.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""比较多个天数 benchmark.csv；保留负例，输出可复算 JSON 与逐方法汇总 CSV。""" +import argparse +import csv +import hashlib +import json +import math +from pathlib import Path +import re +import statistics +import sys + + +KEYS = ("dtype", "batch", "seq", "heads", "dim", "scale") +OPERATIONS = ("transform", "split", "fused") +CONDITIONS = ("rows", "repeats", "seed", "input_read_only", "input_working_set_bytes") +REQUIRED = set(KEYS + CONDITIONS + ("method", "group", "order", "kernel_us", "logical_io_bytes", "logical_GBs")) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def integer(value, name, minimum=1): + number = int(value) + if number < minimum: + raise ValueError(f"{name} must be >= {minimum}") + return number + + +def number(value, name): + result = float(value) + if not math.isfinite(result) or result <= 0: + raise ValueError(f"{name} must be finite and positive") + return result + + +def stats(values): + average = statistics.mean(values) + median = statistics.median(values) + return {"count": len(values), "median_us": median, "mean_us": average, + "minimum_us": min(values), "maximum_us": max(values), + "population_stddev_us": statistics.pstdev(values), + "cv_percent": statistics.pstdev(values) / average * 100, + "range_over_median_percent": (max(values) - min(values)) / median * 100} + + +def load_run(path, run_index): + buckets = {} + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + if not REQUIRED <= set(reader.fieldnames or ()): + raise ValueError(f"{path.name}: missing columns {sorted(REQUIRED - set(reader.fieldnames or ()))}") + for line, row in enumerate(reader, start=2): + try: + item = dict(row) + for name in ("batch", "seq", "heads", "dim", "rows", "repeats", "input_working_set_bytes", "logical_io_bytes"): + item[name] = integer(row[name], name) + for name in ("group", "order", "seed"): + item[name] = integer(row[name], name, 0) + for name in ("scale", "kernel_us", "logical_GBs"): + item[name] = number(row[name], name) + if item["dtype"] not in ("fp16", "bf16") or not re.fullmatch(r"[a-z][a-z0-9_]*_(transform|split|fused)", item["method"]): + raise ValueError("unsupported dtype or method") + if item["input_read_only"] not in ("true", "false"): + raise ValueError("input_read_only must be true or false") + if item["rows"] != item["batch"] * item["seq"] * item["heads"]: + raise ValueError("rows differs from batch*seq*heads") + if item["dim"] > 256 or item["dim"] & (item["dim"] - 1): + raise ValueError("dim must be a power of two <=256") + if item["input_working_set_bytes"] != item["rows"] * item["dim"] * 2: + raise ValueError("input working set does not match FP16/BF16 tensor size") + key = tuple(item[name] for name in KEYS) + groups = buckets.setdefault((key, item["method"]), {}) + if item["group"] in groups: + raise ValueError("duplicate shape/method/group; do not concatenate repeated runs into one CSV") + groups[item["group"]] = item + except (ValueError, TypeError, KeyError) as error: + raise ValueError(f"{path.name}:{line}: {error}") from error + if not buckets: + raise ValueError(f"{path.name}: no samples") + methods = {method for key, method in buckets} + prefixes = {method.rsplit("_", 1)[0] for method in methods} + if "baseline" not in prefixes or len(prefixes) < 2: + raise ValueError(f"{path.name}: require baseline and at least one candidate implementation") + if methods != {prefix + "_" + op for prefix in prefixes for op in OPERATIONS}: + raise ValueError(f"{path.name}: every implementation must include transform/split/fused") + keys = {key for key, method in buckets} + for key in keys: + if {method for case, method in buckets if case == key} != methods: + raise ValueError(f"{path.name}: incomplete method matrix for {key}") + reference = buckets[(key, "baseline_transform")] + if set(reference) != set(range(len(reference))): + raise ValueError(f"{path.name}: group IDs must be contiguous from zero for {key}") + condition = tuple(reference[0][name] for name in CONDITIONS) + for method in methods: + groups = buckets[(key, method)] + if set(groups) != set(reference): + raise ValueError(f"{path.name}: methods have different group coverage for {key}") + if any(tuple(item[name] for name in CONDITIONS) != condition for item in groups.values()): + raise ValueError(f"{path.name}: mixed measurement conditions for {key}") + if len({item["logical_io_bytes"] for item in groups.values()}) != 1: + raise ValueError(f"{path.name}: changing logical I/O estimate for {key}/{method}") + for group in reference: + if {buckets[(key, method)][group]["order"] for method in methods} != set(range(len(methods))): + raise ValueError(f"{path.name}: duplicated/missing method order in group {group}") + return {"run": run_index, "file": path.name, "sha256": digest(path), "bytes": path.stat().st_size, + "raw_rows": sum(len(values) for values in buckets.values()), "shape_dtype_scale_cases": len(keys), + "buckets": buckets} + + +def analyze(paths): + if len(paths) < 2: + raise ValueError("provide at least two separate run CSV files") + if len({p.resolve() for p in paths}) != len(paths): + raise ValueError("the same input path was supplied twice") + runs = [load_run(path, index) for index, path in enumerate(paths, start=1)] + reference = runs[0]["buckets"] + for run in runs[1:]: + if set(run["buckets"]) != set(reference): + raise ValueError(f"run {run['run']}: shape/dtype/scale/method matrix differs; compare matching runs") + for key, groups in reference.items(): + peer = run["buckets"][key] + if set(peer) != set(groups): + raise ValueError(f"run {run['run']}: number of groups differs for {key}") + names = CONDITIONS + ("logical_io_bytes",) + if tuple(groups[0][name] for name in names) != tuple(peer[0][name] for name in names): + raise ValueError(f"run {run['run']}: seed/repeats/working-set conditions differ for {key}") + methods, by_key = [], {} + for (key, method) in sorted(reference): + entry = dict(zip(KEYS, key), method=method) + first = reference[(key, method)][0] + entry["conditions"] = {name: first[name] for name in CONDITIONS + ("logical_io_bytes",)} + entry["per_run"] = [] + for run in runs: + samples = [sample for _, sample in sorted(run["buckets"][(key, method)].items())] + values = [sample["kernel_us"] for sample in samples] + entry["per_run"].append({"run": run["run"], **stats(values), + "samples": [{name: s[name] for name in ("group", "order", "kernel_us", "logical_GBs")} for s in samples]}) + entry["across_run_medians"] = stats([value["median_us"] for value in entry["per_run"]]) + methods.append(entry) + by_key[(key, method)] = entry + comparisons = [] + for key, candidate_method in sorted(reference): + prefix, operation = candidate_method.rsplit("_", 1) + if prefix != "baseline": + base = by_key[(key, "baseline_" + operation)] + opt = by_key[(key, candidate_method)] + paired = [] + for a, b in zip(base["per_run"], opt["per_run"]): + paired.append({"run": a["run"], "baseline_median_us": a["median_us"], + "candidate_median_us": b["median_us"], + "baseline_over_candidate": a["median_us"] / b["median_us"], + "time_reduction_percent": 100 * (1 - b["median_us"] / a["median_us"])}) + gains = [p["time_reduction_percent"] for p in paired] + comparison = dict(zip(KEYS, key), candidate=prefix, method=candidate_method, operation=operation, per_run=paired, + minimum_time_reduction_percent=min(gains), maximum_time_reduction_percent=max(gains), + stable_candidate_every_run_at_least_5_percent=min(gains) >= 5, + every_run_faster=min(gains) > 0, any_run_slowdown=min(gains) < 0, + every_run_slowdown=max(gains) < 0, any_run_regression_over_3_percent=min(gains) < -3, + every_run_regression_over_3_percent=max(gains) < -3) + comparisons.append(comparison) + opt["comparison_to_baseline"] = comparison + counters = {name: sum(c[name] for c in comparisons) for name in ( + "stable_candidate_every_run_at_least_5_percent", "every_run_faster", "any_run_slowdown", + "every_run_slowdown", "any_run_regression_over_3_percent", "every_run_regression_over_3_percent")} + return { + "status": "ANALYSIS_COMPLETE", "correctness_status": "NOT_INFERRED_FROM_TIMING_CSV", + "inputs": [{k: v for k, v in run.items() if k != "buckets"} for run in runs], + "methodology": { + "metric": "kernel_us is the backend device event interval divided by repeats, then median across groups per run. The backend API is identified in the run manifest, not inferred from CSV. Split methods include transform and quantization kernel launches.", + "excluded_from_interval": ["allocation", "H2D/D2H", "warmup", "validation"], + "scope_limits": "Not host end-to-end or isolated single-kernel timing. Event interval may include device idle gaps between host launches; same seeded read-only input is reused, with warm-cache effects.", + "logical_GBs": "Reported logical tensor-I/O estimate only; not measured physical memory bandwidth.", + "comparison": "Each non-baseline implementation paired with baseline by dtype/full shape/scale/operation within each run. Time reduction (%) = 100*(1-candidate_median/baseline_median). Candidate only if >=5% in EVERY run. Different method matrices cannot be mixed.", + "variation": "Within-run population stddev/CV describe group means. Across-run stddev/CV describe run medians. Neither is a confidence interval; repeats are not counted as new independent test cases.", + "missing_evidence": "CSV does not encode hardware identity, compiler/driver versions, clock/temperature, source revision, process independence or final validation status. Verify accompanying run manifests before making platform or correctness claims.", + }, + "counts": {"runs": len(runs), "total_raw_rows": sum(r["raw_rows"] for r in runs), + "distinct_shape_dtype_scale_cases": runs[0]["shape_dtype_scale_cases"], + "distinct_method_cases": len(methods), "paired_candidate_operation_cases": len(comparisons), **counters}, + "counts_by_candidate": {candidate: { + "paired_operation_cases": sum(c["candidate"] == candidate for c in comparisons), + **{name: sum(c[name] for c in comparisons if c["candidate"] == candidate) for name in counters}, + } for candidate in sorted({c["candidate"] for c in comparisons})}, + "all_method_statistics": methods, "all_comparisons_including_slowdowns": comparisons, + } + + +def write_outputs(report, output): + output.mkdir(parents=True, exist_ok=False) + (output / "analysis.json").write_text(json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8") + rows = [] + for method in report["all_method_statistics"]: + row = {key: method[key] for key in KEYS + ("method",)} + row.update(method["conditions"]) + for value in method["per_run"]: + prefix = f"run{value['run']}_" + row.update({prefix + key: item for key, item in value.items() if key not in ("run", "samples")}) + row.update({"across_run_medians_" + key: value for key, value in method["across_run_medians"].items()}) + comparison = method.get("comparison_to_baseline") + row["comparison_role"] = "candidate_vs_matching_baseline" if comparison else "baseline" + for run in range(1, report["counts"]["runs"] + 1): + for name in ("baseline_over_candidate", "time_reduction_percent"): + row[f"run{run}_{name}"] = comparison["per_run"][run - 1][name] if comparison else "" + for key in ("minimum_time_reduction_percent", "maximum_time_reduction_percent", "stable_candidate_every_run_at_least_5_percent", + "every_run_faster", "any_run_slowdown", "every_run_slowdown", "any_run_regression_over_3_percent", "every_run_regression_over_3_percent"): + row[key] = comparison[key] if comparison else "" + rows.append(row) + with (output / "method_summary.csv").open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("csv", nargs="+", type=Path, help="按时间顺序传入独立运行的 CSV") + parser.add_argument("--output", required=True, type=Path, help="新的输出目录，禁止覆盖已有结果") + args = parser.parse_args() + try: + if args.output.exists(): + raise ValueError("output already exists; select a fresh directory") + report = analyze(args.csv) + report["analyzer_sha256"] = digest(Path(__file__)) + write_outputs(report, args.output) + print(json.dumps({"status": report["status"], "counts": report["counts"], "output": str(args.output)}, ensure_ascii=False)) + return 0 + except (OSError, ValueError, TypeError, KeyError) as error: + print("ANALYSIS_FAILED:", error, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/hadamard_api.cu b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/hadamard_api.cu new file mode 100644 index 00000000..c4191e64 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/hadamard_api.cu @@ -0,0 +1,349 @@ +#include "hadamard_api.h" + +#include +#include + +namespace hadamard::iluvatar { +namespace { + +template __device__ float read_value(T value); +template<> __device__ float read_value(__half value) { return __half2float(value); } +template<> __device__ float read_value(__nv_bfloat16 value) { + return __bfloat162float(value); +} +template __device__ T store_value(float value); +template<> __device__ __half store_value(float value) { return __float2half_rn(value); } +template<> __device__ __nv_bfloat16 store_value(float value) { + return __float2bfloat16_rn(value); +} + +__device__ std::uint8_t quantize_pair(const float* values, int pair, int n, + float row_scale) { + std::uint8_t byte = 0; + for (int k = 0; k < 2 && pair * 2 + k < n; ++k) { + const float x = values[pair * 2 + k] / row_scale; + // 明确实现最近偶数舍入，不依赖编译器默认取整模式。 + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + byte |= static_cast((q & 15) << (4 * k)); + } + return byte; +} + +// 保留首轮实机通过的算法：一元素一线程、每层两次屏障、线程 0 串行归约。 +// 与 optimized 独立保留蝶形计算，供同一运行环境下公平比较。 +template +__global__ void baseline_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float row_scale; + const int i = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + if (i < n) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + float next = 0; + if (i < n) { + const float a = values[i], b = values[i ^ stride]; + next = (i & stride) ? b - a : a + b; + } + __syncthreads(); + if (i < n) values[i] = next; + __syncthreads(); + } + if (i < n) values[i] = read_value(store_value(values[i] * scale)); + __syncthreads(); + } + if constexpr (!Quantize) { + if (i < n) output[offset + i] = store_value(values[i]); + } else { + if (i == 0) { + float magnitude = 0; + for (int j = 0; j < n; ++j) + magnitude = fmaxf(magnitude, fabsf(values[j])); + row_scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + if (i < bytes) packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + // 多行复用同一 block 时，防止下行加载覆盖仍在打包/写出的上一行。 + if (rows - row > gridDim.x) __syncthreads(); + } +} + +// 每个线程独占一个蝶形的两个输入/输出，因此单层没有线程间读写冲突， +// 只需在本层结束后同步。此算法不假定 warp 宽度，不使用 warp 级隐式同步。 +template +__global__ void optimized_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float maxima[128]; + __shared__ float row_scale; + const int tid = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + for (int i = tid; i < n; i += blockDim.x) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + if (tid < n / 2) { + const int low = ((tid & ~(stride - 1)) << 1) | (tid & (stride - 1)); + const float a = values[low], b = values[low + stride]; + values[low] = a + b; + values[low + stride] = a - b; + } + __syncthreads(); + } + } + if constexpr (!Quantize) { + for (int i = tid; i < n; i += blockDim.x) + output[offset + i] = store_value(values[i] * scale); + } else { + float magnitude = 0; + for (int i = tid; i < n; i += blockDim.x) { + float value = values[i]; + if constexpr (Transform) { + value = read_value(store_value(value * scale)); + values[i] = value; + } + magnitude = fmaxf(magnitude, fabsf(value)); + } + maxima[tid] = magnitude; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 1; stride /= 2) { + if (tid < stride) maxima[tid] = fmaxf(maxima[tid], maxima[tid + stride]); + __syncthreads(); + } + if (tid == 0) { + // 最后两个值只由线程 0 消费，与 scale 写入合并，少一次屏障。 + const float maximum = fmaxf(maxima[0], maxima[1]); + row_scale = maximum == 0 ? 1.0f : maximum / 7.0f; + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + for (int i = tid; i < bytes; i += blockDim.x) + packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + if (rows - row > gridDim.x) __syncthreads(); + } +} + +#if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + +// 仅在天数 COREX 构建中启用。legacy shuffle 的 width=64 已由实机独立探针验证。 +// 这里不使用 NVIDIA 的 32 位 active mask，也不将 32-lane 测试当作正确性证据。 +__device__ int warp64_quantized_nibble(float value, float scale) { + const float x = value / scale; + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + return q & 15; +} + +template +__global__ void warp64_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float scale) { + constexpr int width = 64; + constexpr int rows_per_block = 4; + constexpr int registers = N > width ? N / width : 1; + const int lane = threadIdx.x % width; + const int warp = threadIdx.x / width; + const std::size_t first_row = static_cast(blockIdx.x) * rows_per_block + warp; + const std::size_t row_stride = static_cast(gridDim.x) * rows_per_block; + // 同一 warp 的 64 个 lane 具有相同 row 与循环次数。尾行只跳过完整 warp， + // 所有参与行均有完整的 64 个 lane 执行每次 shuffle。 + for (std::size_t row = first_row; row < rows; row += row_stride) { + const std::size_t offset = row * N; + float values[registers]; + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + values[r] = i < N ? read_value(input[offset + i]) : 0.0f; + } + if constexpr (Transform) { + #pragma unroll + for (int stride = 1; stride < (N < width ? N : width); stride *= 2) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const float current = values[r]; + const float peer = __shfl_xor(current, stride, width); + values[r] = (lane & stride) ? peer - current : current + peer; + } + } + // stride=64/128 的搭档位于同一 lane 的其他寄存器中， + // 顺序与标量共享内存 FWHT 的从低位到高位阶段一致。 + if constexpr (N >= 128) { + const float a = values[0], b = values[1]; + values[0] = a + b; + values[1] = a - b; + } + if constexpr (N == 256) { + const float c = values[2], d = values[3]; + values[2] = c + d; + values[3] = c - d; + const float a = values[0], b = values[2]; + const float e = values[1], f = values[3]; + values[0] = a + b; + values[2] = a - b; + values[1] = e + f; + values[3] = e - f; + } + } + if constexpr (!Quantize) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + if (i < N) output[offset + i] = store_value(values[r] * scale); + } + } else { + float magnitude = 0.0f; + #pragma unroll + for (int r = 0; r < registers; ++r) { + if constexpr (Transform) values[r] = read_value(store_value(values[r] * scale)); + magnitude = fmaxf(magnitude, fabsf(values[r])); + } + #pragma unroll + for (int stride = width / 2; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor(magnitude, stride, width)); + const float row_scale = magnitude == 0.0f ? 1.0f : magnitude / 7.0f; + if (lane == 0) scales[row] = row_scale; + #pragma unroll + for (int r = 0; r < registers; ++r) { + // shuffle 在分支之前：奇数 lane 同样必须提供其相邻元素。 + const float peer = __shfl_xor(values[r], 1, width); + const int i = lane + r * width; + if ((lane & 1) == 0 && i < N) { + const int low = warp64_quantized_nibble(values[r], row_scale); + const int high = i + 1 < N ? warp64_quantized_nibble(peer, row_scale) : 0; + packed[row * ((N + 1) / 2) + i / 2] + = static_cast(low | (high << 4)); + } + } + } + } +} + +template +cudaError_t launch_warp64(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, cudaStream_t stream) { + // rows <= SIZE_MAX/4 已由 launch 校验，因此 (rows+3)/4 不会溢出。 + const std::size_t requested = (rows + 3) / 4; + const unsigned int blocks = static_cast(requested < 65535 ? requested : 65535); + #define LAUNCH_WARP64(N) case N: \ + warp64_kernel<<>>( \ + input, output, packed, scales, rows, scale); \ + break + switch (n) { + LAUNCH_WARP64(1); + LAUNCH_WARP64(2); + LAUNCH_WARP64(4); + LAUNCH_WARP64(8); + LAUNCH_WARP64(16); + LAUNCH_WARP64(32); + LAUNCH_WARP64(64); + LAUNCH_WARP64(128); + LAUNCH_WARP64(256); + default: return cudaErrorInvalidValue; + } + #undef LAUNCH_WARP64 + return cudaGetLastError(); +} + +#endif // HADAMARD_ILUVATAR_WARP64 + +bool valid_range(const void* pointer, std::size_t bytes, std::size_t alignment) { + const auto address = reinterpret_cast(pointer); + return pointer != nullptr && address % alignment == 0 + && bytes <= std::numeric_limits::max() - address; +} + +bool overlaps(const void* left, std::size_t left_size, + const void* right, std::size_t right_size) { + // valid_range 已确保两个区间端点的加法不会溢出。 + const auto a = reinterpret_cast(left); + const auto b = reinterpret_cast(right); + return a < b + right_size && b < a + left_size; +} + +template +cudaError_t launch(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, cudaStream_t stream, Method method) { + if (n < 1 || n > 256 || (n & (n - 1)) != 0 + || !std::isfinite(scale) || scale <= 0 + || (method != Method::Baseline && method != Method::Optimized && method != Method::Warp64)) + return cudaErrorInvalidValue; + if (rows == 0) return cudaSuccess; + + constexpr auto maximum = std::numeric_limits::max(); + const std::size_t row_bytes = static_cast(n) * sizeof(T); + if (rows > maximum / row_bytes || rows > maximum / sizeof(float)) + return cudaErrorInvalidValue; + const std::size_t input_bytes = rows * row_bytes; + if (!valid_range(input, input_bytes, alignof(T))) return cudaErrorInvalidValue; + if constexpr (Quantize) { + const std::size_t packed_bytes = rows * static_cast((n + 1) / 2); + const std::size_t scale_bytes = rows * sizeof(float); + if (!valid_range(packed, packed_bytes, alignof(std::uint8_t)) + || !valid_range(scales, scale_bytes, alignof(float)) + || overlaps(input, input_bytes, packed, packed_bytes) + || overlaps(input, input_bytes, scales, scale_bytes) + || overlaps(packed, packed_bytes, scales, scale_bytes)) return cudaErrorInvalidValue; + } else { + if (!valid_range(output, input_bytes, alignof(T)) + || (input != output && overlaps(input, input_bytes, output, input_bytes))) + return cudaErrorInvalidValue; + } + + if (method == Method::Warp64) { + #if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + return launch_warp64(input, output, packed, scales, rows, n, scale, stream); + #else + return cudaErrorNotSupported; + #endif + } + + // 保守上限兼容不同设备的 grid.x 限制，超过上限由 block 顺序处理多行。 + const unsigned int blocks = static_cast(rows < 65535 ? rows : 65535); + if (method == Method::Baseline) { + const int threads = n < 64 ? 64 : n; + baseline_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } else { + const int threads = n / 2 < 64 ? 64 : n / 2; + optimized_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } + return cudaGetLastError(); +} + +} // namespace + +#define DEFINE_TYPED_API(T) \ +cudaError_t transform(const T* input, T* output, std::size_t rows, int n, float scale, \ + cudaStream_t stream, Method method) { \ + return launch(input, output, nullptr, nullptr, rows, n, scale, stream, method); \ +} \ +cudaError_t quantize_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, cudaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, 1.0f, stream, method); \ +} \ +cudaError_t transform_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, float scale, cudaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, scale, stream, method); \ +} + +DEFINE_TYPED_API(__half) +DEFINE_TYPED_API(__nv_bfloat16) +#undef DEFINE_TYPED_API + +} // namespace hadamard::iluvatar diff --git a/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/hadamard_api.h b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/hadamard_api.h new file mode 100644 index 00000000..9beb6db9 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/hadamard_api.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace hadamard::iluvatar { + +enum class Method { Baseline, Optimized, Warp64 }; + +// Warp64 为天数 64-lane warp 专用候选，须使用 +// -DHADAMARD_ILUVATAR_WARP64 且由 COREX (__ILUVATAR__) 编译；其他构建的 +// 非空操作返回 cudaErrorNotSupported。调用方必须预先确认当前设备 warpSize=64； +// API 不在发射路径中查询设备属性。rows=0 仍遵循下方的无操作约定。 + +// 连续设备内存：[rows, n]，n 为 1..256 的二次幂。scale 必须有限且为正数。 +// 支持 FP16/BF16 存储，内部 FP32；输入应有限，变换后的值应在输出类型范围内。 +// 所有操作仅在调用方的 stream 上发射，不分配内存、不复制、不等待。 +// rows=0 时只检查 n、scale、method，允许空指针，不发射内核。 +// 输入和变换输出只要求 2 字节对齐；scales 要求 float 对齐。 +// transform 允许 input==output 原位变换，其余重叠均返回 cudaErrorInvalidValue。 +// 不检查分配容量/设备归属；调用方须提供当前设备有效且足够大的设备缓冲区。 +// 返回参数/发射错误；异步执行错误由调用方同步 stream 时检查。 +cudaError_t transform(const __half* input, __half* output, std::size_t rows, + int n, float scale, cudaStream_t stream, + Method method = Method::Optimized); +cudaError_t transform(const __nv_bfloat16* input, __nv_bfloat16* output, + std::size_t rows, int n, float scale, cudaStream_t stream, + Method method = Method::Optimized); + +// 每行 ceil(n/2) 字节；偶数元素在低 4 位，奇数在高 4 位，n=1 的高位为 0。 +// q=clamp(round_to_nearest_even(x/s), -7, 7)，s=max(abs(x))/7；全零行 s=1。 +// scales 为 rows 个 float。所有输入/输出缓冲区必须互不重叠。 +cudaError_t quantize_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, cudaStream_t stream, + Method method = Method::Optimized); +cudaError_t quantize_int4(const __nv_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, + cudaStream_t stream, Method method = Method::Optimized); + +// 融合路径先以最近偶数规则舍入到公开的 FP16/BF16 输出类型，再计算 INT4， +// 因而与 transform 后调用 quantize_int4 的字节和 scales 语义一致。 +cudaError_t transform_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, + cudaStream_t stream, Method method = Method::Optimized); +cudaError_t transform_int4(const __nv_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale, + cudaStream_t stream, Method method = Method::Optimized); + +} // namespace hadamard::iluvatar diff --git a/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/run_platform.py b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/run_platform.py new file mode 100644 index 00000000..61d044c7 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/run_platform.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""在真实 COREX 设备构建、核查并测量；不下载/安装或替换驱动与框架。""" +import argparse +import csv +import hashlib +import json +import os +from pathlib import Path +import statistics +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[2] +PLATFORM = Path(__file__).resolve().parent + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def capture(command): + result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + return {"command": command, "returncode": result.returncode, "output": result.stdout} + + +def run(command, log, stages, expected=0): + print("RUN", " ".join(map(str, command)), flush=True) + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + completed = subprocess.run(list(map(str, command)), cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False) + stages.append({"command": list(map(str, command)), "log": log.name, + "returncode": completed.returncode, "expected_returncode": expected, + "wall_seconds": time.monotonic() - started, "sha256": sha256(log)}) + print("EXIT", completed.returncode, "LOG", log, flush=True) + if completed.returncode != expected: + print(log.read_text(encoding="utf-8", errors="replace")[-16000:], file=sys.stderr) + raise RuntimeError("stage failed: " + log.name) + + +def summarize_benchmark(path): + samples = {} + with path.open(newline="", encoding="utf-8") as stream: + for row in csv.DictReader(stream): + key = tuple(row[k] for k in ("dtype", "batch", "seq", "heads", "dim", "method")) + samples.setdefault(key, []).append(float(row["kernel_us"])) + medians = {} + summary = [] + for key, values in samples.items(): + medians[key] = statistics.median(values) + summary.append(dict(zip(("dtype", "batch", "seq", "heads", "dim", "method"), key), + samples=values, median_us=statistics.median(values), + minimum_us=min(values), maximum_us=max(values))) + comparisons = [] + for key, value in medians.items(): + if not key[-1].startswith("baseline_"): + continue + peer = key[:-1] + (key[-1].replace("baseline_", "optimized_"),) + comparisons.append({"shape_dtype": key[:-1], "operation": key[-1][9:], + "baseline_median_us": value, "optimized_median_us": medians[peer], + "baseline_over_optimized": value / medians[peer]}) + warp_peer = key[:-1] + (key[-1].replace("baseline_", "warp64_"),) + if warp_peer in medians: + comparisons.append({"shape_dtype": key[:-1], "operation": key[-1][9:], "candidate": "warp64", + "baseline_median_us": value, "optimized_median_us": medians[peer], + "warp64_median_us": medians[warp_peer], "baseline_over_warp64": value / medians[warp_peer], + "optimized_over_warp64": medians[peer] / medians[warp_peer]}) + return {"metric": "CUDA-compatible events; warmup excluded; no allocation/copy in interval", + "working_set": "same seeded read-only input reused; warm-cache timing", + "logical_GBs_note": "logical tensor I/O estimate, not measured physical memory bandwidth", + "statistics": summary, "comparisons_including_slowdowns": comparisons} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--corex-root", type=Path, default=Path("/usr/local/corex")) + parser.add_argument("--compiler", type=Path) + parser.add_argument("--output", type=Path, help="必须是新的结果目录，防止覆盖既有证据") + parser.add_argument("--quick", action="store_true", help="仅运行快速调试矩阵，不代表完整验收") + parser.add_argument("--no-benchmark", action="store_true") + parser.add_argument("--warp64", action="store_true", help="显式启用仅供真实 warp64 设备验证的 COREX 专用路径") + parser.add_argument("--repeats", type=int, default=100) + parser.add_argument("--groups", type=int, default=5) + args = parser.parse_args() + if args.repeats < 1 or args.repeats > 10000 or args.groups < 1 or args.groups > 10000: + parser.error("repeats/groups must be between 1 and 10000") + compiler = args.compiler or args.corex_root / "bin/clang++" + if not compiler.is_file(): + parser.error("COREX compiler is missing: " + str(compiler)) + destination = (args.output or ROOT / "results/iluvatar" / time.strftime("%Y%m%d-%H%M%S", time.gmtime())).resolve() + if destination.exists(): + parser.error("output directory already exists; select a fresh path") + destination.mkdir(parents=True) + binary = destination / "validate_and_benchmark" + sources = [PLATFORM / "hadamard_api.h", PLATFORM / "hadamard_api.cu", + PLATFORM / "validate_and_benchmark.cu", PLATFORM / "run_platform.py", ROOT / "include/reference.hpp"] + report = {"status": "RUNNING", "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "quick": args.quick, "warp64_enabled": args.warp64, + "source_sha256": {str(p.relative_to(ROOT)): sha256(p) for p in sources}, + "git_head": capture(["git", "rev-parse", "HEAD"]), + "git_status": capture(["git", "status", "--short"]), + "compiler": capture([str(compiler), "--version"]), + "python_version": sys.version, "stages": []} + manifest = destination / "run_summary.json" + manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + try: + command = [compiler, "-x", "ivcore", "-std=c++17", "-O2", "-I" + str(ROOT / "include"), + PLATFORM / "hadamard_api.cu", PLATFORM / "validate_and_benchmark.cu", + "-L" + str(args.corex_root / "lib"), "-Wl,-rpath," + str(args.corex_root / "lib"), + "-lcudart", "-o", binary] + if args.warp64: + command.insert(6, "-DHADAMARD_ILUVATAR_WARP64") + run(command, destination / "build.log", report["stages"]) + report["binary_sha256"] = sha256(binary) + # CLI 验证在创建设备上下文之前执行。错误输入必须明确返回 2。 + invalid = [[], ["--validate", "--dim", "0"], ["--validate", "--dim", "3"], + ["--validate", "--dim", "512"], ["--validate", "--dtype", "fp32"], + ["--validate", "--batch", "-1"], ["--validate", "--seq", "0"], + ["--validate", "--heads", "x"], ["--validate", "--batch", "18446744073709551615", "--seq", "2"], + ["--validate", "--heads", "999999999999999999999999999999"], + ["--validate", "--dim"], ["--validate", "--unknown", "1"], + ["--benchmark", "--repeats", "0"], ["--benchmark", "--groups", "10001"]] + for i, case in enumerate(invalid): + run([binary] + case, destination / ("invalid_%02d.log" % i), report["stages"], expected=2) + report["cli_rejection_cases"] = len(invalid) + command = [binary, "--validate", "--json", destination / "validation.json"] + if args.quick: + command.append("--quick") + run(command, destination / "validation.log", report["stages"]) + report["validation"] = json.loads((destination / "validation.json").read_text(encoding="utf-8")) + if report["validation"]["status"] != "PASS" or (not args.quick and not report["validation"]["full_matrix"]): + raise RuntimeError("validation JSON does not confirm requested matrix") + if report["validation"]["warp64_enabled"] != args.warp64: + raise RuntimeError("validation JSON does not match requested Warp64 build") + if not args.no_benchmark: + command = [binary, "--benchmark", "--csv", destination / "benchmark.csv", + "--groups", str(args.groups), "--repeats", str(args.repeats)] + if args.quick: + command += ["--batch", "1", "--seq", "17", "--heads", "1", "--dim", "128"] + run(command, destination / "benchmark.log", report["stages"]) + report["benchmark"] = summarize_benchmark(destination / "benchmark.csv") + report["status"] = "PASS" + except (OSError, RuntimeError, ValueError, KeyError) as error: + report["status"] = "FAIL" + report["error"] = str(error) + print("FAIL", error, file=sys.stderr) + finally: + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + report["artifacts"] = {p.name: sha256(p) for p in destination.iterdir() if p.is_file() and p != manifest} + manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "quick": args.quick, "summary": str(manifest)}, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/shared_baseline.cu b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/shared_baseline.cu new file mode 100644 index 00000000..e6763cbc --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/shared_baseline.cu @@ -0,0 +1,208 @@ +#include +#include +#include +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include + +#define CUDA_CHECK(call) do { const cudaError_t e = (call); if (e != cudaSuccess) \ + throw std::runtime_error(std::string(#call) + ": " + cudaGetErrorString(e)); } while (0) + +template __host__ __device__ float read_value(T x); +template<> __host__ __device__ float read_value(__half x) { return __half2float(x); } +template<> __host__ __device__ float read_value(__nv_bfloat16 x) { return __bfloat162float(x); } +template __host__ __device__ T store_value(float x); +template<> __host__ __device__ __half store_value(float x) { return __float2half_rn(x); } +template<> __host__ __device__ __nv_bfloat16 store_value(float x) { return __float2bfloat16_rn(x); } + +// 不依赖 warp 宽度；每层先读到寄存器，再同步写回共享内存。 +template +__global__ void shared_fwht(const T* input, T* output, unsigned char* packed, + float* scales, int n, float scale) { + __shared__ float values[256]; + __shared__ float row_scale; + const int i = threadIdx.x; + const std::size_t offset = static_cast(blockIdx.x) * n; + if (i < n) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + float next = 0; + if (i < n) { + const float a = values[i], b = values[i ^ stride]; + next = (i & stride) ? b - a : a + b; + } + __syncthreads(); + if (i < n) values[i] = next; + __syncthreads(); + } + // 先舍入到公开输出精度，融合量化必须与分步路径保持相同语义。 + if (i < n) values[i] = read_value(store_value(values[i] * scale)); + __syncthreads(); + } + if constexpr (!Quantize) { + if (i < n) output[offset + i] = store_value(values[i]); + } else { + if (i == 0) { + float magnitude = 0; + for (int j = 0; j < n; ++j) magnitude = fmaxf(magnitude, fabsf(values[j])); + row_scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + scales[blockIdx.x] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + if (i < bytes) { + unsigned char byte = 0; + for (int k = 0; k < 2 && 2 * i + k < n; ++k) { + const float x = values[2 * i + k] / row_scale; + const float lower = floorf(x), fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + byte |= static_cast((q & 15) << (4 * k)); + } + packed[static_cast(blockIdx.x) * bytes + i] = byte; + } + } +} + +template T* allocate(std::size_t count) { + T* ptr = nullptr; + CUDA_CHECK(cudaMalloc(reinterpret_cast(&ptr), count * sizeof(T))); + return ptr; +} + +struct Summary { int cases = 0; std::size_t elements = 0; double max_error = 0; }; + +template +void validate_case(cudaStream_t stream, int rows, int n, float scale, int pattern, + const char* dtype, Summary& summary) { + const std::size_t size = static_cast(rows) * n; + const std::size_t bytes = static_cast(rows) * ((n + 1) / 2); + std::vector input(size), output(size); + std::vector rounded(size); + std::uint32_t state = 0x735123abu; + for (std::size_t i = 0; i < size; ++i) { + state = state * 1664525u + 1013904223u; + float value = (static_cast(state >> 8) / 16777216.0f - 0.5f) * 0.5f; + if (pattern == 0) value = 0; + if (pattern == 1) value = i % n == (i / n) % n ? 0.625f : 0; + input[i] = store_value(value); + rounded[i] = read_value(input[i]); + } + std::vector split(bytes), fused(bytes); + std::vector split_scales(rows), fused_scales(rows); + T* dx = allocate(size); + T* dy = allocate(size); + auto* dq = allocate(bytes); + auto* df = allocate(bytes); + float* ds = allocate(rows); + float* dfs = allocate(rows); + CUDA_CHECK(cudaMemcpyAsync(dx, input.data(), size * sizeof(T), cudaMemcpyHostToDevice, stream)); + const int threads = std::max(64, n); + shared_fwht<<>>(dx, dy, nullptr, nullptr, n, scale); + CUDA_CHECK(cudaGetLastError()); + shared_fwht<<>>(dy, nullptr, dq, ds, n, 1.0f); + CUDA_CHECK(cudaGetLastError()); + shared_fwht<<>>(dx, nullptr, df, dfs, n, scale); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaMemcpyAsync(output.data(), dy, size * sizeof(T), cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaMemcpyAsync(split.data(), dq, bytes, cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaMemcpyAsync(fused.data(), df, bytes, cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaMemcpyAsync(split_scales.data(), ds, rows * sizeof(float), cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaMemcpyAsync(fused_scales.data(), dfs, rows * sizeof(float), cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaStreamSynchronize(stream)); + CUDA_CHECK(cudaFree(dx)); CUDA_CHECK(cudaFree(dy)); CUDA_CHECK(cudaFree(dq)); + CUDA_CHECK(cudaFree(df)); CUDA_CHECK(cudaFree(ds)); CUDA_CHECK(cudaFree(dfs)); + + const double tolerance = std::string(dtype) == "fp16" ? 1e-2 : 5e-2; + const std::string context = std::string(dtype) + " rows=" + std::to_string(rows) + + " n=" + std::to_string(n) + " pattern=" + std::to_string(pattern) + + " scale=" + std::to_string(scale); + std::vector actual(size), expected = rounded; + hadamard::fwht(expected.data(), rows, n, scale); + for (std::size_t i = 0; i < size; ++i) { + actual[i] = read_value(output[i]); + const double error = std::abs(static_cast(actual[i]) - expected[i]); + if (!std::isfinite(actual[i]) || !(error < tolerance)) + throw std::runtime_error("CPU FWHT mismatch: " + context + " index=" + std::to_string(i)); + summary.max_error = std::max(summary.max_error, error); + } + for (int row : {0, rows / 2, rows - 1}) { + const std::vector sample(rounded.begin() + row * n, rounded.begin() + (row + 1) * n); + const auto dense = hadamard::dense_reference(sample, n, scale); + for (int i = 0; i < n; ++i) { + const double error = std::abs(actual[row * n + i] - dense[i]); + if (!(error < tolerance)) throw std::runtime_error("Dense oracle mismatch: " + context); + summary.max_error = std::max(summary.max_error, error); + } + } + const auto cpu_quant = hadamard::quantize_int4(actual, n); + if (split != cpu_quant.packed || fused != split || split_scales != cpu_quant.scales + || fused_scales != split_scales) + throw std::runtime_error("INT4 bytes/scales mismatch: " + context); + ++summary.cases; + summary.elements += size; +} + +// 独立验证正负半整数：scale=1，低位/高位和 N=1 的空高位另由全套用例覆盖。 +template void validate_ties(cudaStream_t stream) { + const std::vector values{7, -7, 0.5f, 1.5f, 2.5f, -0.5f, -1.5f, -2.5f}; + std::vector input; + for (float value : values) input.push_back(store_value(value)); + T* dx = allocate(8); + auto* dq = allocate(4); + float* ds = allocate(1); + std::vector packed(4); + float scale = 0; + CUDA_CHECK(cudaMemcpyAsync(dx, input.data(), 8 * sizeof(T), cudaMemcpyHostToDevice, stream)); + shared_fwht<<<1, 64, 0, stream>>>(dx, nullptr, dq, ds, 8, 1.0f); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaMemcpyAsync(packed.data(), dq, 4, cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaMemcpyAsync(&scale, ds, sizeof(float), cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaStreamSynchronize(stream)); + CUDA_CHECK(cudaFree(dx)); CUDA_CHECK(cudaFree(dq)); CUDA_CHECK(cudaFree(ds)); + if (scale != 1.0f || packed != hadamard::quantize_int4(values, 8).packed) + throw std::runtime_error("Positive/negative ties-to-even quantization failed"); +} + +template void validate_dtype(cudaStream_t stream, const char* dtype) { + Summary summary; + for (int n = 1; n <= 256; n *= 2) + for (int rows : {1, 17, 257}) + for (float scale : {1.0f, 1.0f / std::sqrt(static_cast(n))}) + for (int pattern = 0; pattern < 3; ++pattern) + validate_case(stream, rows, n, scale, pattern, dtype, summary); + validate_ties(stream); + std::cout << "PASS dtype=" << dtype << " cases=" << summary.cases + << " elements=" << summary.elements << " max_abs_error=" + << std::setprecision(12) << summary.max_error << " ties_even=PASS\n"; +} + +int main() { + try { + cudaDeviceProp prop{}; + CUDA_CHECK(cudaGetDeviceProperties(&prop, 0)); + CUDA_CHECK(cudaSetDevice(0)); + int runtime = 0, driver = 0; + CUDA_CHECK(cudaRuntimeGetVersion(&runtime)); + CUDA_CHECK(cudaDriverGetVersion(&driver)); + std::printf("VALIDATION_ONLY device=%s warp=%d runtime=%d driver=%d\n", + prop.name, prop.warpSize, runtime, driver); + cudaStream_t stream; + CUDA_CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + validate_dtype<__half>(stream, "fp16"); + validate_dtype<__nv_bfloat16>(stream, "bf16"); + CUDA_CHECK(cudaStreamDestroy(stream)); + std::puts("PASS shared-memory FWHT, CPU oracle, split/fused INT4; no performance claim"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL %s\n", error.what()); + return 1; + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/validate_and_benchmark.cu b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/validate_and_benchmark.cu new file mode 100644 index 00000000..686b7b9b --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/iluvatar/validate_and_benchmark.cu @@ -0,0 +1,519 @@ +#include "hadamard_api.h" +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace api = hadamard::iluvatar; +#define CHECK(call) do { cudaError_t e_ = (call); if (e_ != cudaSuccess) \ + throw std::runtime_error(std::string(#call) + ": " + cudaGetErrorString(e_)); } while (0) + +template float read(T); +template<> float read(__half x) { return __half2float(x); } +template<> float read(__nv_bfloat16 x) { return __bfloat162float(x); } +template T rounded(float); +template<> __half rounded(float x) { return __float2half_rn(x); } +template<> __nv_bfloat16 rounded(float x) { return __float2bfloat16_rn(x); } + +std::vector active_methods() { + std::vector methods{api::Method::Baseline, api::Method::Optimized}; +#if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + methods.push_back(api::Method::Warp64); +#endif + return methods; +} + +const char* method_name(api::Method method) { + if (method == api::Method::Baseline) return "baseline"; + if (method == api::Method::Optimized) return "optimized"; + return "warp64"; +} + +// 每个区间都有前后哨兵。17 个元素的偏移同时覆盖仅 2 字节对齐的 FP16/BF16 指针。 +template class Guarded { + static constexpr std::size_t guard = 17; + T* raw_ = nullptr; + std::size_t count_; + std::vector initial_; + std::string name_; +public: + explicit Guarded(std::size_t count, cudaStream_t stream, const char* name = "unnamed") + : count_(count), initial_((count + 2 * guard) * sizeof(T), 0xa5), name_(name) { + CHECK(cudaMalloc(reinterpret_cast(&raw_), initial_.size())); + // 初始化、上传、kernel、回读使用同一个非阻塞 stream。默认 stream 上的 + // pageable H2D 即使通过 cudaMemcpy 调用，也不保证返回时最终 DMA 已结束。 + // 按字节 memset 避免异步初始化读取随后被 upload 更新的主机 shadow。 + CHECK(cudaMemsetAsync(raw_, 0xa5, initial_.size(), stream)); + } + ~Guarded() { if (raw_) cudaFree(raw_); } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + T* data() { return raw_ + guard; } + void upload(const std::vector& values, cudaStream_t stream) { + if (values.size() != count_) throw std::runtime_error("upload size mismatch"); + std::memcpy(initial_.data() + guard * sizeof(T), values.data(), count_ * sizeof(T)); + CHECK(cudaMemcpyAsync(raw_, initial_.data(), initial_.size(), cudaMemcpyHostToDevice, stream)); + } + std::vector download(cudaStream_t stream, bool unchanged = false, const char* phase = "readback") { + std::vector bytes(initial_.size()); + CHECK(cudaMemcpyAsync(bytes.data(), raw_, bytes.size(), cudaMemcpyDeviceToHost, stream)); + CHECK(cudaStreamSynchronize(stream)); + const std::size_t prefix = guard * sizeof(T), end = prefix + count_ * sizeof(T); + for (std::size_t i = 0; i < bytes.size(); ++i) { + if (bytes[i] == initial_[i] || (!unchanged && i >= prefix && i < end)) continue; + const bool is_guard = i < prefix || i >= end; + throw std::runtime_error(std::string(is_guard ? "device buffer guard overwritten" : "read-only input modified") + + " buffer=" + name_ + " phase=" + phase + " region=" + (i < prefix ? "prefix" : (i >= end ? "suffix" : "payload")) + + " byte_from_payload=" + std::to_string(static_cast(i) - static_cast(prefix)) + + " expected=" + std::to_string(static_cast(initial_[i])) + + " actual=" + std::to_string(static_cast(bytes[i])) + + " elements=" + std::to_string(count_) + " element_bytes=" + std::to_string(sizeof(T))); + } + std::vector result(count_); + std::memcpy(result.data(), bytes.data() + prefix, count_ * sizeof(T)); + return result; + } +}; + +struct Options { + bool validate = false, benchmark = false, custom_shape = false, quick = false; + std::size_t batch = 1, seq = 1, heads = 1; + int dim = 128, repeats = 100, groups = 5; + std::string dtype = "both", csv = "iluvatar_benchmark.csv", json = "iluvatar_validation.json"; +}; + +std::size_t positive(const std::string& text, const char* name) { + if (text.empty() || text.find_first_not_of("0123456789") != std::string::npos) + throw std::invalid_argument(std::string(name) + " must be a positive integer"); + std::size_t used = 0; + const auto value = std::stoull(text, &used); + if (!value || value > std::numeric_limits::max() || used != text.size()) + throw std::invalid_argument(std::string(name) + " is outside its supported range"); + return static_cast(value); +} + +std::size_t multiply(std::size_t a, std::size_t b) { + if (b && a > std::numeric_limits::max() / b) + throw std::invalid_argument("shape product overflows size_t"); + return a * b; +} + +std::size_t checked_shape(std::size_t b, std::size_t s, std::size_t h, int n) { + if (n < 1 || n > 256 || !hadamard::power_of_two(n)) + throw std::invalid_argument("dim must be a power of two in [1,256]"); + const auto rows = multiply(multiply(b, s), h); + if (!rows || rows > static_cast(std::numeric_limits::max())) + throw std::invalid_argument("rows outside supported grid range"); + multiply(multiply(rows, static_cast(n)), sizeof(__half)); + return rows; +} + +Options parse(int argc, char** argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string key = argv[i]; + if (key == "--validate") o.validate = true; + else if (key == "--benchmark") o.benchmark = true; + else if (key == "--quick") o.quick = true; + else { + if (i + 1 == argc) throw std::invalid_argument("missing value for " + key); + const std::string value = argv[++i]; + if (key == "--dtype") o.dtype = value; + else if (key == "--csv") o.csv = value; + else if (key == "--json") o.json = value; + else if (key == "--batch" || key == "--seq" || key == "--heads" || key == "--dim") { + const auto v = positive(value, key.c_str()); + o.custom_shape = true; + if (key == "--batch") o.batch = v; + else if (key == "--seq") o.seq = v; + else if (key == "--heads") o.heads = v; + else { + if (v > 256) throw std::invalid_argument("dim is greater than 256"); + o.dim = static_cast(v); + } + } else if (key == "--repeats" || key == "--groups") { + const auto v = positive(value, key.c_str()); + if (v > 10000) throw std::invalid_argument("repeats/groups exceed 10000"); + if (key == "--repeats") o.repeats = static_cast(v); + else o.groups = static_cast(v); + } else throw std::invalid_argument("unknown argument " + key); + } + } + if (!o.validate && !o.benchmark) throw std::invalid_argument("specify --validate and/or --benchmark"); + if (o.dtype != "both" && o.dtype != "fp16" && o.dtype != "bf16") + throw std::invalid_argument("dtype must be fp16, bf16 or both"); + checked_shape(o.batch, o.seq, o.heads, o.dim); + return o; +} + +template std::vector make_input(std::size_t rows, int n, int pattern, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::normal_distribution normal(0.0f, 0.5f); + std::vector result(rows * n); + for (std::size_t i = 0; i < result.size(); ++i) { + float x = 0; + if (pattern == 0) x = uniform(rng); + else if (pattern == 1) x = normal(rng); + else if (pattern == 2) x = i % n == (i / n + seed) % n ? 8.0f : uniform(rng); + else if (pattern == 4) x = i % n == (i / n) % n ? 1.0f : 0.0f; + result[i] = rounded(x); + } + return result; +} + +struct Summary { + std::size_t cases = 0, elements = 0, exact_transform_elements = 0, contract_checks = 0; + std::size_t warp64_cases = 0, exact_warp64_elements = 0; + std::size_t warp64_grid_stride_cases = 0, warp64_grid_stride_elements = 0, unsupported_warp64_checks = 0; + double max_rounded_error = 0, max_unrounded_error = 0; +}; + +template void one_case(cudaStream_t stream, std::size_t rows, int n, float scale, + int pattern, unsigned seed, const char* dtype, Summary& summary) { + const auto input = make_input(rows, n, pattern, seed); + const std::size_t size = input.size(), bytes = rows * ((n + 1) / 2); + Guarded x(size, stream), baseline(size, stream), optimized(size, stream), inplace(size, stream); + Guarded base_split(bytes, stream), opt_split(bytes, stream), base_fused(bytes, stream), opt_fused(bytes, stream); + Guarded bs(rows, stream), os(rows, stream), bfs(rows, stream), ofs(rows, stream); + x.upload(input, stream); + inplace.upload(input, stream); + CHECK(api::transform(x.data(), baseline.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform(x.data(), optimized.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::transform(inplace.data(), inplace.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::quantize_int4(baseline.data(), base_split.data(), bs.data(), rows, n, stream, api::Method::Baseline)); + CHECK(api::quantize_int4(optimized.data(), opt_split.data(), os.data(), rows, n, stream, api::Method::Optimized)); + CHECK(api::transform_int4(x.data(), base_fused.data(), bfs.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform_int4(x.data(), opt_fused.data(), ofs.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(cudaStreamSynchronize(stream)); + x.download(stream, true); + const auto a = baseline.download(stream), b = optimized.download(stream), ip = inplace.download(stream); + const std::string context = std::string(dtype) + " rows=" + std::to_string(rows) + " n=" + std::to_string(n) + + " pattern=" + std::to_string(pattern) + " seed=" + std::to_string(seed) + " scale=" + std::to_string(scale); + if (std::memcmp(a.data(), b.data(), size * sizeof(T)) || std::memcmp(b.data(), ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/optimized/in-place transform not bitwise identical: " + context); + std::vector fx(size), actual(size); + for (std::size_t i = 0; i < size; ++i) { fx[i] = read(input[i]); actual[i] = read(b[i]); } + // 全部元素使用 O(N^2) FP64 稠密矩阵公式，不调用任何 FWHT 作为正确性 oracle。 + const auto dense = hadamard::dense_reference(fx, n, static_cast(scale)); + const double tolerance = std::string(dtype) == "fp16" ? 1e-2 : 5e-2; + for (std::size_t i = 0; i < size; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + const double error = std::abs(static_cast(actual[i]) - expected); + if (!std::isfinite(actual[i]) || !(error < tolerance)) + throw std::runtime_error("rounded FP64 dense oracle mismatch: " + context + " index=" + std::to_string(i) + + " error=" + std::to_string(error)); + summary.max_rounded_error = std::max(summary.max_rounded_error, error); + summary.max_unrounded_error = std::max(summary.max_unrounded_error, std::abs(actual[i] - dense[i])); + } + const auto expected_q = hadamard::quantize_int4(actual, n); + if (base_split.download(stream) != expected_q.packed || opt_split.download(stream) != expected_q.packed || + base_fused.download(stream) != expected_q.packed || opt_fused.download(stream) != expected_q.packed || + bs.download(stream) != expected_q.scales || os.download(stream) != expected_q.scales || + bfs.download(stream) != expected_q.scales || ofs.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/optimized split/fused INT4 bytes or scales mismatch: " + context); +#if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + Guarded warp_output(size, stream, "warp64.output"), warp_inplace(size, stream, "warp64.inplace"); + Guarded warp_split(bytes, stream, "warp64.split"), warp_fused(bytes, stream, "warp64.fused"); + Guarded warp_split_scales(rows, stream, "warp64.split-scales"), warp_fused_scales(rows, stream, "warp64.fused-scales"); + warp_inplace.upload(input, stream); + CHECK(api::transform(x.data(), warp_output.data(), rows, n, scale, stream, api::Method::Warp64)); + CHECK(api::transform(warp_inplace.data(), warp_inplace.data(), rows, n, scale, stream, api::Method::Warp64)); + CHECK(api::quantize_int4(warp_output.data(), warp_split.data(), warp_split_scales.data(), rows, n, stream, api::Method::Warp64)); + CHECK(api::transform_int4(x.data(), warp_fused.data(), warp_fused_scales.data(), rows, n, scale, stream, api::Method::Warp64)); + const auto warp = warp_output.download(stream), warp_ip = warp_inplace.download(stream); + if (std::memcmp(a.data(), warp.data(), size * sizeof(T)) || std::memcmp(a.data(), warp_ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/Warp64/in-place transform not bitwise identical: " + context); + if (warp_split.download(stream) != expected_q.packed || warp_fused.download(stream) != expected_q.packed || + warp_split_scales.download(stream) != expected_q.scales || warp_fused_scales.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/Warp64 split/fused INT4 bytes or scales mismatch: " + context); + x.download(stream, true, "after-warp64"); + ++summary.warp64_cases; + summary.exact_warp64_elements += size; +#endif + ++summary.cases; + summary.elements += size; + summary.exact_transform_elements += size; +} + +template void contract_tests(cudaStream_t stream, Summary& summary) { + Guarded input(64, stream, "contract.input"), output(64, stream, "contract.output"); + Guarded packed(32, stream, "contract.packed"); + Guarded scales(8, stream, "contract.scales"); + input.download(stream, true, "initialized"); + output.download(stream, true, "initialized"); + packed.download(stream, true, "initialized"); + scales.download(stream, true, "initialized"); + auto reject = [&](cudaError_t status) { + if (status != cudaErrorInvalidValue) throw std::runtime_error("invalid API input did not return cudaErrorInvalidValue"); + ++summary.contract_checks; + }; + auto success = [&](cudaError_t status) { CHECK(status); ++summary.contract_checks; }; + for (const auto method : active_methods()) { + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=invalid-parameters" << std::endl; + for (int n : {0, 3, 512}) { + reject(api::transform(input.data(), output.data(), 1, n, 1, stream, method)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, n, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, n, 1, stream, method)); + } + for (float scale : {0.0f, -1.0f, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { + reject(api::transform(input.data(), output.data(), 1, 8, scale, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, scale, stream, method)); + } + reject(api::transform(static_cast(nullptr), output.data(), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), static_cast(nullptr), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), input.data() + 1, 1, 8, 1, stream, method)); + reject(api::transform(input.data(), output.data(), std::numeric_limits::max(), 256, 1, stream, method)); + reject(api::quantize_int4(input.data(), reinterpret_cast(input.data()), scales.data(), 1, 8, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), nullptr, 1, 8, 1, stream, method)); + auto* odd = reinterpret_cast(reinterpret_cast(input.data()) + 1); + reject(api::transform(odd, output.data(), 1, 8, 1, stream, method)); + auto* bad_scale = reinterpret_cast(reinterpret_cast(scales.data()) + 2); + reject(api::quantize_int4(input.data(), packed.data(), bad_scale, 1, 8, stream, method)); + success(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, method)); + success(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, method)); + success(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, method)); + input.download(stream, true, "after-invalid-and-zero-rows"); + output.download(stream, true, "after-invalid-and-zero-rows"); + packed.download(stream, false, "after-invalid-and-zero-rows"); + scales.download(stream, false, "after-invalid-and-zero-rows"); + // 正负半整数：预期手写，避免舍入测试仅复用 CPU 参考实现。 + const std::vector ties{7, -7, .5f, 1.5f, 2.5f, -.5f, -1.5f, -2.5f}; + std::vector t(64, rounded(0)); + for (std::size_t i = 0; i < ties.size(); ++i) t[i] = rounded(ties[i]); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-upload" << std::endl; + input.upload(t, stream); + input.download(stream, true, "after-ties-upload"); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-quantize" << std::endl; + CHECK(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, method)); + const auto q = packed.download(stream, false, "after-ties-quantize"); + const auto s = scales.download(stream, false, "after-ties-quantize"); + const std::array expected{{0x97, 0x20, 0x02, 0xee}}; + if (!std::equal(expected.begin(), expected.end(), q.begin()) || s[0] != 1.0f) + throw std::runtime_error("positive/negative ties-to-even test failed"); + input.download(stream, true, "after-ties-quantize"); + ++summary.contract_checks; + } + const auto invalid = static_cast(-1); + reject(api::transform(input.data(), output.data(), 1, 8, 1, stream, invalid)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, invalid)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, invalid)); +#if !defined(HADAMARD_ILUVATAR_WARP64) || !defined(__ILUVATAR__) + auto unsupported = [&](cudaError_t status) { + if (status != cudaErrorNotSupported) throw std::runtime_error("uncompiled Warp64 path did not return cudaErrorNotSupported"); + ++summary.unsupported_warp64_checks; + }; + unsupported(api::transform(input.data(), output.data(), 1, 8, 1, stream, api::Method::Warp64)); + unsupported(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, api::Method::Warp64)); + unsupported(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, api::Method::Warp64)); + CHECK(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, api::Method::Warp64)); + CHECK(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, api::Method::Warp64)); + CHECK(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, api::Method::Warp64)); + summary.unsupported_warp64_checks += 3; +#endif + CHECK(cudaStreamSynchronize(stream)); + output.download(stream, true); +} + + +template Summary validate(cudaStream_t stream, const char* dtype, const Options& options) { + Summary result; + const std::vector dims = options.custom_shape ? std::vector{options.dim} + : (options.quick ? std::vector{1, 64, 256} : std::vector{1, 2, 4, 8, 16, 32, 64, 128, 256}); + const std::vector rows = options.custom_shape + ? std::vector{checked_shape(options.batch, options.seq, options.heads, options.dim)} + : (options.quick ? std::vector{3} : std::vector{1, 3, 17, 257}); + for (int n : dims) { + for (auto r : rows) { + for (int normalized = 0; normalized < (n == 1 ? 1 : 2); ++normalized) { + const float scale = normalized ? 1.0f / std::sqrt(static_cast(n)) : 1.0f; + for (int pattern = 0; pattern < 5; ++pattern) { + // 零值/脉冲无随机性，只计一次；N=1 的两种 scale 相同，也只计一次。 + const int seeds = pattern < 3 && !options.quick ? 3 : 1; + for (int seed = 0; seed < seeds; ++seed) + one_case(stream, r, n, scale, pattern, 123 + 7919 * seed, dtype, result); + } + } + } + std::cout << "VALIDATION_PROGRESS dtype=" << dtype << " n=" << n << " cases=" << result.cases << std::endl; + } + if (!options.quick && !options.custom_shape) { + // 超过 65535 个 block 的网格上限，让同一个 block 必须处理下一行。 + for (int n : {1, 2}) one_case(stream, 65537, n, 1.0f, 0, 1847, dtype, result); +#if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + // 每个 Warp64 CTA 处理 4 行，262145 同时覆盖网格复用和不满 CTA 的尾行。 + for (int n : {1, 2}) { + one_case(stream, 262145, n, 1.0f, 0, 4541, dtype, result); + ++result.warp64_grid_stride_cases; + result.warp64_grid_stride_elements += 262145u * n; + } +#endif + } + contract_tests(stream, result); + std::cout << "VALIDATION_PASS dtype=" << dtype << " cases=" << result.cases << " elements=" << result.elements + << " max_rounded_error=" << std::setprecision(12) << result.max_rounded_error + << " max_unrounded_error=" << result.max_unrounded_error << " contract_checks=" << result.contract_checks + << " warp64_cases=" << result.warp64_cases << " exact_warp64_elements=" << result.exact_warp64_elements + << " warp64_grid_stride_cases=" << result.warp64_grid_stride_cases + << " unsupported_warp64_checks=" << result.unsupported_warp64_checks << std::endl; + return result; +} + +template void benchmark(cudaStream_t stream, const char* dtype, const Options& o, std::ofstream& csv) { + struct Shape { std::size_t b, s, h; int n; }; + std::vector shapes; + if (o.custom_shape) shapes.push_back({o.batch, o.seq, o.heads, o.dim}); + else for (int n : {64, 128, 256}) for (auto rows : {1, 17, 257, 4096, 16384}) + shapes.push_back({rows >= 4096 ? static_cast(rows / 1024) : 1, + rows >= 4096 ? 64 : static_cast(rows), rows >= 4096 ? 16u : 1u, n}); + struct Configuration { api::Method method; int operation; const char* name; }; + // 无 Warp64 宏时保留原来的 6 路顺序；启用后追加 3 路，独立输出新的原始样本。 + std::vector configurations{ + {api::Method::Baseline, 0, "baseline_transform"}, {api::Method::Optimized, 0, "optimized_transform"}, + {api::Method::Baseline, 1, "baseline_split"}, {api::Method::Optimized, 1, "optimized_split"}, + {api::Method::Baseline, 2, "baseline_fused"}, {api::Method::Optimized, 2, "optimized_fused"}}; +#if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + configurations.push_back({api::Method::Warp64, 0, "warp64_transform"}); + configurations.push_back({api::Method::Warp64, 1, "warp64_split"}); + configurations.push_back({api::Method::Warp64, 2, "warp64_fused"}); +#endif + const int configuration_count = static_cast(configurations.size()); + for (const auto shape : shapes) { + const auto rows = checked_shape(shape.b, shape.s, shape.h, shape.n), count = rows * shape.n; + const auto input = make_input(rows, shape.n, 0, 2909); + Guarded x(count, stream), y(count, stream); + Guarded q(rows * ((shape.n + 1) / 2), stream); + Guarded s(rows, stream); + x.upload(input, stream); + const float scale = 1.0f; + auto launch = [&](int which) { + const auto cfg = configurations[which]; + if (cfg.operation < 2) CHECK(api::transform(x.data(), y.data(), rows, shape.n, scale, stream, cfg.method)); + if (cfg.operation == 1) CHECK(api::quantize_int4(y.data(), q.data(), s.data(), rows, shape.n, stream, cfg.method)); + if (cfg.operation == 2) CHECK(api::transform_int4(x.data(), q.data(), s.data(), rows, shape.n, scale, stream, cfg.method)); + }; + for (int which = 0; which < configuration_count; ++which) for (int i = 0; i < 10; ++i) launch(which); + CHECK(cudaStreamSynchronize(stream)); + cudaEvent_t begin, end; + CHECK(cudaEventCreate(&begin)); CHECK(cudaEventCreate(&end)); + for (int group = 0; group < o.groups; ++group) { + // 各组轮换方法顺序；两端事件之间无分配、CPU 参考或主机设备复制。 + for (int order = 0; order < configuration_count; ++order) { + const int which = (order + group) % configuration_count; + const auto cfg = configurations[which]; + CHECK(cudaEventRecord(begin, stream)); + for (int i = 0; i < o.repeats; ++i) launch(which); + CHECK(cudaEventRecord(end, stream)); + CHECK(cudaEventSynchronize(end)); + float elapsed = 0; + CHECK(cudaEventElapsedTime(&elapsed, begin, end)); + const double us = static_cast(elapsed) * 1000.0 / o.repeats; + if (!(us > 0) || !std::isfinite(us)) throw std::runtime_error("invalid event timing"); + const std::size_t logical_bytes = cfg.operation == 0 ? count * sizeof(T) * 2 + : (cfg.operation == 1 ? count * sizeof(T) * 3 : count * sizeof(T)) + rows * ((shape.n + 1) / 2) + rows * sizeof(float); + csv << dtype << ',' << shape.b << ',' << shape.s << ',' << shape.h << ',' << shape.n << ',' << rows + << ',' << cfg.name << ',' << group << ',' << order << ',' << o.repeats << ',' << std::setprecision(12) << us + << ',' << logical_bytes << ',' << logical_bytes / us / 1000.0 << ',' << count * sizeof(T) + << ",2909,true,1\n"; + csv.flush(); + } + } + CHECK(cudaEventDestroy(begin)); CHECK(cudaEventDestroy(end)); + x.download(stream, true); + const auto output = y.download(stream); + const auto packed = q.download(stream); + const auto scales = s.download(stream); + std::vector actual(count); + for (std::size_t i = 0; i < count; ++i) actual[i] = read(output[i]); + const auto quantized = hadamard::quantize_int4(actual, shape.n); + if (packed != quantized.packed || scales != quantized.scales) + throw std::runtime_error("benchmark-size fused INT4 differs from CPU quantization"); + for (std::size_t row : {std::size_t(0), rows / 2, rows - 1}) { + std::vector sample(shape.n); + for (int i = 0; i < shape.n; ++i) sample[i] = read(input[row * shape.n + i]); + const auto expected = hadamard::dense_reference(sample, shape.n, scale); + for (int i = 0; i < shape.n; ++i) { + const double error = std::abs(actual[row * shape.n + i] - read(rounded(static_cast(expected[i])))); + if (!(error < (std::string(dtype) == "fp16" ? .01 : .05))) + throw std::runtime_error("benchmark-size sampled dense oracle mismatch"); + } + } + std::cout << "BENCHMARK_PROGRESS dtype=" << dtype << " rows=" << rows << " n=" << shape.n << std::endl; + } +} + +void write_summary(std::ostream& f, const char* dtype, const Summary& s) { + f << '"' << dtype << "\":{\"cases\":" << s.cases << ",\"elements\":" << s.elements + << ",\"exact_baseline_optimized_elements\":" << s.exact_transform_elements + << ",\"warp64_cases\":" << s.warp64_cases << ",\"exact_baseline_warp64_elements\":" << s.exact_warp64_elements + << ",\"warp64_grid_stride_cases\":" << s.warp64_grid_stride_cases + << ",\"warp64_grid_stride_elements\":" << s.warp64_grid_stride_elements + << ",\"unsupported_warp64_checks\":" << s.unsupported_warp64_checks + << ",\"max_abs_error_rounded_fp64\":" << std::setprecision(15) << s.max_rounded_error + << ",\"max_abs_error_unrounded_fp64\":" << s.max_unrounded_error + << ",\"api_contract_checks\":" << s.contract_checks << '}'; +} + +int main(int argc, char** argv) { + Options options; + try { options = parse(argc, argv); } + catch (const std::exception& e) { std::cerr << "INVALID_ARGUMENT " << e.what() << '\n'; return 2; } + try { + CHECK(cudaSetDevice(0)); + cudaDeviceProp prop{}; + CHECK(cudaGetDeviceProperties(&prop, 0)); +#if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + if (prop.warpSize != 64) throw std::runtime_error("Warp64 build requires a real device reporting warpSize=64"); +#endif + int runtime = 0, driver = 0; + CHECK(cudaRuntimeGetVersion(&runtime)); CHECK(cudaDriverGetVersion(&driver)); + std::cout << "DEVICE name=" << prop.name << " warp=" << prop.warpSize << " runtime=" << runtime << " driver=" << driver << std::endl; + cudaStream_t stream; + CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + Summary fp16, bf16; + if (options.validate) { + if (options.dtype != "bf16") fp16 = validate<__half>(stream, "fp16", options); + if (options.dtype != "fp16") bf16 = validate<__nv_bfloat16>(stream, "bf16", options); + std::ofstream json(options.json); + if (!json) throw std::runtime_error("cannot create validation JSON " + options.json); + json << "{\"status\":\"PASS\",\"full_matrix\":" << ((!options.quick && !options.custom_shape && options.dtype == "both") ? "true" : "false") +#if defined(HADAMARD_ILUVATAR_WARP64) && defined(__ILUVATAR__) + << ",\"warp64_enabled\":true,\"methods\":[\"baseline\",\"optimized\",\"warp64\"]" +#else + << ",\"warp64_enabled\":false,\"methods\":[\"baseline\",\"optimized\"]" +#endif + << ",\"oracle\":\"all-element FP64 dense, rounded to output dtype\",\"fp16_tolerance_strict\":0.01,\"bf16_tolerance_strict\":0.05," + << "\"warmup_not_counted\":true,\"dtypes\":{"; + bool comma = false; + if (options.dtype != "bf16") { write_summary(json, "fp16", fp16); comma = true; } + if (options.dtype != "fp16") { if (comma) json << ','; write_summary(json, "bf16", bf16); } + json << "}}\n"; + if (!json) throw std::runtime_error("failed writing validation JSON"); + } + if (options.benchmark) { + std::ofstream csv(options.csv); + if (!csv) throw std::runtime_error("cannot create benchmark CSV " + options.csv); + csv << "dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale\n"; + if (options.dtype != "bf16") benchmark<__half>(stream, "fp16", options, csv); + if (options.dtype != "fp16") benchmark<__nv_bfloat16>(stream, "bf16", options, csv); + if (!csv) throw std::runtime_error("failed writing benchmark CSV"); + } + CHECK(cudaStreamDestroy(stream)); + std::cout << "PASS requested validation/benchmark operations completed" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "FAIL " << e.what() << std::endl; + return 1; + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/metax/README.md b/03_hadamard_tc/a962695448-rgb/platforms/metax/README.md new file mode 100644 index 00000000..1bff1fad --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/metax/README.md @@ -0,0 +1,162 @@ +# 沐曦 C500 MACA 后端 + +本目录提供 Hadamard 项目的独立沐曦 MACA/cu-bridge 后端。固定源码 **`fe44aa33a865e27a9d52120c94084994dbbcb8de`** 已在 **C500 25% 计算配额、16000 MiB 显存配额的 sGPU** 上完成 1504 组变换验证、180 项 API 契约检查、14 项 CLI 拒绝检查，以及三次独立进程的 4050 条基准组样本。所有请求阶段成功退出；真实编译弃用告警保留在原始日志，不能称为“编译无警告”。 + +算法起点是本项目天数后端固定提交 `a387db3332c6f9b01f128dd681848260c9691281`，沐曦代码使用独立命名空间、编译宏和结果记录。天数与本次沐曦设备都报告 warp64，不代表它们具有相同的编译、同步、舍入或性能行为；天数的 1504 组结果不能计为沐曦通过用例。 + +## 已核验的开发环境与实际配额 + +| 项目 | 当前实机/镜像信息 | +|---|---| +| 设备 | 沐曦 C500，sGPU 计算配额 **25%** | +| 可用资源配额 | `mx-smi` 明确显示 **16000 MiB** 显存配额；租赁界面标为 16GB | +| 物理设备属性 | 65536 MiB 显存、104 个 SM；这是物理属性，不能当成当前实例独占资源 | +| MACA/驱动工具报告 | MACA `3.0.0.8`，Kernel Mode `3.8.30`，`mx-smi 2.2.6` | +| 供应商镜像标签 | PyTorch `2.4`、Python `3.10.10`、MACA `3.0.0.4`；镜像标签与驱动工具报告分别记录，不合并成一个版本 | +| 编译器 | `cucc` / `mxcc 1.0.0`，版本标识 `df29922f9c` | +| 兼容运行时返回值 | runtime API 和 driver API 均返回 `11060`；这是兼容 API 的数值，不证明正在使用 NVIDIA CUDA 11.6 | +| warp | 设备报告 64；已对两个完整 warp 执行最小交换探针 | + +**本实例是 25% 计算配额的 C500 sGPU，不是独占完整 C500。**后续性能必须标注该配额，不能与整卡结果直接比较，也不能简单将吞吐量乘以四外推整卡性能。尚未取得干扰隔离、固定时钟和性能计数器权限的证据。 + +记录环境时可运行： + +```bash +mx-smi +/opt/maca/tools/cu-bridge/bin/cucc --version +/opt/maca/mxgpu_llvm/bin/mxcc --version +python3 --version +``` + +供应商的镜像和宿主机驱动可以分别提供组件。当前版本组合只以已运行探针为初步兼容证据，不据此承诺其他 MACA 版本组合可用。 + +## MACA 原生接口与本项目的 cu-bridge 路径 + +MACA 原生开发使用 `mxcc`、MACA 头文件与运行时接口，例如 `mc_runtime.h`、`mc_common.h` 和 `mcMalloc` 等；原生源文件需要按 MACA API 编写。这与保留 CUDA 命名的兼容源文件是两种开发入口，不能把它们的头文件和链接参数随意混用。 + +当前移植选择 **MACA 的 cu-bridge 编译路径**：`/opt/maca/tools/cu-bridge/bin/cucc` 编译设备程序，源码保留 `cuda_runtime.h`、`cuda_fp16.h`、`cuda_bf16.h`、`__half`、`__nv_bfloat16` 等兼容名称。已在该机器确认这些头文件及转换入口可用。保留 CUDA 名称不等于由 NVIDIA 编译器执行，也不等于 CPU 回退。 + +不要把天数的 `/usr/local/corex`、`-x ivcore`、`__ILUVATAR__`，或 NVIDIA 的 `-arch=sm_89` 参数带入沐曦构建。沐曦 Warp64 分支使用 `HADAMARD_METAX_WARP64` 和真实 MACA 编译环境的 `__MACACC__`；不手动伪造编译器识别宏。 + +## 最小实机探针已经证明什么 + +- FP16、BF16 各 257 个输入，包含正负值、正负零、舍入中点及边界样例。设备存储位模式与独立宿主机整数最近偶数舍入参考一致，随后转为 FP32 的简单算术结果位模式一致。 +- 输入保持不变、分配区前后各 17 个元素的哨兵检查通过；覆盖只满足 2 字节对齐的半精度存储及不满线程块的尾部。 +- 两个完整的 64-lane warp，对 XOR 距离 `1、2、4、8、16、32` 执行整数 `__shfl_xor` 交换，共 768 个观察值全部符合预期，哨兵检查通过。 + +单独的探针只能证明该有限范围，不证明完整 FWHT、跨寄存器蝶形、多行/尾行同步或融合性能；后文记录了随后独立执行的完整验证。BF16 证明的是存储/转换及 FP32 计算链路，**不宣称 C500 的原生 BF16 算术加速已经由本探针证实**。原始 [探针源码](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/initial/metax_smoke.cu)、[编译日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/initial/smoke_build.log)、[运行日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/initial/smoke_run.log) 与 [初始环境](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/initial/environment_initial.json) 均单独归档，没有借用天数日志。 + +## C++ API 与调用契约 + +接口见 [hadamard_api.h](hadamard_api.h)，实现见 [hadamard_api.cu](hadamard_api.cu)，命名空间为 `hadamard::metax`。现有声明包括： + +| 入口 | 目标行为 | +|---|---| +| `transform` | FP16/BF16 Hadamard 变换，内部 FP32，允许完全原位变换 | +| `quantize_int4` | 对已存储的 FP16/BF16 数据执行每行对称 INT4 量化 | +| `transform_int4` | 融合变换与量化，先按 FP16/BF16 输出类型舍入，再量化 | + +方法枚举包含 `Baseline`、共享内存 `Optimized` 和条件编译的 `Warp64`，默认仍是 `Optimized`。本次完整沐曦运行检查了三种方法。对当前已经验证的 C500/SDK/配额，可在满足下方编译与设备检查条件后显式选择 `Method::Warp64`；不自动改变其他设备的默认派发。 + +输入为连续设备内存 `[rows,n]`，N 是 1～256 的 2 次幂；连续 `[batch,seq,heads,head_dim]` 可在检查乘法溢出后令 `rows=batch*seq*heads`、`n=head_dim`。API 不接受 stride 参数，不自动转换非连续视图。scale 应有限且为正，输入应有限，变换后的值应位于输出存储类型的有限范围内。 + +输入/变换输出各需 `rows*n` 个对应 dtype，INT4 需 `rows*ceil(n/2)` 字节，量化 scale 需 `rows` 个 float。输入及变换输出要求 2 字节对齐，scales 要求 float 对齐；量化涉及的缓冲区互不重叠。transform 只允许完全原位或互不重叠。API 不查询真实分配容量或设备归属，调用方必须保证它们有效。 + +INT4 约定为 `[-7,7]`，每行 `s=max(abs(x))/7`，全零行取 `s=1`；最近偶数舍入，偶数元素存低四位，N=1 的空高四位为零。本次完整运行中融合路径与分步路径的 packed bytes 及 float scales 已分别对照并精确一致。 + +操作只在调用方 stream 上发射，不分配、不复制、不等待。检查参数/发射返回值，并在回读前检查 stream 同步的返回值。合法 `rows=0` 不发射内核，仍检查 N、scale 和方法；不能把零行成功当作设备路径验证。 + +Warp64 必须用真实 MACA/cu-bridge 编译，显式启用 `-DHADAMARD_METAX_WARP64`；独立 API 调用方还须在设备初始化时通过 `cudaGetDeviceProperties` 检查 `prop.warpSize == 64`。未编入该路径时，非空 Warp64 调用返回 `cudaErrorNotSupported`，不静默回退。仅通过该属性检查仍不足以替代本平台完整测试。 + +## 完整复现入口 + +本平台使用独立的 [run_platform.py](run_platform.py) 和 [validate_and_benchmark.cu](validate_and_benchmark.cu)。以下命令重现已验证版本的执行方式；从项目 `03_hadamard_tc/a962695448-rgb` 目录执行并固定上述源码提交，结果目录必须尚不存在： + +```bash +# 快速检查，不能替代完整矩阵或性能验收。 +python3 platforms/metax/run_platform.py --warp64 --quick --no-benchmark \ + --output results/metax/warp64_quick_01 + +# 完整矩阵与默认基准。 +python3 platforms/metax/run_platform.py --warp64 --repeats 100 --groups 5 \ + --output results/metax/warp64_full_01 + +# 仅共享实现的单独复测入口；本次公开全量记录为上面的 Warp64 构建。 +python3 platforms/metax/run_platform.py --no-benchmark \ + --output results/metax/shared_full_01 +``` + +`--maca-root` 默认为 `/opt/maca`，`--compiler` 默认为该目录下 `tools/cu-bridge/bin/cucc`。runner 保留 `cucc` 入口名称，不因解析符号链接而改变兼容模式；实际编译为 cucc、C++17、`-O2`、项目 include 路径和两个 `.cu` 文件，Warp64 模式额外定义 `HADAMARD_METAX_WARP64`，由 cucc 处理兼容运行库链接。 + +runner 只为本次子进程设置 `MACA_PATH`、指向 cu-bridge 的 `CUDA_PATH`/`CUCC_PATH`、工具搜索路径及存在的 SDK 库目录；保留继承环境，不安装/替换驱动或框架。`--warp64` 是构建选项，生成的程序会在设备运行前检查真实 warp 宽度。不满足条件时应明确失败，不能把该条件静默跳过后写成支持。 + +本次 Warp64 完整模式实际执行 1504 个参数组合、180 项 API 契约检查，另有 runner 的 14 项 CLI 拒绝检查；基准为九条路径，每轮 1350 组样本。不启用 Warp64 的独立构建具有 1500 个参数组合、122 项 API 契约检查、12 项未编入路径检查和六条基准路径；该构建入口不与本次启用 Warp64 的实测数量相加。 + +每个新目录保存 `build.log`、`invalid_*.log`、`validation.log/json`、可选 `benchmark.log/csv` 和 `run_summary.json`。摘要包含本次沐曦源码 hash、Git HEAD/状态、MACA 环境、编译器、各阶段命令/退出码及产物 hash。`adapted_from` 是算法来源的天数提交，不能误认成当前沐曦实际测试源码版本。后续多轮须使用新的文件/目录；同一正确性矩阵重复运行不累计成新用例覆盖。 + +## 完整实机结果 + +`fe44aa33a865e27a9d52120c94084994dbbcb8de` 在测试时工作树干净，五个参与构建/运行的源文件 hash 已与该 Git 提交逐一核对。`run_summary.json` 为 `PASS`，`quick=false`、`warp64_enabled=true`、`full_matrix=true`。三种方法和原位路径的实际运行证据与阈值都记录在本平台，未把其他平台结果改名使用。 + +| 验收项 | 当前状态 | 原始记录 | +|---|---|---| +| 最小存储/舍入/算术、整数 shuffle | 通过上述有限范围 | [initial](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/initial/smoke_run.log) | +| 三种方法 cucc 构建 | 退出 0，有 `__shfl_xor` 弃用告警 | [build.log](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/build.log) | +| 两 dtype 完整矩阵、API、内存和 INT4 | 1504 组、180 项 API、14 项 CLI 通过 | [validation.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/validation.json)、[运行日志](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/validation.log) | +| 多行、尾行与网格复用 | 覆盖 65537 行及 262145 行边界路径 | 同一完整验证记录，不重复计数 | +| 同配额性能 | 三进程均退出 0，4050 条原始样本 | [repeat_runs.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/repeat_runs.json) 与后文 CSV | +| 版本与归档 | 源码和原始文本均按 hash 固定 | [run_summary.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/run_summary.json)、[公开清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/manifest.json) | + +### 矩阵、误差和量化一致性 + +N 覆盖 `{1,2,4,8,16,32,64,128,256}`，普通矩阵行数为 `{1,3,17,257}`，scale 为 1 和 `1/sqrt(N)`，N=1 不重复计相同 scale。输入含均匀 `[-1,1)`、标准差 0.5 的正态分布、幅度 8 的离群值、全零和单位脉冲；随机分布使用 `123、8042、15961` 三个种子，确定性零值/脉冲只计一次。另有 N=1/2 的 65537 行和 262145 行用例，分别覆盖共享实现及每 CTA 四行的 Warp64 网格复用/尾行。 + +| 精度 | 实际参数组合 | 检查元素 | API 契约检查 | 已舍入参考最大绝对误差 | 未舍入参考最大绝对误差 | +|---|---:|---:|---:|---:|---:| +| FP16 | 752 | 4,105,264 | 90 | 0.00390625 | 0.015620231628418 | +| BF16 | 752 | 4,105,264 | 90 | 0.000003814697265625 | 0.124984741210938 | + +所有三种方法的变换和原位结果逐字节一致；FP16/BF16 两类下，CPU 对实际设备变换输出量化、三种方法的分步量化和融合量化，packed bytes 和 scale 全部一致。非法参数、对齐、重叠、溢出、空指针、零行、输入保持、前后哨兵及手写正负半整数 RNE 预期均在本次运行检查。每 dtype 的 262145 行 N=1/2 用例共 786,435 个元素，已计入上表。 + +正确性沿用项目已舍入参考定义：独立 FP64 稠密结果经 FP32 转换后舍入到输出 dtype，与实际输出比较，FP16 绝对误差严格小于 `1e-2`、BF16 严格小于 `5e-2`。**阈值判定只对应已舍入列；未舍入列包含输出存储精度的舍入误差，不能也说它小于同一阈值。**两列均公开保留，没有放宽阈值、缩小测试分布或跳过 BF16。 + +### 编译告警与兼容范围 + +cucc 编译使用 `mxcc 1.0.0 (df29922f9c)`，SDK 实际解析为 `/opt/maca-3.0.0`。完整 [build.log](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/build.log) 保留 219034 字节原始输出，包括 legacy `__shfl_xor` 被 SDK 标记为 deprecated 的警告及模板实例化信息；最小探针的编译也有相应告警。构建和真实运行退出 0 不等于无告警。 + +本次版本继续使用经该 MACA/设备组合独立验证的 legacy shuffle。它可能在其他或未来 SDK 中变化；升级或替换原语时必须独立核验 lane 参与、width/mask 语义、全量正确性和性能。当前报告既不删除告警，也不声称已经证明所有未来版本兼容。 + +### 三轮性能和原始数据 + +三次独立基准进程使用相同二进制 SHA256 `8f7ac15ea904a47de0b26635094cdc0740a49b9bc4039490d5dde0e52cbf35e1`，均退出 0。每轮覆盖两 dtype、N=64/128/256、rows=1/17/257/4096/16384，三方法各有 transform/split/fused 三种操作，共 30 个 shape/dtype 条件和九条路径。各路径先预热 10 次，五组、每组重复 100 次，组间轮换方法次序；每轮 1350 条组样本，合计 4050。 + +原始样本见 [第一轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/benchmark.csv)、[第二轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/benchmark_run2.csv)、[第三轮](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/benchmark_run3.csv)，完整统计见 [analysis.json](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/analysis/analysis.json) 和 [method_summary.csv](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/analysis/method_summary.csv)。每轮内部先取五组中位数，再对相同 shape/dtype/scale/操作匹配比较；不与天数或旧实验的测量相除。 + +Warp64 的 90/90 组匹配比较在每一轮均比本机基线减少至少 5% 耗时，没有相对基线退化的样例。共享内存 `Optimized` 则有 55/90 组每轮减少至少 5%，30 组至少一轮退化、29 组每轮退化、14 组每轮退化超过 3%。分析总数 145 是 **共享版 55 + Warp64 90**，不能全算成 Warp64 收益；共享版负例完整保留。 + +下表是本次全部对应形状和三轮的“对照中位耗时 / Warp64 中位耗时”范围： + +| 操作 | 基线 / Warp64 | 共享内存优化版 / Warp64 | +|---|---:|---:| +| transform | 1.06974554～2.53939018 倍 | 1.11071430～1.69060521 倍 | +| split | 1.07633461～5.29428683 倍 | 1.06692802～1.44593072 倍 | +| fused | 1.28999182～6.05722033 倍 | 1.02615774～1.43636359 倍 | + +相对共享内存优化版，Warp64 在本轮矩阵中没有退化，但只有 **88/90** 组每轮都减少至少 5% 耗时。两个例外是 FP16/BF16 的 `[16,64,16,128]` fused，改善只有约 2.55%～3.84%；不能写成相对共享版也是 90/90 达到 5%。 + +只比较 Warp64 自身分步与融合路径，在同一轮相同 shape/dtype/scale 下，`split/fused` 为 **1.27218808～1.70823860 倍**，对应融合耗时减少 **21.3953%～41.4602%**；30/30 个形状条件在每轮都降低至少 5%。这些量化收益使用当前沐曦九路径实验，与其他平台或旧版本的时间无关。 + +负例和波动同样保留：共享版 FP16 `[1,17,1,128]` transform 比基线慢约 14.50%、5.57%、6.64%；Warp64 相对基线的最小收益来自 BF16 `[1,1,1,128]` transform，其三轮降时为约 6.52%、22.66%、21.92%。少数重复进程无法证明任意共享负载下的性能稳定性；均值/标准差/CV 描述采样，不是置信区间。 + +本次固定分配与只读输入，复用输入可能受缓存影响。设备事件区间排除分配、H2D/D2H、预热和验证，可能包含主机发射之间的设备空闲间隔；它不是端到端时间，也不是隔离测得的纯单内核执行时间。逻辑读写量换算的 GB/s 不是实测物理显存带宽。速度比仅覆盖 N=64/128/256 的性能矩阵，N=1～32 虽通过正确性，不能外推相同收益。天数和沐曦处于不同硬件/配额，不直接用两张表相除宣称平台优劣。 + +## 交付范围、公开清单与局限 + +- 本次独立验证对象是上述 MACA 3.0 系列/cu-bridge 和 C500 **25% sGPU**，不是物理整卡；没有证据证明同宿主机其他租户始终空闲或设备时钟固定。前后 `mx-smi` 快照随结果保留，只反映采样时刻。 +- 支持的 C++ 接口为连续 FP16/BF16、N=1～256 的 2 次幂、调用方 stream、原位变换、每行 INT4 分步与融合。非连续视图、任意长度、容量不足、跨设备指针、非有限输入及超出存储范围不在当前契约内。 +- 默认方法仍为共享内存 `Optimized`，MR-V100 的结果未用于决定 C500 通过。对本次已经验证的 C500 环境推荐显式选择 `hadamard::metax::Method::Warp64`；调用方必须满足编译宏及实际 `warpSize==64` 预检查。 +- 未提供沐曦 PyTorch Python 扩展，没有证明原生 BF16 算术或矩阵单元加速，没有硬件性能计数器或主机端到端延迟数据。其他 SDK/设备/配额应重新验证；deprecated shuffle 的未来兼容性不由本次结果保证。 + +[结果目录清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/metax/fe44aa3/manifest.json) 按字节数与 SHA256 列出原始运行、三轮 CSV、派生分析和初始探针/环境。29 份完整运行原始文件通过传输清单校验，五个实际源码 hash 与固定 Git 提交匹配；目录 `.gitattributes` 使用 `-text`，避免 Git 换行转换破坏日志和证据 hash。原始编译告警保持完整，没有复制本机二进制和私人租赁收据。 + +公开材料排除实例访问地址、租赁编号和凭据；保留足以复现的设备型号、配额、软件版本、命令和日志。本平台扩展独立记录构建、数值与性能结果，后续优化保持同一量化契约和可复现测量；PR 与课程登记在项目所有者验收后单独执行。 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/metax/hadamard_api.cu b/03_hadamard_tc/a962695448-rgb/platforms/metax/hadamard_api.cu new file mode 100644 index 00000000..b0c828ba --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/metax/hadamard_api.cu @@ -0,0 +1,351 @@ +// 算法来源：本项目天数后端固定提交 a387db3332c6f9b01f128dd681848260c9691281。 +// 沐曦后端必须独立完成正确性与性能验证，不继承其他平台的验收结果。 +#include "hadamard_api.h" + +#include +#include + +namespace hadamard::metax { +namespace { + +template __device__ float read_value(T value); +template<> __device__ float read_value(__half value) { return __half2float(value); } +template<> __device__ float read_value(__nv_bfloat16 value) { + return __bfloat162float(value); +} +template __device__ T store_value(float value); +template<> __device__ __half store_value(float value) { return __float2half_rn(value); } +template<> __device__ __nv_bfloat16 store_value(float value) { + return __float2bfloat16_rn(value); +} + +__device__ std::uint8_t quantize_pair(const float* values, int pair, int n, + float row_scale) { + std::uint8_t byte = 0; + for (int k = 0; k < 2 && pair * 2 + k < n; ++k) { + const float x = values[pair * 2 + k] / row_scale; + // 明确实现最近偶数舍入，不依赖编译器默认取整模式。 + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + byte |= static_cast((q & 15) << (4 * k)); + } + return byte; +} + +// 沿用固定来源版本的基线算法：一元素一线程、每层两次屏障、线程 0 串行归约。 +// 与 optimized 独立保留蝶形计算，供同一运行环境下公平比较。 +template +__global__ void baseline_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float row_scale; + const int i = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + if (i < n) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + float next = 0; + if (i < n) { + const float a = values[i], b = values[i ^ stride]; + next = (i & stride) ? b - a : a + b; + } + __syncthreads(); + if (i < n) values[i] = next; + __syncthreads(); + } + if (i < n) values[i] = read_value(store_value(values[i] * scale)); + __syncthreads(); + } + if constexpr (!Quantize) { + if (i < n) output[offset + i] = store_value(values[i]); + } else { + if (i == 0) { + float magnitude = 0; + for (int j = 0; j < n; ++j) + magnitude = fmaxf(magnitude, fabsf(values[j])); + row_scale = magnitude == 0 ? 1.0f : magnitude / 7.0f; + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + if (i < bytes) packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + // 多行复用同一 block 时，防止下行加载覆盖仍在打包/写出的上一行。 + if (rows - row > gridDim.x) __syncthreads(); + } +} + +// 每个线程独占一个蝶形的两个输入/输出，因此单层没有线程间读写冲突， +// 只需在本层结束后同步。此算法不假定 warp 宽度，不使用 warp 级隐式同步。 +template +__global__ void optimized_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float maxima[128]; + __shared__ float row_scale; + const int tid = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + for (int i = tid; i < n; i += blockDim.x) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + if (tid < n / 2) { + const int low = ((tid & ~(stride - 1)) << 1) | (tid & (stride - 1)); + const float a = values[low], b = values[low + stride]; + values[low] = a + b; + values[low + stride] = a - b; + } + __syncthreads(); + } + } + if constexpr (!Quantize) { + for (int i = tid; i < n; i += blockDim.x) + output[offset + i] = store_value(values[i] * scale); + } else { + float magnitude = 0; + for (int i = tid; i < n; i += blockDim.x) { + float value = values[i]; + if constexpr (Transform) { + value = read_value(store_value(value * scale)); + values[i] = value; + } + magnitude = fmaxf(magnitude, fabsf(value)); + } + maxima[tid] = magnitude; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 1; stride /= 2) { + if (tid < stride) maxima[tid] = fmaxf(maxima[tid], maxima[tid + stride]); + __syncthreads(); + } + if (tid == 0) { + // 最后两个值只由线程 0 消费，与 scale 写入合并，少一次屏障。 + const float maximum = fmaxf(maxima[0], maxima[1]); + row_scale = maximum == 0 ? 1.0f : maximum / 7.0f; + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + for (int i = tid; i < bytes; i += blockDim.x) + packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + if (rows - row > gridDim.x) __syncthreads(); + } +} + +#if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + +// 仅在沐曦 MACA/cu-bridge 构建中启用。legacy shuffle 的 width=64 已由沐曦实机独立探针验证。 +// 这里不使用 NVIDIA 的 32 位 active mask，也不将 32-lane 测试当作正确性证据。 +__device__ int warp64_quantized_nibble(float value, float scale) { + const float x = value / scale; + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + return q & 15; +} + +template +__global__ void warp64_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float scale) { + constexpr int width = 64; + constexpr int rows_per_block = 4; + constexpr int registers = N > width ? N / width : 1; + const int lane = threadIdx.x % width; + const int warp = threadIdx.x / width; + const std::size_t first_row = static_cast(blockIdx.x) * rows_per_block + warp; + const std::size_t row_stride = static_cast(gridDim.x) * rows_per_block; + // 同一 warp 的 64 个 lane 具有相同 row 与循环次数。尾行只跳过完整 warp， + // 所有参与行均有完整的 64 个 lane 执行每次 shuffle。 + for (std::size_t row = first_row; row < rows; row += row_stride) { + const std::size_t offset = row * N; + float values[registers]; + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + values[r] = i < N ? read_value(input[offset + i]) : 0.0f; + } + if constexpr (Transform) { + #pragma unroll + for (int stride = 1; stride < (N < width ? N : width); stride *= 2) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const float current = values[r]; + const float peer = __shfl_xor(current, stride, width); + values[r] = (lane & stride) ? peer - current : current + peer; + } + } + // stride=64/128 的搭档位于同一 lane 的其他寄存器中， + // 顺序与标量共享内存 FWHT 的从低位到高位阶段一致。 + if constexpr (N >= 128) { + const float a = values[0], b = values[1]; + values[0] = a + b; + values[1] = a - b; + } + if constexpr (N == 256) { + const float c = values[2], d = values[3]; + values[2] = c + d; + values[3] = c - d; + const float a = values[0], b = values[2]; + const float e = values[1], f = values[3]; + values[0] = a + b; + values[2] = a - b; + values[1] = e + f; + values[3] = e - f; + } + } + if constexpr (!Quantize) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + if (i < N) output[offset + i] = store_value(values[r] * scale); + } + } else { + float magnitude = 0.0f; + #pragma unroll + for (int r = 0; r < registers; ++r) { + if constexpr (Transform) values[r] = read_value(store_value(values[r] * scale)); + magnitude = fmaxf(magnitude, fabsf(values[r])); + } + #pragma unroll + for (int stride = width / 2; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor(magnitude, stride, width)); + const float row_scale = magnitude == 0.0f ? 1.0f : magnitude / 7.0f; + if (lane == 0) scales[row] = row_scale; + #pragma unroll + for (int r = 0; r < registers; ++r) { + // shuffle 在分支之前：奇数 lane 同样必须提供其相邻元素。 + const float peer = __shfl_xor(values[r], 1, width); + const int i = lane + r * width; + if ((lane & 1) == 0 && i < N) { + const int low = warp64_quantized_nibble(values[r], row_scale); + const int high = i + 1 < N ? warp64_quantized_nibble(peer, row_scale) : 0; + packed[row * ((N + 1) / 2) + i / 2] + = static_cast(low | (high << 4)); + } + } + } + } +} + +template +cudaError_t launch_warp64(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, cudaStream_t stream) { + // rows <= SIZE_MAX/4 已由 launch 校验，因此 (rows+3)/4 不会溢出。 + const std::size_t requested = (rows + 3) / 4; + const unsigned int blocks = static_cast(requested < 65535 ? requested : 65535); + #define LAUNCH_WARP64(N) case N: \ + warp64_kernel<<>>( \ + input, output, packed, scales, rows, scale); \ + break + switch (n) { + LAUNCH_WARP64(1); + LAUNCH_WARP64(2); + LAUNCH_WARP64(4); + LAUNCH_WARP64(8); + LAUNCH_WARP64(16); + LAUNCH_WARP64(32); + LAUNCH_WARP64(64); + LAUNCH_WARP64(128); + LAUNCH_WARP64(256); + default: return cudaErrorInvalidValue; + } + #undef LAUNCH_WARP64 + return cudaGetLastError(); +} + +#endif // HADAMARD_METAX_WARP64 + +bool valid_range(const void* pointer, std::size_t bytes, std::size_t alignment) { + const auto address = reinterpret_cast(pointer); + return pointer != nullptr && address % alignment == 0 + && bytes <= std::numeric_limits::max() - address; +} + +bool overlaps(const void* left, std::size_t left_size, + const void* right, std::size_t right_size) { + // valid_range 已确保两个区间端点的加法不会溢出。 + const auto a = reinterpret_cast(left); + const auto b = reinterpret_cast(right); + return a < b + right_size && b < a + left_size; +} + +template +cudaError_t launch(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, cudaStream_t stream, Method method) { + if (n < 1 || n > 256 || (n & (n - 1)) != 0 + || !std::isfinite(scale) || scale <= 0 + || (method != Method::Baseline && method != Method::Optimized && method != Method::Warp64)) + return cudaErrorInvalidValue; + if (rows == 0) return cudaSuccess; + + constexpr auto maximum = std::numeric_limits::max(); + const std::size_t row_bytes = static_cast(n) * sizeof(T); + if (rows > maximum / row_bytes || rows > maximum / sizeof(float)) + return cudaErrorInvalidValue; + const std::size_t input_bytes = rows * row_bytes; + if (!valid_range(input, input_bytes, alignof(T))) return cudaErrorInvalidValue; + if constexpr (Quantize) { + const std::size_t packed_bytes = rows * static_cast((n + 1) / 2); + const std::size_t scale_bytes = rows * sizeof(float); + if (!valid_range(packed, packed_bytes, alignof(std::uint8_t)) + || !valid_range(scales, scale_bytes, alignof(float)) + || overlaps(input, input_bytes, packed, packed_bytes) + || overlaps(input, input_bytes, scales, scale_bytes) + || overlaps(packed, packed_bytes, scales, scale_bytes)) return cudaErrorInvalidValue; + } else { + if (!valid_range(output, input_bytes, alignof(T)) + || (input != output && overlaps(input, input_bytes, output, input_bytes))) + return cudaErrorInvalidValue; + } + + if (method == Method::Warp64) { + #if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + return launch_warp64(input, output, packed, scales, rows, n, scale, stream); + #else + return cudaErrorNotSupported; + #endif + } + + // 保守上限兼容不同设备的 grid.x 限制，超过上限由 block 顺序处理多行。 + const unsigned int blocks = static_cast(rows < 65535 ? rows : 65535); + if (method == Method::Baseline) { + const int threads = n < 64 ? 64 : n; + baseline_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } else { + const int threads = n / 2 < 64 ? 64 : n / 2; + optimized_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } + return cudaGetLastError(); +} + +} // namespace + +#define DEFINE_TYPED_API(T) \ +cudaError_t transform(const T* input, T* output, std::size_t rows, int n, float scale, \ + cudaStream_t stream, Method method) { \ + return launch(input, output, nullptr, nullptr, rows, n, scale, stream, method); \ +} \ +cudaError_t quantize_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, cudaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, 1.0f, stream, method); \ +} \ +cudaError_t transform_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, float scale, cudaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, scale, stream, method); \ +} + +DEFINE_TYPED_API(__half) +DEFINE_TYPED_API(__nv_bfloat16) +#undef DEFINE_TYPED_API + +} // namespace hadamard::metax diff --git a/03_hadamard_tc/a962695448-rgb/platforms/metax/hadamard_api.h b/03_hadamard_tc/a962695448-rgb/platforms/metax/hadamard_api.h new file mode 100644 index 00000000..324032da --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/metax/hadamard_api.h @@ -0,0 +1,54 @@ +// 算法来源：本项目天数后端固定提交 a387db3332c6f9b01f128dd681848260c9691281。 +// 沐曦后端必须独立完成正确性与性能验证，不继承其他平台的验收结果。 +#pragma once + +#include +#include +#include +#include +#include + +namespace hadamard::metax { + +enum class Method { Baseline, Optimized, Warp64 }; + +// Warp64 为沐曦 64-lane warp 专用候选，须使用 +// -DHADAMARD_METAX_WARP64 且由 MACA/cu-bridge (__MACACC__) 编译；其他构建的 +// 非空操作返回 cudaErrorNotSupported。调用方必须预先确认当前设备 warpSize=64； +// API 不在发射路径中查询设备属性。rows=0 仍遵循下方的无操作约定。 + +// 连续设备内存：[rows, n]，n 为 1..256 的二次幂。scale 必须有限且为正数。 +// 支持 FP16/BF16 存储，内部 FP32；输入应有限，变换后的值应在输出类型范围内。 +// 所有操作仅在调用方的 stream 上发射，不分配内存、不复制、不等待。 +// rows=0 时只检查 n、scale、method，允许空指针，不发射内核。 +// 输入和变换输出只要求 2 字节对齐；scales 要求 float 对齐。 +// transform 允许 input==output 原位变换，其余重叠均返回 cudaErrorInvalidValue。 +// 不检查分配容量/设备归属；调用方须提供当前设备有效且足够大的设备缓冲区。 +// 返回参数/发射错误；异步执行错误由调用方同步 stream 时检查。 +cudaError_t transform(const __half* input, __half* output, std::size_t rows, + int n, float scale, cudaStream_t stream, + Method method = Method::Optimized); +cudaError_t transform(const __nv_bfloat16* input, __nv_bfloat16* output, + std::size_t rows, int n, float scale, cudaStream_t stream, + Method method = Method::Optimized); + +// 每行 ceil(n/2) 字节；偶数元素在低 4 位，奇数在高 4 位，n=1 的高位为 0。 +// q=clamp(round_to_nearest_even(x/s), -7, 7)，s=max(abs(x))/7；全零行 s=1。 +// scales 为 rows 个 float。所有输入/输出缓冲区必须互不重叠。 +cudaError_t quantize_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, cudaStream_t stream, + Method method = Method::Optimized); +cudaError_t quantize_int4(const __nv_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, + cudaStream_t stream, Method method = Method::Optimized); + +// 融合路径先以最近偶数规则舍入到公开的 FP16/BF16 输出类型，再计算 INT4， +// 因而与 transform 后调用 quantize_int4 的字节和 scales 语义一致。 +cudaError_t transform_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, + cudaStream_t stream, Method method = Method::Optimized); +cudaError_t transform_int4(const __nv_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale, + cudaStream_t stream, Method method = Method::Optimized); + +} // namespace hadamard::metax diff --git a/03_hadamard_tc/a962695448-rgb/platforms/metax/run_platform.py b/03_hadamard_tc/a962695448-rgb/platforms/metax/run_platform.py new file mode 100644 index 00000000..a1fb213a --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/metax/run_platform.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""在真实 MACA/沐曦设备构建、核查并测量；不安装或替换驱动与框架。""" +import argparse +import csv +import hashlib +import json +import os +from pathlib import Path +import statistics +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[2] +PLATFORM = Path(__file__).resolve().parent +SOURCE_COMMIT = "a387db3332c6f9b01f128dd681848260c9691281" + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def capture(command, env=None): + result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False, env=env) + return {"command": command, "returncode": result.returncode, "output": result.stdout} + + +def run(command, log, stages, expected=0, env=None): + print("RUN", " ".join(map(str, command)), flush=True) + started = time.monotonic() + with log.open("w", encoding="utf-8") as stream: + completed = subprocess.run(list(map(str, command)), cwd=ROOT, stdout=stream, + stderr=subprocess.STDOUT, check=False, env=env) + stages.append({"command": list(map(str, command)), "log": log.name, + "returncode": completed.returncode, "expected_returncode": expected, + "wall_seconds": time.monotonic() - started, "sha256": sha256(log)}) + print("EXIT", completed.returncode, "LOG", log, flush=True) + if completed.returncode != expected: + print(log.read_text(encoding="utf-8", errors="replace")[-16000:], file=sys.stderr) + raise RuntimeError("stage failed: " + log.name) + + +def summarize_benchmark(path): + samples = {} + with path.open(newline="", encoding="utf-8") as stream: + for row in csv.DictReader(stream): + key = tuple(row[k] for k in ("dtype", "batch", "seq", "heads", "dim", "method")) + samples.setdefault(key, []).append(float(row["kernel_us"])) + medians = {} + summary = [] + for key, values in samples.items(): + medians[key] = statistics.median(values) + summary.append(dict(zip(("dtype", "batch", "seq", "heads", "dim", "method"), key), + samples=values, median_us=statistics.median(values), + minimum_us=min(values), maximum_us=max(values))) + comparisons = [] + for key, value in medians.items(): + if not key[-1].startswith("baseline_"): + continue + peer = key[:-1] + (key[-1].replace("baseline_", "optimized_"),) + comparisons.append({"shape_dtype": key[:-1], "operation": key[-1][9:], + "baseline_median_us": value, "optimized_median_us": medians[peer], + "baseline_over_optimized": value / medians[peer]}) + warp_peer = key[:-1] + (key[-1].replace("baseline_", "warp64_"),) + if warp_peer in medians: + comparisons.append({"shape_dtype": key[:-1], "operation": key[-1][9:], "candidate": "warp64", + "baseline_median_us": value, "optimized_median_us": medians[peer], + "warp64_median_us": medians[warp_peer], "baseline_over_warp64": value / medians[warp_peer], + "optimized_over_warp64": medians[peer] / medians[warp_peer]}) + return {"metric": "CUDA-compatible events; warmup excluded; no allocation/copy in interval", + "working_set": "same seeded read-only input reused; warm-cache timing", + "logical_GBs_note": "logical tensor I/O estimate, not measured physical memory bandwidth", + "statistics": summary, "comparisons_including_slowdowns": comparisons} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--maca-root", type=Path, default=Path("/opt/maca")) + parser.add_argument("--compiler", type=Path) + parser.add_argument("--output", type=Path, help="必须是新的结果目录，防止覆盖既有证据") + parser.add_argument("--quick", action="store_true", help="仅运行快速调试矩阵，不代表完整验收") + parser.add_argument("--no-benchmark", action="store_true") + parser.add_argument("--warp64", action="store_true", help="显式启用仅供真实 warp64 设备验证的 MACA 专用路径") + parser.add_argument("--repeats", type=int, default=100) + parser.add_argument("--groups", type=int, default=5) + args = parser.parse_args() + if args.repeats < 1 or args.repeats > 10000 or args.groups < 1 or args.groups > 10000: + parser.error("repeats/groups must be between 1 and 10000") + maca = args.maca_root.resolve() + bridge = maca / "tools/cu-bridge" + # 保留 cucc 入口名称，避免解析符号链接后改变编译器按 argv[0] 选择的兼容模式。 + compiler = (args.compiler or bridge / "bin/cucc").absolute() + if not compiler.is_file(): + parser.error("MACA cucc compiler is missing: " + str(compiler)) + # 仅配置本次子进程环境；不改系统环境、驱动或用户现有开发环境。 + runtime_env = dict(os.environ, MACA_PATH=str(maca), CUDA_PATH=str(bridge), CUCC_PATH=str(bridge)) + runtime_env["PATH"] = os.pathsep.join([str(bridge / "tools"), str(bridge / "bin"), str(maca / "tools/bin"), + str(maca / "bin"), os.environ.get("PATH", "")]) + library_paths = [str(p) for p in (maca / "lib", maca / "lib64", bridge / "lib", bridge / "lib64") if p.is_dir()] + inherited_libraries = os.environ.get("LD_LIBRARY_PATH", "") + runtime_env["LD_LIBRARY_PATH"] = os.pathsep.join(library_paths + ([inherited_libraries] if inherited_libraries else [])) + destination = (args.output or ROOT / "results/metax" / time.strftime("%Y%m%d-%H%M%S", time.gmtime())).resolve() + if destination.exists(): + parser.error("output directory already exists; select a fresh path") + destination.mkdir(parents=True) + binary = destination / "validate_and_benchmark" + sources = [PLATFORM / "hadamard_api.h", PLATFORM / "hadamard_api.cu", + PLATFORM / "validate_and_benchmark.cu", PLATFORM / "run_platform.py", ROOT / "include/reference.hpp"] + report = {"status": "RUNNING", "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "platform": "metax", "adapted_from": {"platform": "iluvatar", "commit": SOURCE_COMMIT}, + "quick": args.quick, "warp64_enabled": args.warp64, + "sdk_environment": {key: runtime_env[key] for key in ("MACA_PATH", "CUDA_PATH", "CUCC_PATH")}, + "source_sha256": {str(p.relative_to(ROOT)): sha256(p) for p in sources}, + "git_head": capture(["git", "rev-parse", "HEAD"]), + "git_status": capture(["git", "status", "--short"]), + "compiler": capture([str(compiler), "--version"], env=runtime_env), + "python_version": sys.version, "stages": []} + manifest = destination / "run_summary.json" + manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + try: + command = [compiler, "-std=c++17", "-O2", "-I" + str(ROOT / "include")] + if args.warp64: + command.append("-DHADAMARD_METAX_WARP64") + command += [PLATFORM / "hadamard_api.cu", PLATFORM / "validate_and_benchmark.cu", "-o", binary] + run(command, destination / "build.log", report["stages"], env=runtime_env) + report["binary_sha256"] = sha256(binary) + # CLI 验证在创建设备上下文之前执行。错误输入必须明确返回 2。 + invalid = [[], ["--validate", "--dim", "0"], ["--validate", "--dim", "3"], + ["--validate", "--dim", "512"], ["--validate", "--dtype", "fp32"], + ["--validate", "--batch", "-1"], ["--validate", "--seq", "0"], + ["--validate", "--heads", "x"], ["--validate", "--batch", "18446744073709551615", "--seq", "2"], + ["--validate", "--heads", "999999999999999999999999999999"], + ["--validate", "--dim"], ["--validate", "--unknown", "1"], + ["--benchmark", "--repeats", "0"], ["--benchmark", "--groups", "10001"]] + for i, case in enumerate(invalid): + run([binary] + case, destination / ("invalid_%02d.log" % i), report["stages"], expected=2, env=runtime_env) + report["cli_rejection_cases"] = len(invalid) + command = [binary, "--validate", "--json", destination / "validation.json"] + if args.quick: + command.append("--quick") + run(command, destination / "validation.log", report["stages"], env=runtime_env) + report["validation"] = json.loads((destination / "validation.json").read_text(encoding="utf-8")) + if report["validation"]["status"] != "PASS" or (not args.quick and not report["validation"]["full_matrix"]): + raise RuntimeError("validation JSON does not confirm requested matrix") + if report["validation"]["warp64_enabled"] != args.warp64: + raise RuntimeError("validation JSON does not match requested Warp64 build") + if not args.no_benchmark: + command = [binary, "--benchmark", "--csv", destination / "benchmark.csv", + "--groups", str(args.groups), "--repeats", str(args.repeats)] + if args.quick: + command += ["--batch", "1", "--seq", "17", "--heads", "1", "--dim", "128"] + run(command, destination / "benchmark.log", report["stages"], env=runtime_env) + report["benchmark"] = summarize_benchmark(destination / "benchmark.csv") + report["status"] = "PASS" + except (OSError, RuntimeError, ValueError, KeyError) as error: + report["status"] = "FAIL" + report["error"] = str(error) + print("FAIL", error, file=sys.stderr) + finally: + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + report["artifacts"] = {p.name: sha256(p) for p in destination.iterdir() if p.is_file() and p != manifest} + manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "quick": args.quick, "summary": str(manifest)}, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/platforms/metax/validate_and_benchmark.cu b/03_hadamard_tc/a962695448-rgb/platforms/metax/validate_and_benchmark.cu new file mode 100644 index 00000000..eccd6d3a --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/metax/validate_and_benchmark.cu @@ -0,0 +1,521 @@ +// 验证口径移植自天数已验收提交 a387db3332c6f9b01f128dd681848260c9691281。 +// 沐曦结果必须在 MACA/C500 独立编译运行获得，不复用天数通过结论或性能数据。 +#include "hadamard_api.h" +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace api = hadamard::metax; +#define CHECK(call) do { cudaError_t e_ = (call); if (e_ != cudaSuccess) \ + throw std::runtime_error(std::string(#call) + ": " + cudaGetErrorString(e_)); } while (0) + +template float read(T); +template<> float read(__half x) { return __half2float(x); } +template<> float read(__nv_bfloat16 x) { return __bfloat162float(x); } +template T rounded(float); +template<> __half rounded(float x) { return __float2half_rn(x); } +template<> __nv_bfloat16 rounded(float x) { return __float2bfloat16_rn(x); } + +std::vector active_methods() { + std::vector methods{api::Method::Baseline, api::Method::Optimized}; +#if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + methods.push_back(api::Method::Warp64); +#endif + return methods; +} + +const char* method_name(api::Method method) { + if (method == api::Method::Baseline) return "baseline"; + if (method == api::Method::Optimized) return "optimized"; + return "warp64"; +} + +// 每个区间都有前后哨兵。17 个元素的偏移同时覆盖仅 2 字节对齐的 FP16/BF16 指针。 +template class Guarded { + static constexpr std::size_t guard = 17; + T* raw_ = nullptr; + std::size_t count_; + std::vector initial_; + std::string name_; +public: + explicit Guarded(std::size_t count, cudaStream_t stream, const char* name = "unnamed") + : count_(count), initial_((count + 2 * guard) * sizeof(T), 0xa5), name_(name) { + CHECK(cudaMalloc(reinterpret_cast(&raw_), initial_.size())); + // 初始化、上传、kernel、回读使用同一个非阻塞 stream。默认 stream 上的 + // pageable H2D 即使通过 cudaMemcpy 调用，也不保证返回时最终 DMA 已结束。 + // 按字节 memset 避免异步初始化读取随后被 upload 更新的主机 shadow。 + CHECK(cudaMemsetAsync(raw_, 0xa5, initial_.size(), stream)); + } + ~Guarded() { if (raw_) cudaFree(raw_); } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + T* data() { return raw_ + guard; } + void upload(const std::vector& values, cudaStream_t stream) { + if (values.size() != count_) throw std::runtime_error("upload size mismatch"); + std::memcpy(initial_.data() + guard * sizeof(T), values.data(), count_ * sizeof(T)); + CHECK(cudaMemcpyAsync(raw_, initial_.data(), initial_.size(), cudaMemcpyHostToDevice, stream)); + } + std::vector download(cudaStream_t stream, bool unchanged = false, const char* phase = "readback") { + std::vector bytes(initial_.size()); + CHECK(cudaMemcpyAsync(bytes.data(), raw_, bytes.size(), cudaMemcpyDeviceToHost, stream)); + CHECK(cudaStreamSynchronize(stream)); + const std::size_t prefix = guard * sizeof(T), end = prefix + count_ * sizeof(T); + for (std::size_t i = 0; i < bytes.size(); ++i) { + if (bytes[i] == initial_[i] || (!unchanged && i >= prefix && i < end)) continue; + const bool is_guard = i < prefix || i >= end; + throw std::runtime_error(std::string(is_guard ? "device buffer guard overwritten" : "read-only input modified") + + " buffer=" + name_ + " phase=" + phase + " region=" + (i < prefix ? "prefix" : (i >= end ? "suffix" : "payload")) + + " byte_from_payload=" + std::to_string(static_cast(i) - static_cast(prefix)) + + " expected=" + std::to_string(static_cast(initial_[i])) + + " actual=" + std::to_string(static_cast(bytes[i])) + + " elements=" + std::to_string(count_) + " element_bytes=" + std::to_string(sizeof(T))); + } + std::vector result(count_); + std::memcpy(result.data(), bytes.data() + prefix, count_ * sizeof(T)); + return result; + } +}; + +struct Options { + bool validate = false, benchmark = false, custom_shape = false, quick = false; + std::size_t batch = 1, seq = 1, heads = 1; + int dim = 128, repeats = 100, groups = 5; + std::string dtype = "both", csv = "metax_benchmark.csv", json = "metax_validation.json"; +}; + +std::size_t positive(const std::string& text, const char* name) { + if (text.empty() || text.find_first_not_of("0123456789") != std::string::npos) + throw std::invalid_argument(std::string(name) + " must be a positive integer"); + std::size_t used = 0; + const auto value = std::stoull(text, &used); + if (!value || value > std::numeric_limits::max() || used != text.size()) + throw std::invalid_argument(std::string(name) + " is outside its supported range"); + return static_cast(value); +} + +std::size_t multiply(std::size_t a, std::size_t b) { + if (b && a > std::numeric_limits::max() / b) + throw std::invalid_argument("shape product overflows size_t"); + return a * b; +} + +std::size_t checked_shape(std::size_t b, std::size_t s, std::size_t h, int n) { + if (n < 1 || n > 256 || !hadamard::power_of_two(n)) + throw std::invalid_argument("dim must be a power of two in [1,256]"); + const auto rows = multiply(multiply(b, s), h); + if (!rows || rows > static_cast(std::numeric_limits::max())) + throw std::invalid_argument("rows outside supported grid range"); + multiply(multiply(rows, static_cast(n)), sizeof(__half)); + return rows; +} + +Options parse(int argc, char** argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string key = argv[i]; + if (key == "--validate") o.validate = true; + else if (key == "--benchmark") o.benchmark = true; + else if (key == "--quick") o.quick = true; + else { + if (i + 1 == argc) throw std::invalid_argument("missing value for " + key); + const std::string value = argv[++i]; + if (key == "--dtype") o.dtype = value; + else if (key == "--csv") o.csv = value; + else if (key == "--json") o.json = value; + else if (key == "--batch" || key == "--seq" || key == "--heads" || key == "--dim") { + const auto v = positive(value, key.c_str()); + o.custom_shape = true; + if (key == "--batch") o.batch = v; + else if (key == "--seq") o.seq = v; + else if (key == "--heads") o.heads = v; + else { + if (v > 256) throw std::invalid_argument("dim is greater than 256"); + o.dim = static_cast(v); + } + } else if (key == "--repeats" || key == "--groups") { + const auto v = positive(value, key.c_str()); + if (v > 10000) throw std::invalid_argument("repeats/groups exceed 10000"); + if (key == "--repeats") o.repeats = static_cast(v); + else o.groups = static_cast(v); + } else throw std::invalid_argument("unknown argument " + key); + } + } + if (!o.validate && !o.benchmark) throw std::invalid_argument("specify --validate and/or --benchmark"); + if (o.dtype != "both" && o.dtype != "fp16" && o.dtype != "bf16") + throw std::invalid_argument("dtype must be fp16, bf16 or both"); + checked_shape(o.batch, o.seq, o.heads, o.dim); + return o; +} + +template std::vector make_input(std::size_t rows, int n, int pattern, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::normal_distribution normal(0.0f, 0.5f); + std::vector result(rows * n); + for (std::size_t i = 0; i < result.size(); ++i) { + float x = 0; + if (pattern == 0) x = uniform(rng); + else if (pattern == 1) x = normal(rng); + else if (pattern == 2) x = i % n == (i / n + seed) % n ? 8.0f : uniform(rng); + else if (pattern == 4) x = i % n == (i / n) % n ? 1.0f : 0.0f; + result[i] = rounded(x); + } + return result; +} + +struct Summary { + std::size_t cases = 0, elements = 0, exact_transform_elements = 0, contract_checks = 0; + std::size_t warp64_cases = 0, exact_warp64_elements = 0; + std::size_t warp64_grid_stride_cases = 0, warp64_grid_stride_elements = 0, unsupported_warp64_checks = 0; + double max_rounded_error = 0, max_unrounded_error = 0; +}; + +template void one_case(cudaStream_t stream, std::size_t rows, int n, float scale, + int pattern, unsigned seed, const char* dtype, Summary& summary) { + const auto input = make_input(rows, n, pattern, seed); + const std::size_t size = input.size(), bytes = rows * ((n + 1) / 2); + Guarded x(size, stream), baseline(size, stream), optimized(size, stream), inplace(size, stream); + Guarded base_split(bytes, stream), opt_split(bytes, stream), base_fused(bytes, stream), opt_fused(bytes, stream); + Guarded bs(rows, stream), os(rows, stream), bfs(rows, stream), ofs(rows, stream); + x.upload(input, stream); + inplace.upload(input, stream); + CHECK(api::transform(x.data(), baseline.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform(x.data(), optimized.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::transform(inplace.data(), inplace.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::quantize_int4(baseline.data(), base_split.data(), bs.data(), rows, n, stream, api::Method::Baseline)); + CHECK(api::quantize_int4(optimized.data(), opt_split.data(), os.data(), rows, n, stream, api::Method::Optimized)); + CHECK(api::transform_int4(x.data(), base_fused.data(), bfs.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform_int4(x.data(), opt_fused.data(), ofs.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(cudaStreamSynchronize(stream)); + x.download(stream, true); + const auto a = baseline.download(stream), b = optimized.download(stream), ip = inplace.download(stream); + const std::string context = std::string(dtype) + " rows=" + std::to_string(rows) + " n=" + std::to_string(n) + + " pattern=" + std::to_string(pattern) + " seed=" + std::to_string(seed) + " scale=" + std::to_string(scale); + if (std::memcmp(a.data(), b.data(), size * sizeof(T)) || std::memcmp(b.data(), ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/optimized/in-place transform not bitwise identical: " + context); + std::vector fx(size), actual(size); + for (std::size_t i = 0; i < size; ++i) { fx[i] = read(input[i]); actual[i] = read(b[i]); } + // 全部元素使用 O(N^2) FP64 稠密矩阵公式，不调用任何 FWHT 作为正确性 oracle。 + const auto dense = hadamard::dense_reference(fx, n, static_cast(scale)); + const double tolerance = std::string(dtype) == "fp16" ? 1e-2 : 5e-2; + for (std::size_t i = 0; i < size; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + const double error = std::abs(static_cast(actual[i]) - expected); + if (!std::isfinite(actual[i]) || !(error < tolerance)) + throw std::runtime_error("rounded FP64 dense oracle mismatch: " + context + " index=" + std::to_string(i) + + " error=" + std::to_string(error)); + summary.max_rounded_error = std::max(summary.max_rounded_error, error); + summary.max_unrounded_error = std::max(summary.max_unrounded_error, std::abs(actual[i] - dense[i])); + } + const auto expected_q = hadamard::quantize_int4(actual, n); + if (base_split.download(stream) != expected_q.packed || opt_split.download(stream) != expected_q.packed || + base_fused.download(stream) != expected_q.packed || opt_fused.download(stream) != expected_q.packed || + bs.download(stream) != expected_q.scales || os.download(stream) != expected_q.scales || + bfs.download(stream) != expected_q.scales || ofs.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/optimized split/fused INT4 bytes or scales mismatch: " + context); +#if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + Guarded warp_output(size, stream, "warp64.output"), warp_inplace(size, stream, "warp64.inplace"); + Guarded warp_split(bytes, stream, "warp64.split"), warp_fused(bytes, stream, "warp64.fused"); + Guarded warp_split_scales(rows, stream, "warp64.split-scales"), warp_fused_scales(rows, stream, "warp64.fused-scales"); + warp_inplace.upload(input, stream); + CHECK(api::transform(x.data(), warp_output.data(), rows, n, scale, stream, api::Method::Warp64)); + CHECK(api::transform(warp_inplace.data(), warp_inplace.data(), rows, n, scale, stream, api::Method::Warp64)); + CHECK(api::quantize_int4(warp_output.data(), warp_split.data(), warp_split_scales.data(), rows, n, stream, api::Method::Warp64)); + CHECK(api::transform_int4(x.data(), warp_fused.data(), warp_fused_scales.data(), rows, n, scale, stream, api::Method::Warp64)); + const auto warp = warp_output.download(stream), warp_ip = warp_inplace.download(stream); + if (std::memcmp(a.data(), warp.data(), size * sizeof(T)) || std::memcmp(a.data(), warp_ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/Warp64/in-place transform not bitwise identical: " + context); + if (warp_split.download(stream) != expected_q.packed || warp_fused.download(stream) != expected_q.packed || + warp_split_scales.download(stream) != expected_q.scales || warp_fused_scales.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/Warp64 split/fused INT4 bytes or scales mismatch: " + context); + x.download(stream, true, "after-warp64"); + ++summary.warp64_cases; + summary.exact_warp64_elements += size; +#endif + ++summary.cases; + summary.elements += size; + summary.exact_transform_elements += size; +} + +template void contract_tests(cudaStream_t stream, Summary& summary) { + Guarded input(64, stream, "contract.input"), output(64, stream, "contract.output"); + Guarded packed(32, stream, "contract.packed"); + Guarded scales(8, stream, "contract.scales"); + input.download(stream, true, "initialized"); + output.download(stream, true, "initialized"); + packed.download(stream, true, "initialized"); + scales.download(stream, true, "initialized"); + auto reject = [&](cudaError_t status) { + if (status != cudaErrorInvalidValue) throw std::runtime_error("invalid API input did not return cudaErrorInvalidValue"); + ++summary.contract_checks; + }; + auto success = [&](cudaError_t status) { CHECK(status); ++summary.contract_checks; }; + for (const auto method : active_methods()) { + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=invalid-parameters" << std::endl; + for (int n : {0, 3, 512}) { + reject(api::transform(input.data(), output.data(), 1, n, 1, stream, method)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, n, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, n, 1, stream, method)); + } + for (float scale : {0.0f, -1.0f, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { + reject(api::transform(input.data(), output.data(), 1, 8, scale, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, scale, stream, method)); + } + reject(api::transform(static_cast(nullptr), output.data(), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), static_cast(nullptr), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), input.data() + 1, 1, 8, 1, stream, method)); + reject(api::transform(input.data(), output.data(), std::numeric_limits::max(), 256, 1, stream, method)); + reject(api::quantize_int4(input.data(), reinterpret_cast(input.data()), scales.data(), 1, 8, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), nullptr, 1, 8, 1, stream, method)); + auto* odd = reinterpret_cast(reinterpret_cast(input.data()) + 1); + reject(api::transform(odd, output.data(), 1, 8, 1, stream, method)); + auto* bad_scale = reinterpret_cast(reinterpret_cast(scales.data()) + 2); + reject(api::quantize_int4(input.data(), packed.data(), bad_scale, 1, 8, stream, method)); + success(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, method)); + success(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, method)); + success(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, method)); + input.download(stream, true, "after-invalid-and-zero-rows"); + output.download(stream, true, "after-invalid-and-zero-rows"); + packed.download(stream, false, "after-invalid-and-zero-rows"); + scales.download(stream, false, "after-invalid-and-zero-rows"); + // 正负半整数：预期手写，避免舍入测试仅复用 CPU 参考实现。 + const std::vector ties{7, -7, .5f, 1.5f, 2.5f, -.5f, -1.5f, -2.5f}; + std::vector t(64, rounded(0)); + for (std::size_t i = 0; i < ties.size(); ++i) t[i] = rounded(ties[i]); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-upload" << std::endl; + input.upload(t, stream); + input.download(stream, true, "after-ties-upload"); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-quantize" << std::endl; + CHECK(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, method)); + const auto q = packed.download(stream, false, "after-ties-quantize"); + const auto s = scales.download(stream, false, "after-ties-quantize"); + const std::array expected{{0x97, 0x20, 0x02, 0xee}}; + if (!std::equal(expected.begin(), expected.end(), q.begin()) || s[0] != 1.0f) + throw std::runtime_error("positive/negative ties-to-even test failed"); + input.download(stream, true, "after-ties-quantize"); + ++summary.contract_checks; + } + const auto invalid = static_cast(-1); + reject(api::transform(input.data(), output.data(), 1, 8, 1, stream, invalid)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, invalid)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, invalid)); +#if !defined(HADAMARD_METAX_WARP64) || !defined(__MACACC__) + auto unsupported = [&](cudaError_t status) { + if (status != cudaErrorNotSupported) throw std::runtime_error("uncompiled Warp64 path did not return cudaErrorNotSupported"); + ++summary.unsupported_warp64_checks; + }; + unsupported(api::transform(input.data(), output.data(), 1, 8, 1, stream, api::Method::Warp64)); + unsupported(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, api::Method::Warp64)); + unsupported(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, api::Method::Warp64)); + CHECK(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, api::Method::Warp64)); + CHECK(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, api::Method::Warp64)); + CHECK(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, api::Method::Warp64)); + summary.unsupported_warp64_checks += 3; +#endif + CHECK(cudaStreamSynchronize(stream)); + output.download(stream, true); +} + + +template Summary validate(cudaStream_t stream, const char* dtype, const Options& options) { + Summary result; + const std::vector dims = options.custom_shape ? std::vector{options.dim} + : (options.quick ? std::vector{1, 64, 256} : std::vector{1, 2, 4, 8, 16, 32, 64, 128, 256}); + const std::vector rows = options.custom_shape + ? std::vector{checked_shape(options.batch, options.seq, options.heads, options.dim)} + : (options.quick ? std::vector{3} : std::vector{1, 3, 17, 257}); + for (int n : dims) { + for (auto r : rows) { + for (int normalized = 0; normalized < (n == 1 ? 1 : 2); ++normalized) { + const float scale = normalized ? 1.0f / std::sqrt(static_cast(n)) : 1.0f; + for (int pattern = 0; pattern < 5; ++pattern) { + // 零值/脉冲无随机性，只计一次；N=1 的两种 scale 相同，也只计一次。 + const int seeds = pattern < 3 && !options.quick ? 3 : 1; + for (int seed = 0; seed < seeds; ++seed) + one_case(stream, r, n, scale, pattern, 123 + 7919 * seed, dtype, result); + } + } + } + std::cout << "VALIDATION_PROGRESS dtype=" << dtype << " n=" << n << " cases=" << result.cases << std::endl; + } + if (!options.quick && !options.custom_shape) { + // 超过 65535 个 block 的网格上限，让同一个 block 必须处理下一行。 + for (int n : {1, 2}) one_case(stream, 65537, n, 1.0f, 0, 1847, dtype, result); +#if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + // 每个 Warp64 CTA 处理 4 行，262145 同时覆盖网格复用和不满 CTA 的尾行。 + for (int n : {1, 2}) { + one_case(stream, 262145, n, 1.0f, 0, 4541, dtype, result); + ++result.warp64_grid_stride_cases; + result.warp64_grid_stride_elements += 262145u * n; + } +#endif + } + contract_tests(stream, result); + std::cout << "VALIDATION_PASS dtype=" << dtype << " cases=" << result.cases << " elements=" << result.elements + << " max_rounded_error=" << std::setprecision(12) << result.max_rounded_error + << " max_unrounded_error=" << result.max_unrounded_error << " contract_checks=" << result.contract_checks + << " warp64_cases=" << result.warp64_cases << " exact_warp64_elements=" << result.exact_warp64_elements + << " warp64_grid_stride_cases=" << result.warp64_grid_stride_cases + << " unsupported_warp64_checks=" << result.unsupported_warp64_checks << std::endl; + return result; +} + +template void benchmark(cudaStream_t stream, const char* dtype, const Options& o, std::ofstream& csv) { + struct Shape { std::size_t b, s, h; int n; }; + std::vector shapes; + if (o.custom_shape) shapes.push_back({o.batch, o.seq, o.heads, o.dim}); + else for (int n : {64, 128, 256}) for (auto rows : {1, 17, 257, 4096, 16384}) + shapes.push_back({rows >= 4096 ? static_cast(rows / 1024) : 1, + rows >= 4096 ? 64 : static_cast(rows), rows >= 4096 ? 16u : 1u, n}); + struct Configuration { api::Method method; int operation; const char* name; }; + // 无 Warp64 宏时保留原来的 6 路顺序；启用后追加 3 路，独立输出新的原始样本。 + std::vector configurations{ + {api::Method::Baseline, 0, "baseline_transform"}, {api::Method::Optimized, 0, "optimized_transform"}, + {api::Method::Baseline, 1, "baseline_split"}, {api::Method::Optimized, 1, "optimized_split"}, + {api::Method::Baseline, 2, "baseline_fused"}, {api::Method::Optimized, 2, "optimized_fused"}}; +#if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + configurations.push_back({api::Method::Warp64, 0, "warp64_transform"}); + configurations.push_back({api::Method::Warp64, 1, "warp64_split"}); + configurations.push_back({api::Method::Warp64, 2, "warp64_fused"}); +#endif + const int configuration_count = static_cast(configurations.size()); + for (const auto shape : shapes) { + const auto rows = checked_shape(shape.b, shape.s, shape.h, shape.n), count = rows * shape.n; + const auto input = make_input(rows, shape.n, 0, 2909); + Guarded x(count, stream), y(count, stream); + Guarded q(rows * ((shape.n + 1) / 2), stream); + Guarded s(rows, stream); + x.upload(input, stream); + const float scale = 1.0f; + auto launch = [&](int which) { + const auto cfg = configurations[which]; + if (cfg.operation < 2) CHECK(api::transform(x.data(), y.data(), rows, shape.n, scale, stream, cfg.method)); + if (cfg.operation == 1) CHECK(api::quantize_int4(y.data(), q.data(), s.data(), rows, shape.n, stream, cfg.method)); + if (cfg.operation == 2) CHECK(api::transform_int4(x.data(), q.data(), s.data(), rows, shape.n, scale, stream, cfg.method)); + }; + for (int which = 0; which < configuration_count; ++which) for (int i = 0; i < 10; ++i) launch(which); + CHECK(cudaStreamSynchronize(stream)); + cudaEvent_t begin, end; + CHECK(cudaEventCreate(&begin)); CHECK(cudaEventCreate(&end)); + for (int group = 0; group < o.groups; ++group) { + // 各组轮换方法顺序；两端事件之间无分配、CPU 参考或主机设备复制。 + for (int order = 0; order < configuration_count; ++order) { + const int which = (order + group) % configuration_count; + const auto cfg = configurations[which]; + CHECK(cudaEventRecord(begin, stream)); + for (int i = 0; i < o.repeats; ++i) launch(which); + CHECK(cudaEventRecord(end, stream)); + CHECK(cudaEventSynchronize(end)); + float elapsed = 0; + CHECK(cudaEventElapsedTime(&elapsed, begin, end)); + const double us = static_cast(elapsed) * 1000.0 / o.repeats; + if (!(us > 0) || !std::isfinite(us)) throw std::runtime_error("invalid event timing"); + const std::size_t logical_bytes = cfg.operation == 0 ? count * sizeof(T) * 2 + : (cfg.operation == 1 ? count * sizeof(T) * 3 : count * sizeof(T)) + rows * ((shape.n + 1) / 2) + rows * sizeof(float); + csv << dtype << ',' << shape.b << ',' << shape.s << ',' << shape.h << ',' << shape.n << ',' << rows + << ',' << cfg.name << ',' << group << ',' << order << ',' << o.repeats << ',' << std::setprecision(12) << us + << ',' << logical_bytes << ',' << logical_bytes / us / 1000.0 << ',' << count * sizeof(T) + << ",2909,true,1\n"; + csv.flush(); + } + } + CHECK(cudaEventDestroy(begin)); CHECK(cudaEventDestroy(end)); + x.download(stream, true); + const auto output = y.download(stream); + const auto packed = q.download(stream); + const auto scales = s.download(stream); + std::vector actual(count); + for (std::size_t i = 0; i < count; ++i) actual[i] = read(output[i]); + const auto quantized = hadamard::quantize_int4(actual, shape.n); + if (packed != quantized.packed || scales != quantized.scales) + throw std::runtime_error("benchmark-size fused INT4 differs from CPU quantization"); + for (std::size_t row : {std::size_t(0), rows / 2, rows - 1}) { + std::vector sample(shape.n); + for (int i = 0; i < shape.n; ++i) sample[i] = read(input[row * shape.n + i]); + const auto expected = hadamard::dense_reference(sample, shape.n, scale); + for (int i = 0; i < shape.n; ++i) { + const double error = std::abs(actual[row * shape.n + i] - read(rounded(static_cast(expected[i])))); + if (!(error < (std::string(dtype) == "fp16" ? .01 : .05))) + throw std::runtime_error("benchmark-size sampled dense oracle mismatch"); + } + } + std::cout << "BENCHMARK_PROGRESS dtype=" << dtype << " rows=" << rows << " n=" << shape.n << std::endl; + } +} + +void write_summary(std::ostream& f, const char* dtype, const Summary& s) { + f << '"' << dtype << "\":{\"cases\":" << s.cases << ",\"elements\":" << s.elements + << ",\"exact_baseline_optimized_elements\":" << s.exact_transform_elements + << ",\"warp64_cases\":" << s.warp64_cases << ",\"exact_baseline_warp64_elements\":" << s.exact_warp64_elements + << ",\"warp64_grid_stride_cases\":" << s.warp64_grid_stride_cases + << ",\"warp64_grid_stride_elements\":" << s.warp64_grid_stride_elements + << ",\"unsupported_warp64_checks\":" << s.unsupported_warp64_checks + << ",\"max_abs_error_rounded_fp64\":" << std::setprecision(15) << s.max_rounded_error + << ",\"max_abs_error_unrounded_fp64\":" << s.max_unrounded_error + << ",\"api_contract_checks\":" << s.contract_checks << '}'; +} + +int main(int argc, char** argv) { + Options options; + try { options = parse(argc, argv); } + catch (const std::exception& e) { std::cerr << "INVALID_ARGUMENT " << e.what() << '\n'; return 2; } + try { + CHECK(cudaSetDevice(0)); + cudaDeviceProp prop{}; + CHECK(cudaGetDeviceProperties(&prop, 0)); +#if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + if (prop.warpSize != 64) throw std::runtime_error("Warp64 build requires a real device reporting warpSize=64"); +#endif + int runtime = 0, driver = 0; + CHECK(cudaRuntimeGetVersion(&runtime)); CHECK(cudaDriverGetVersion(&driver)); + std::cout << "DEVICE name=" << prop.name << " warp=" << prop.warpSize << " runtime=" << runtime << " driver=" << driver << std::endl; + cudaStream_t stream; + CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + Summary fp16, bf16; + if (options.validate) { + if (options.dtype != "bf16") fp16 = validate<__half>(stream, "fp16", options); + if (options.dtype != "fp16") bf16 = validate<__nv_bfloat16>(stream, "bf16", options); + std::ofstream json(options.json); + if (!json) throw std::runtime_error("cannot create validation JSON " + options.json); + json << "{\"status\":\"PASS\",\"full_matrix\":" << ((!options.quick && !options.custom_shape && options.dtype == "both") ? "true" : "false") +#if defined(HADAMARD_METAX_WARP64) && defined(__MACACC__) + << ",\"warp64_enabled\":true,\"methods\":[\"baseline\",\"optimized\",\"warp64\"]" +#else + << ",\"warp64_enabled\":false,\"methods\":[\"baseline\",\"optimized\"]" +#endif + << ",\"oracle\":\"all-element FP64 dense, rounded to output dtype\",\"fp16_tolerance_strict\":0.01,\"bf16_tolerance_strict\":0.05," + << "\"warmup_not_counted\":true,\"dtypes\":{"; + bool comma = false; + if (options.dtype != "bf16") { write_summary(json, "fp16", fp16); comma = true; } + if (options.dtype != "fp16") { if (comma) json << ','; write_summary(json, "bf16", bf16); } + json << "}}\n"; + if (!json) throw std::runtime_error("failed writing validation JSON"); + } + if (options.benchmark) { + std::ofstream csv(options.csv); + if (!csv) throw std::runtime_error("cannot create benchmark CSV " + options.csv); + csv << "dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale\n"; + if (options.dtype != "bf16") benchmark<__half>(stream, "fp16", options, csv); + if (options.dtype != "fp16") benchmark<__nv_bfloat16>(stream, "bf16", options, csv); + if (!csv) throw std::runtime_error("failed writing benchmark CSV"); + } + CHECK(cudaStreamDestroy(stream)); + std::cout << "PASS requested validation/benchmark operations completed" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "FAIL " << e.what() << std::endl; + return 1; + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/INTEGRATION.md b/03_hadamard_tc/a962695448-rgb/platforms/moore/INTEGRATION.md new file mode 100644 index 00000000..11aac69a --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/INTEGRATION.md @@ -0,0 +1,22 @@ +# 集成与复现 + +仓库：a962695448-rgb/Learning-CUDA，分支：feat/hadamard-cuda。 +移植基础提交：cb82ea6e1d5c8f78b48b6922b0f7af279696cc44。 + +本目录位于 03_hadamard_tc/a962695448-rgb/platforms/moore，使用原生 MUSA API、 +独立命名空间和构建入口。共享参考采用现有 include/reference.hpp；服务器测试副本 +比仓库原文件多一个末尾空行，代码内容一致，VERIFICATION.json 保留两份字节哈希。 + +在已安装 MUSA SDK 的 Linux 环境，从项目目录执行： + +~~~bash +python3 platforms/moore/run_platform.py --probe-only --output results/moore/probe-new +python3 platforms/moore/run_platform.py --shuffle32 --no-benchmark --output results/moore/validation-new +~~~ + +输出目录须为新目录。完整运行参数、性能计时范围和 INT4 语义见 README.md； +S4000 实测结果见 REPORT.zh-CN.md，算法推导见 QUANTIZATION_METHOD.md。 + +本次 GitHub 提交发布已完成实机验证的源码及文档。发布过程中服务器保持关机， +没有因提交操作重复运行 GPU 测试。完整原始文本、计时 CSV 和测试夹具保存在 +本地 hadamard-moore-optimized-20260916.zip 交付包中。 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/Makefile b/03_hadamard_tc/a962695448-rgb/platforms/moore/Makefile new file mode 100644 index 00000000..67318f78 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/Makefile @@ -0,0 +1,26 @@ +MUSA_ROOT ?= /usr/local/musa +MCC ?= $(MUSA_ROOT)/bin/mcc +ARCH ?= mp_22 +SHUFFLE32 ?= 0 +BUILD_DIR ?= build/$(ARCH)/shuffle$(SHUFFLE32) +CXXFLAGS := -std=c++17 -O2 -fno-fast-math -ffp-contract=off --offload-arch=$(ARCH) +CPPFLAGS := -I../../include +LDLIBS := -L$(MUSA_ROOT)/lib -lmusart +ifeq ($(SHUFFLE32),1) +CPPFLAGS += -DHADAMARD_MOORE_SHUFFLE32 +endif + +.PHONY: all probe +all: $(BUILD_DIR)/validate_and_benchmark + +$(BUILD_DIR): + mkdir -p $@ + +$(BUILD_DIR)/validate_and_benchmark: hadamard_api.mu hadamard_api.h exact_int4.hpp validate_and_benchmark.mu ../../include/reference.hpp | $(BUILD_DIR) + $(MCC) $(CXXFLAGS) $(CPPFLAGS) hadamard_api.mu validate_and_benchmark.mu $(LDLIBS) -o $@ + +$(BUILD_DIR)/probe: probe.mu | $(BUILD_DIR) + $(MCC) $(CXXFLAGS) probe.mu $(LDLIBS) -o $@ + +probe: $(BUILD_DIR)/probe + LD_LIBRARY_PATH=$(MUSA_ROOT)/lib:$(LD_LIBRARY_PATH) $(BUILD_DIR)/probe diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/OBSERVED_SERVER_STATUS.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/OBSERVED_SERVER_STATUS.json new file mode 100644 index 00000000..7aec4f30 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/OBSERVED_SERVER_STATUS.json @@ -0,0 +1,61 @@ +{ + "scope": "实机状态及验证摘要的历史快照，GitHub 提交状态以分支历史为准。", + "base_commit": "cb82ea6e1d5c8f78b48b6922b0f7af279696cc44", + "device": "MTT S4000", + "sdk": "MUSA 5.1.0", + "driver": "5.1.0-server", + "target_arch": "mp_22", + "reported_warp_size": 128, + "device_code_warp_size": 32, + "pipeline_status": "PASS", + "full_validation": { + "status": "PASS", + "platform": "moore", + "warp_size": 128, + "full_matrix": true, + "shuffle32_enabled": true, + "methods": [ + "baseline", + "optimized", + "shuffle32" + ], + "oracle": "all-element FP64 dense, rounded to output dtype", + "fp16_tolerance_strict": 0.01, + "bf16_tolerance_strict": 0.05, + "warmup_not_counted": true, + "dtypes": { + "fp16": { + "cases": 752, + "elements": 4891672, + "exact_baseline_optimized_elements": 4891672, + "shuffle32_cases": 752, + "exact_baseline_shuffle32_elements": 4891672, + "shuffle32_grid_stride_cases": 2, + "shuffle32_grid_stride_elements": 1572843, + "unsupported_shuffle32_checks": 0, + "max_abs_error_rounded_fp64": 0.00390625, + "max_abs_error_unrounded_fp64": 0.015620231628418, + "api_contract_checks": 96 + }, + "bf16": { + "cases": 752, + "elements": 4891672, + "exact_baseline_optimized_elements": 4891672, + "shuffle32_cases": 752, + "exact_baseline_shuffle32_elements": 4891672, + "shuffle32_grid_stride_cases": 2, + "shuffle32_grid_stride_elements": 1572843, + "unsupported_shuffle32_checks": 0, + "max_abs_error_rounded_fp64": 3.814697265625e-06, + "max_abs_error_unrounded_fp64": 0.124984741210938, + "api_contract_checks": 96 + } + } + }, + "cli_rejection_cases": 14, + "paired_observations": 6750, + "power_state_at_delivery": "OFF_CONFIRMED", + "text_files_verified": 77, + "fixture_verified_locally": true, + "all_server_binaries_downloaded": false +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/QUANTIZATION_METHOD.md b/03_hadamard_tc/a962695448-rgb/platforms/moore/QUANTIZATION_METHOD.md new file mode 100644 index 00000000..12cb584f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/QUANTIZATION_METHOD.md @@ -0,0 +1,55 @@ +# Exact INT4 comparison optimization + +The target contract is `clamp(RNE_integer(RNE_float32(value / scale)), -7, 7)`. +The public scale remains the original correctly rounded FP32 `max(abs(x))/7`. +Only per-element quantization in Optimized and Shuffle32 changes; Baseline +retains the original native division implementation. + +## Why ordinary thresholds are insufficient + +The first rounding step maps an interval of exact ratios to each binary32 +half-integer. The following integer tie must go to the even integer. Therefore, +comparing only against `(k + 0.5) * scale` would not preserve the contract. + +For the transition between positive integers `k` and `k+1`: + +- If `k` is even, transition only above `k+0.5 + half_ulp(k+0.5)`. +- If `k` is odd, transition at or above `k+0.5 - half_ulp(k+0.5)`. + +The half-integers have even binary32 significands. At k=0, the relevant upper +half-ULP is 2^-25. For k=1, k=2..3, and k=4..6 the half-ULP is respectively +2^-24, 2^-23, and 2^-22. Expressing all boundaries over 2^25 gives integer +numerators no larger than 218103816. Multiplication by a 24-bit input +significand fits in 52 bits. The implementation uses a three-step binary search +over these seven increasing boundaries and restores the sign at the end. + +The exponent handling also represents binary32 subnormals. A negative alignment +shift means the value is below every transition. A shift of at least 30 means +it is above every transition; these cases avoid unsafe shifts. The remaining +left shifts occupy at most 53 bits. This proof assumes finite value and finite +positive scale in the row-quantization contract; it does not define NaN handling. + +## Validation boundaries + +The actual portable C++ implementation is tested against the existing project +CPU divide/round helper, not against another copy of the integer algorithm. +Checks include all finite nonnegative FP16/BF16 encodings under selected row +maxima, sign reversals, signed zero, subnormals and binary32 rounding-boundary +neighbors. There are 17,527,208 CPU checks; they are not a claim of exhaustive +coverage of every possible pair of binary32 inputs. + +The 2,074,568-record GPU fixture is a subset of these classes and must be checked +against both the original native `__fdiv_rn` path and independently generated +CPU labels. GPU and CPU counts overlap and must not be added together. + +The paired benchmark links the frozen control in a separate namespace and the +candidate in one executable. Their inputs, output checks, stream and timing +protocol are shared. Three independent benchmark processes rotate method order. +Both full Hadamard regression and the disabled-Shuffle32 compatibility build +must pass before the candidate is accepted. + +The acceptance target is the default Optimized split/fused INT4 paths over the +existing 30 shape/dtype configurations. Each must preserve exact results and +reduce event time by at least 5% in all three paired rounds. This criterion is +recorded before inspecting the paired results. Transform-only and optional +Shuffle32 results are reported in full, including regressions. diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/README.md b/03_hadamard_tc/a962695448-rgb/platforms/moore/README.md new file mode 100644 index 00000000..b390ef43 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/README.md @@ -0,0 +1,129 @@ +# Moore Threads MTT S4000 / MUSA + +Native MUSA adaptation of the Hadamard transform and INT4 fusion. The source +contract and the independent FP64 dense reference are inherited from the +project at `cb82ea6e1d5c8f78b48b6922b0f7af279696cc44`; performance and correctness +must be measured independently on this device. + +## Verified delivery status + +The exact INT4 optimization passed native S4000 validation on 2026-09-16. +Default Optimized fusion is 2.012–2.898 times as fast as its previous +implementation across all 30 tested shape/dtype configurations and all three +paired rounds. Optional Shuffle32 fusion is 2.884–7.781 times as fast as its own +previous implementation. These are old/new comparisons from the same executable, +not comparisons with a different platform or an application-level speedup. + +The actual integer implementation passed 17,527,208 CPU comparisons. A +2,074,568-record overlapping GPU fixture exactly matched both CPU labels and +the original native division path. Full validation passed 1,504 inputs, 192 API +checks and 14 CLI rejection checks. INT4 bytes and scales remain exact. + +Three paired processes produced 6,750 original event measurements. Default +Optimized split/fused paths reduce time by at least 5% in every measured round +and configuration. Transform-only times have no consistent improvement. +Shuffle32 still loses to Optimized on some small shapes, so the default remains +Optimized. See [the Chinese report](REPORT.zh-CN.md), +[algorithm derivation](QUANTIZATION_METHOD.md), and +[verification scope and hashes](VERIFICATION.json). + +All 77 optimization text files and the byte-identical fixture are backed up +locally. Six Moore source files match the GPU run byte-for-byte. The shared +reference in GitHub differs from the tested copy only by one trailing blank +line; VERIFICATION.json records both hashes. +The previous port's 145 text files are preserved separately. Seven new build +binaries and ten previous executables were not downloaded; the stopped server +has a finite retention period. Source, logs, raw CSVs and fixture are included +in the local optimized delivery ZIP. The core recorded measurements, frozen +control, fixture generator and offline analysis are now available in the +[public reproduction directory](repro/README.zh-CN.md). + +The initial device probe compiled and executed on MTT S4000, MUSA SDK/mcc 5.1.0, +driver 5.1.0-server, target `mp_22`. Host device properties report warp size 128; +device-code `warpSize` is 32. The diagnostic observed zero errors for width-32 +shuffle, 256 errors for width-64, and 512 for width-128. All 512 simple FP16/BF16 +conversion checks passed. Those wider shuffle experiments are retained as +negative evidence and are not enabled in the implementation. + +The first quick run found a one-ULP row-scale difference for FP16 input +`0.5009765625 / 7`: ordinary device division produced bits `1033015882`, while +the CPU reference produced `1033015881`. An additional compiler divide/sqrt +flag did not change that case. Explicit native `__fdiv_rn` closed the difference +and passed the quick matrix without changing reference tolerances. A permanent +scale witness now exercises all three methods and both dtypes. + +## API and implementation + +- `hadamard_api.h` declares asynchronous `transform`, `quantize_int4`, and + `transform_int4`, using native `musaStream_t`, `__half`, and `__mt_bfloat16`. +- Inputs are contiguous `[rows, dim]`; a four-dimensional tensor is flattened + as `rows = batch * seq * heads`. `dim` supports powers of two from 1 to 256. +- Storage is FP16/BF16; arithmetic is FP32. Fused quantization first rounds the + transform to its public storage type. INT4 is symmetric `[-7,7]`, nearest-even, + with even elements in low nibbles and one FP32 scale per row. +- `Baseline` uses one thread per element with two barriers per butterfly stage. +- `Optimized`, the default, gives each thread exclusive ownership of a butterfly + pair and uses one barrier per stage. It makes no warp-width assumption. +- Optional `Shuffle32` uses 32-lane logical row groups and local-register higher + butterfly stages. All 256 threads execute each exchange even when the last + row batch is incomplete. It is gated by a dedicated device probe. +- Optimized and Shuffle32 quantization use exact integer comparisons that + preserve both binary32 quotient rounding and ties-to-even integer rounding. + Row scales retain native correctly rounded division; Baseline retains the + original per-element division path. See exact_int4.hpp. +- The API does not allocate, copy, or synchronize. Exact in-place transformation + is supported; partial overlap and quantization buffer overlap are rejected. + The caller must supply valid, adequately sized buffers on the current device. +- FP8, autograd, arbitrary strides, and dimensions above 256 are outside this port. + +## Reproduce + +Run from this project's `03_hadamard_tc/a962695448-rgb` directory on Linux with +the installed MUSA SDK. The runner changes environment variables only for its +subprocesses; it does not replace the driver or framework. + +```bash +python3 platforms/moore/run_platform.py --probe-only --output results/moore/probe-new +python3 platforms/moore/run_platform.py --quick --shuffle32 --no-benchmark --output results/moore/quick-new +python3 platforms/moore/run_platform.py --shuffle32 --output results/moore/full-new +``` + +Each output directory must be new. Without `--shuffle32`, the runner tests the +baseline/shared paths and verifies that unavailable Shuffle32 calls fail clearly. +The default full run performs three separate benchmark processes, five timing +groups per method/configuration, and 100 calls per event interval. + +The commands above reproduce validation and the current-version baseline +benchmark. The [public reproduction guide](repro/README.zh-CN.md) contains the +old/new paired harness, frozen control, raw measurements and CPU fixture +generator. Offline reanalysis uses only the Python standard library. The paired +6,750-row dataset and older 4,050-row baseline dataset are separate experiments. + +## Validation and timing boundaries + +The matrix covers both dtypes, all supported dimensions, multiple row counts, +normalization, seeded uniform/normal/outlier data, zero/impulse inputs, guarded +and minimally aligned buffers, exact in-place behavior, non-default streams, +large grid-stride tails, API rejection cases, and 14 invalid CLI cases. The +oracle computes an independent FP64 dense Hadamard product and rounds to the +public output dtype. FP16/BF16 strict absolute tolerances remain 0.01/0.05. +CPU, separate, and fused INT4 bytes/scales must agree exactly. Hand-written +rounding witnesses include positive/negative ties and `[0.75, 0.25] -> 0x37`. + +Every benchmark method is checked before timing and after its final timing +group. Large timing shapes use three independently checked dense-reference rows +plus full-array method equality and INT4 checks; this is stated separately from +the all-element dense oracle used in the validation matrix. + +MUSA event timing excludes allocation, host/device transfers and output checks. +It can include device idle gaps caused by host submission; it is not application +latency. CSV provides microseconds and milliseconds. Logical I/O throughput is an +estimate, not measured physical bandwidth. Slower cases remain in the report. + +## Sources + +- [MUSA 5.1 MCC manual](https://docs.mthreads.com/musa-sdk/version-5.1.0/toolkits/mcc_compiler/) +- [MUSA warp functions](https://docs.mthreads.com/en/musa-sdk/musa-sdk-doc-online/programming_guide/musa_cpp_syntax/warp_functions/) +- `SOURCE_PROVENANCE.json` records the original source and local snapshot hashes. + +Results apply to the recorded MUSA 5.1.0 / S4000 environment and measured input matrix. diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/REPORT.zh-CN.md b/03_hadamard_tc/a962695448-rgb/platforms/moore/REPORT.zh-CN.md new file mode 100644 index 00000000..cd667bc7 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/REPORT.zh-CN.md @@ -0,0 +1,104 @@ +# Hadamard：摩尔线程 S4000 适配与 INT4 优化报告 + +实机优化与本地归档：2026-09-16（北京时间）。 + +## 结论 + +原生 MUSA 版本完成一轮精确 INT4 量化优化，并通过完整回归。默认 Optimized 的融合路径相对上一版同一路径，在 30 个形状/精度配置、三轮配对测试中的加速比为 **2.012～2.898×**；全部配置每轮降时均超过 5%。可选 Shuffle32 融合路径相对它自己的旧版为 **2.884～7.781×**。 + +这两个范围比较的是本轮同一程序内的旧版与新版。此前适配报告的“GPU 基线 / 候选”比值属于另一组实验，不能与本轮范围相乘。公开 API、默认方法、输出精度和量化舍入契约保持原样。 + +GPU 测试与备份结束后，已按用户要求关机；本地收尾只整理文档和交付文件。本轮针对 Hadamard 的 MUSA 路径，未对 NineToothed CPU 解释器项目产生新的优化或实机验证结论。 + +## 实现变化与适用范围 + +- 基础来源：a962695448-rgb/Learning-CUDA，feat/hadamard-cuda，固定提交 cb82ea6e1d5c8f78b48b6922b0f7af279696cc44。 +- 设备及环境：MTT S4000，48 GiB 规格，MUSA SDK/mcc 5.1.0，驱动 5.1.0-server，目标 mp_22。 +- FP16/BF16 存储、内部 FP32；连续四维输入展平为 rows × dim，dim 支持 1～256 的二次幂。支持原位变换、调用方 stream 和打包 INT4 输出。 +- Optimized 与 Shuffle32 的逐元素量化改为 exact_int4.hpp 中的整数位运算与三次边界比较，替换逐元素浮点精确除法及取整。普通 Baseline 保留原算法，作为对照。 +- 每行公开 scale 仍使用原生 __fdiv_rn(max(abs(x)), 7)，零行 scale 为 1。融合路径先将变换结果舍入到公开存储类型，再量化。 + +目标契约是 clamp(RNE_integer(RNE_float32(value / scale)), -7, 7)，RNE 表示就近舍入、正中间取偶数。直接比较通常的半整数阈值会漏掉第一次 FP32 舍入的区间。实现显式纳入这些区间，利用 binary32 的尾数和指数将比较转换为最多 53 位的无符号整数运算；有限输入和正有限 scale 的推导见 [QUANTIZATION_METHOD.md](QUANTIZATION_METHOD.md)。 + +FP8、autograd、任意步长、dim > 256 和 MUSA MMA/Tensor Core 实现仍不在本移植范围内。没有把接口移植称为这些功能已经完成。 + +## 正确性验证 + +| 验证层次 | 结果 | 说明 | +|---|---:|---| +| CPU 实际 C++ 算法对比 | 17,527,208 项，零差异 | 包括所选行最大值下的有限 FP16/BF16 编码、正负数、零、次正规数及 980,000 个边界检查 | +| S4000 GPU 量化原语 | 2,074,568 项，零差异 | CPU 标签、旧版原生精确除法、新版整数算法三方一致 | +| Hadamard 完整矩阵 | 1,504 组输入通过 | 每种精度 752 组、4,891,672 个元素 | +| API / CLI | 192 项 / 14 项通过 | API 契约及非法 CLI 输入拒绝 | +| 关闭 Shuffle32 构建 | 50 组快速输入通过 | 另含 130 项 API、12 项不可用路径行为检查 | + +CPU 检查在 macOS（NumPy 2.3.5）与服务器 Linux（NumPy 1.23.5）分别执行，生成的 GPU 夹具字节和 SHA-256 完全一致。GPU 夹具与 CPU 检查类别重叠，不将两者相加当成独立用例总量；CPU 检查也不是全部 binary32 输入对的穷举。 + +完整矩阵中所有变换方法位一致；CPU、分步及融合的 INT4 字节和 scales 全量一致。对舍入后独立 FP64 稠密参考的最大绝对误差为 FP16 0.00390625、BF16 0.000003814697265625，分别满足严格阈值 < 0.01、< 0.05。未舍入 FP64 参考的误差为 0.015620231628418 / 0.124984741210938，包含公开存储精度的舍入误差，不能套用前述验收阈值。 + +多方法复用同一输入、关闭候选的兼容性检查以及多轮重复均不累计为更多独立完整矩阵输入。 + +## 配对性能结果 + +旧版冻结到独立命名空间，与新版链接进同一个可执行程序。对照源码除命名空间和头文件名外，与先前实测版本一致；程序使用相同输入、stream 和正确性检查，并轮换方法顺序。 + +矩阵为两种精度 × rows={1,17,257,4096,16384} × dim={64,128,256}，共 30 个配置。每个配置包含 15 个方法/操作组合；每轮 5 个分组、每组 100 次调用，三个独立进程共产生 **6,750 条原始计时**。每轮结果取五个分组平均耗时的中位数。 + +比值为旧版同一路径耗时 / 新版同一路径耗时，小于 1 表示新版更慢；范围包含所有配置和三轮结果。 + +| 方法 | 操作 | 旧版 / 新版耗时 | 三轮均更快 | 三轮均降时至少 5% | +|---|---|---:|---:|---:| +| 共享内存 Optimized（默认） | 变换 | 0.986–1.014× | 3/30 | 0/30 | +| 共享内存 Optimized（默认） | 分步变换 + INT4 | 1.848–2.759× | 30/30 | 30/30 | +| 共享内存 Optimized（默认） | 融合变换 + INT4 | 2.012–2.898× | 30/30 | 30/30 | +| Shuffle32（可选） | 变换 | 0.985–1.011× | 1/30 | 0/30 | +| Shuffle32（可选） | 分步变换 + INT4 | 2.611–7.345× | 30/30 | 30/30 | +| Shuffle32（可选） | 融合变换 + INT4 | 2.884–7.781× | 30/30 | 30/30 | + +### 代表配置 + +单位为微秒；每格为三轮中位数的中位数，表内比值由对应两格计算。完整分组数据与各轮结果保留在交付包中。 + +| 精度 / rows / dim | 共享融合旧 → 新（µs） | 比值 | Shuffle32 融合旧 → 新（µs） | 比值 | +|---|---:|---:|---:|---:| +| fp16 / 16384 / 256 | 1824.049 → 631.357 | 2.889× | 2902.644 → 391.899 | 7.407× | +| bf16 / 16384 / 256 | 1823.585 → 631.601 | 2.887× | 3042.024 → 391.056 | 7.779× | +| fp16 / 1 / 64 | 58.662 → 28.538 | 2.056× | 89.245 → 29.746 | 3.000× | +| bf16 / 17 / 256 | 61.593 → 30.245 | 2.036× | 301.219 → 61.470 | 4.900× | + +纯变换路径的时间变化约在 ±1.5% 内，没有稳定改善；本轮收益来自含量化的路径。Shuffle32 在大行数配置下表现更好，但 BF16 / rows=17 / dim=256 的新版融合仍需 61.470 µs，高于共享实现的 30.245 µs，因此默认仍选 Optimized。 + +验收目标预先限定为：默认 Optimized 的分步、融合路径在所有 30 个配置的三轮中保持精确结果且降时至少 5%。本轮满足该目标。三轮重复及工程阈值不构成统计显著性证明。 + +计时采用热输入上的 MUSA event 区间，排除分配、复制和输出检查，可能包含主机发射造成的设备空隙。结果不代表应用端到端延迟，也不能推断 CPU、NVIDIA 或其他国产设备上的加速比。没有使用硬件计数器证明唯一性能瓶颈。 + +## 原生适配的保留回归 + +1. 主机属性报告 warp size=128，设备代码 warpSize=32。独立诊断中 width=32 零错误，width=64/128 分别出现 256/512 个错误。实现只启用已验证的 32-lane 交换并处理尾行参与；最终探针的 1,280 项交换检查通过。 +2. FP16 值 0.5009765625 / 7 曾出现普通设备除法与 CPU 一 ULP 的 scale 差异。显式 __fdiv_rn 修复了已测见证；本轮仍保留该 scale 计算和永久回归，没有放宽容差。 + +## 证据、复现与交付范围 + +公开复现入口：[repro/README.zh-CN.md](repro/README.zh-CN.md)。其中包含 14 份原始结果文本、冻结旧版对照、配对计时程序、CPU/GPU 量化对照源码和可重新生成夹具的入口；[MANIFEST.json](repro/MANIFEST.json) 记录来源与哈希。三轮 CSV 和主要验收清单已可直接从仓库取得，其他历史归档仍保留在本地交付包。 + +从项目目录执行以下命令，可离线重算全部配对结果，输出目录须为新目录： + +~~~bash +python3 platforms/moore/repro/analyze_optimization.py --output /tmp/moore-analysis-new +~~~ + +本次公开整理没有改变 GPU 内核，也没有重新租用或启动 GPU。使用仓库原参考头文件重新编译的 CPU 对照生成了与原实机输入字节一致的夹具；公开复算器的全部数值结果与原分析一致。 + +以下保留完整本地交付包的索引，便于追溯其余文件： + +- 最新源码及实机哈希：本目录及 [VERIFICATION.json](VERIFICATION.json)。六份 Moore 执行相关源文件与实测版本字节一致；仓库共享参考比实测副本少一个末尾空行，代码内容一致，两份字节哈希分别记录。 +- 交付 ZIP 的 optimization/evidence/raw：本轮 77 份已逐字节校验的源码、日志、清单与 CSV。 +- 完整矩阵和关闭候选的原始清单：上述目录中的 runs/full/run_summary.json、runs/disabled/run_summary.json。 +- 三轮配对数据：runs/paired-round1.csv 至 paired-round3.csv；本轮 analysis.json 和 paired-comparisons.csv 位于 ZIP 的 optimization/analysis。 +- 离线复算：在解压包运行 python3 optimization/analyze_optimization.py。算法推导的验收标准补充在 optimization/METHOD.md；服务器原始 METHOD.md 保持原样。 +- GPU 夹具：optimization/quant_fixture.bin，共 24,894,816 字节；SHA-256 为 b404de04128361ed75347666113cfe2545f7c6e52aa828b070c1812c967ce230。 +- 原移植阶段的 145 份文本证据及先前离线报告保存在 ZIP 的 history 目录，供追溯。不要把旧版性能结论当成当前实现的计时。 + +本轮七个 Linux 构建二进制、先前十个可执行文件及原始服务器 tar.gz 没有下载；源文件、全部文本结果和精确匹配的测试夹具已本地保存。服务器已关机，平台关机数据保留期为 36 小时，不能把服务器副本视为长期备份。账户及关机截图只在本地交付包的 evidence 中，不加入源码补丁。 + +本目录将已验证的原生 MUSA 源码、运行入口、中文报告和验证摘要集成到 feat/hadamard-cuda 分支。完整原始数据及夹具保存在本地交付包，实际提交信息以 GitHub 历史为准。发布时仅更新文档和状态说明，没有重新执行 GPU 测试。 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/SOURCE_PROVENANCE.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/SOURCE_PROVENANCE.json new file mode 100644 index 00000000..edf80449 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/SOURCE_PROVENANCE.json @@ -0,0 +1,21 @@ +{ + "base_repository": "https://github.com/a962695448-rgb/Learning-CUDA", + "base_commit": "cb82ea6e1d5c8f78b48b6922b0f7af279696cc44", + "note": "原始源码由已连接的 GitHub 读取；MUSA 实现和共享参考的实机字节哈希见 VERIFICATION.json。", + "source_files": { + "platforms/metax/hadamard_api.h": "26705d5a10487fce2588d52fe6d4d10a7953c7159d220874ab450d26564ec147", + "platforms/metax/hadamard_api.cu": "afb08180a1fb9e1cb567b2c5f0fbeaa20449056675cf9fa7e204d24c4f7fbdc5", + "platforms/metax/validate_and_benchmark.cu": "d9049bba5f8df7ef2a37572ed31dc230724bf394bb1c732bc5a893b6826fb065", + "platforms/metax/run_platform.py": "6a11acfd6679b1493df0b8518c0ece1ba34c30f330206cebde7ac88f2f7056c3", + "include/reference.hpp": "1ff2c70cc8849020bd413a1d174718683bd89d5b6d8f25ee0a01d1c9b8876888" + }, + "optimization": { + "name": "exact_int4_integer_comparisons", + "date_asia_shanghai": "2026-09-16", + "contract_reference_sha256": "1ff2c70cc8849020bd413a1d174718683bd89d5b6d8f25ee0a01d1c9b8876888", + "previous_hadamard_api_sha256": "5d0b7b3db74bc3649831269cbb4736e731cf95e2b6baa270658c529475e17776", + "tested_hadamard_api_sha256": "f9c7fb325cbdc19e57e5c02ce58ec973197b75039fff38eb4b1fbac4daec4a80", + "verification": "VERIFICATION.json" + }, + "preparation_method": "从固定提交读取源码后在独立本地快照中准备 MUSA 移植，实机验证后集成到项目分支。" +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/VERIFICATION.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/VERIFICATION.json new file mode 100644 index 00000000..17c07ae3 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/VERIFICATION.json @@ -0,0 +1,237 @@ +{ + "status": "VERIFIED_AND_ACCEPTED", + "verified_date_asia_shanghai": "2026-09-16", + "base_commit": "cb82ea6e1d5c8f78b48b6922b0f7af279696cc44", + "pipeline_started_utc": "2026-09-15T16:42:52Z", + "pipeline_finished_utc": "2026-09-15T16:47:23Z", + "text_export_sha256": "dbb499c30755c376894eb93022919df4c3a426758e0cf727a80e57e86ad13d01", + "text_files_verified": 77, + "tested_source_sha256": { + "platforms/moore/hadamard_api.h": "cba4d5c187defec8f70083a02ba252da8d73f5fba5b1b7692efb91dd1a0c4d93", + "platforms/moore/hadamard_api.mu": "f9c7fb325cbdc19e57e5c02ce58ec973197b75039fff38eb4b1fbac4daec4a80", + "platforms/moore/validate_and_benchmark.mu": "2a23a7938bd3140742b3c45f6b85db08d539bf03d4cb44ee9a48be41a268459e", + "platforms/moore/probe.mu": "2e777447c5153b55a7eee8bd7729170517637c4e04cb3cdbe89c6c51ec58bccd", + "platforms/moore/run_platform.py": "d92b236ca4bcaff910921e2d0ef21f452968bb66f1bd98a1a157c0a7c8a88773", + "platforms/moore/exact_int4.hpp": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a", + "include/reference.hpp": "1ff2c70cc8849020bd413a1d174718683bd89d5b6d8f25ee0a01d1c9b8876888" + }, + "recorded_makefile_sha256": "ccd363d6e1152d1fb3e47f856ecf05bbd431519329effb9f46924cc434e4917c", + "makefile_note": "The Makefile was an experiment input; actual GPU builds used run_platform.py and explicit mcc commands.", + "cpu_checks": 17527208, + "gpu_fixture_checks": 2074568, + "cpu_gpu_counts_overlap": true, + "fixture_bytes": 24894816, + "fixture_sha256": "b404de04128361ed75347666113cfe2545f7c6e52aa828b070c1812c967ce230", + "fixture_verified_locally": true, + "full_validation": { + "status": "PASS", + "platform": "moore", + "warp_size": 128, + "full_matrix": true, + "shuffle32_enabled": true, + "methods": [ + "baseline", + "optimized", + "shuffle32" + ], + "oracle": "all-element FP64 dense, rounded to output dtype", + "fp16_tolerance_strict": 0.01, + "bf16_tolerance_strict": 0.05, + "warmup_not_counted": true, + "dtypes": { + "fp16": { + "cases": 752, + "elements": 4891672, + "exact_baseline_optimized_elements": 4891672, + "shuffle32_cases": 752, + "exact_baseline_shuffle32_elements": 4891672, + "shuffle32_grid_stride_cases": 2, + "shuffle32_grid_stride_elements": 1572843, + "unsupported_shuffle32_checks": 0, + "max_abs_error_rounded_fp64": 0.00390625, + "max_abs_error_unrounded_fp64": 0.015620231628418, + "api_contract_checks": 96 + }, + "bf16": { + "cases": 752, + "elements": 4891672, + "exact_baseline_optimized_elements": 4891672, + "shuffle32_cases": 752, + "exact_baseline_shuffle32_elements": 4891672, + "shuffle32_grid_stride_cases": 2, + "shuffle32_grid_stride_elements": 1572843, + "unsupported_shuffle32_checks": 0, + "max_abs_error_rounded_fp64": 3.814697265625e-06, + "max_abs_error_unrounded_fp64": 0.124984741210938, + "api_contract_checks": 96 + } + } + }, + "cli_rejection_cases": 14, + "disabled_validation": { + "status": "PASS", + "platform": "moore", + "warp_size": 128, + "full_matrix": false, + "shuffle32_enabled": false, + "methods": [ + "baseline", + "optimized" + ], + "oracle": "all-element FP64 dense, rounded to output dtype", + "fp16_tolerance_strict": 0.01, + "bf16_tolerance_strict": 0.05, + "warmup_not_counted": true, + "dtypes": { + "fp16": { + "cases": 25, + "elements": 9615, + "exact_baseline_optimized_elements": 9615, + "shuffle32_cases": 0, + "exact_baseline_shuffle32_elements": 0, + "shuffle32_grid_stride_cases": 0, + "shuffle32_grid_stride_elements": 0, + "unsupported_shuffle32_checks": 6, + "max_abs_error_rounded_fp64": 0, + "max_abs_error_unrounded_fp64": 0.00782299041748047, + "api_contract_checks": 65 + }, + "bf16": { + "cases": 25, + "elements": 9615, + "exact_baseline_optimized_elements": 9615, + "shuffle32_cases": 0, + "exact_baseline_shuffle32_elements": 0, + "shuffle32_grid_stride_cases": 0, + "shuffle32_grid_stride_elements": 0, + "unsupported_shuffle32_checks": 6, + "max_abs_error_rounded_fp64": 0, + "max_abs_error_unrounded_fp64": 0.062225341796875, + "api_contract_checks": 65 + } + } + }, + "paired_benchmark_observations": 6750, + "paired_benchmark_rounds": 3, + "paired_benchmark_summary": [ + { + "method": "optimized", + "operation": "transform", + "configurations": 30, + "minimum_speedup": 0.9856188401474171, + "maximum_speedup": 1.0142284761551534, + "faster_every_round": 3, + "at_least_five_percent_every_round": 0 + }, + { + "method": "optimized", + "operation": "split", + "configurations": 30, + "minimum_speedup": 1.8477204187342988, + "maximum_speedup": 2.759253803464851, + "faster_every_round": 30, + "at_least_five_percent_every_round": 30 + }, + { + "method": "optimized", + "operation": "fused", + "configurations": 30, + "minimum_speedup": 2.0122778318800956, + "maximum_speedup": 2.897511084201142, + "faster_every_round": 30, + "at_least_five_percent_every_round": 30 + }, + { + "method": "shuffle32", + "operation": "transform", + "configurations": 30, + "minimum_speedup": 0.9850463266345918, + "maximum_speedup": 1.0106747041757898, + "faster_every_round": 1, + "at_least_five_percent_every_round": 0 + }, + { + "method": "shuffle32", + "operation": "split", + "configurations": 30, + "minimum_speedup": 2.61086478258448, + "maximum_speedup": 7.345290638104018, + "faster_every_round": 30, + "at_least_five_percent_every_round": 30 + }, + { + "method": "shuffle32", + "operation": "fused", + "configurations": 30, + "minimum_speedup": 2.8836821850668595, + "maximum_speedup": 7.780916092345045, + "faster_every_round": 30, + "at_least_five_percent_every_round": 30 + } + ], + "paired_csv_sha256": { + "paired-round1.csv": "90a10dcfe163bb44425ada09f28e84c6a5cf0159872728b2b0a2975ceadd0cdb", + "paired-round2.csv": "a90fc97aee1ede505c0c83b86d5369bba8931ead53149a3a4e5c36a506dbb363", + "paired-round3.csv": "a9846fc8bb1d8efba317b3aba1929a5cf5102335833c2e2799475653a6eee033" + }, + "acceptance": "Default Optimized split/fused: exact results and at least 5% time reduction in all three rounds for every one of 30 shape/dtype configurations.", + "acceptance_passed": true, + "new_binary_build_artifacts_not_downloaded": [ + { + "path": "libcheck_quant.dylib", + "bytes": 15456, + "sha256": "560b68ba97d02f0076eba1f3b56a40232bf49fe6c76513a3eb6adab7ac64b805" + }, + { + "path": "runs/disabled/probe", + "bytes": 25872, + "sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0" + }, + { + "path": "runs/disabled/validate_and_benchmark", + "bytes": 331272, + "sha256": "593faf17d6b34c45ff826b103ceb2957d71cd9befdaa51ecb0cfa8312c6e141a" + }, + { + "path": "runs/full/probe", + "bytes": 25872, + "sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0" + }, + { + "path": "runs/full/validate_and_benchmark", + "bytes": 1010176, + "sha256": "1155d721fe4e48c51bcde8c6fa0cf1f962be313e742ea102962c0a3353a91cd8" + }, + { + "path": "runs/paired_benchmark", + "bytes": 2720056, + "sha256": "151500968b461a024b62de40a0c80adf6d4d42f9b8b566335c170494cbb2e38f" + }, + { + "path": "runs/quant_gpu", + "bytes": 34752, + "sha256": "77178f5e0a0a7cd41be4a2c25de63a578e23b48a1d9a3a1162482ed97994e3fb" + } + ], + "prior_port_text_files_verified": 145, + "prior_port_binary_executables_not_downloaded": 10, + "server_power_state_at_delivery": "OFF_CONFIRMED", + "server_retention_hours_after_stop": 36, + "post_test_changes": "发布阶段仅更新文档和状态说明；六份 Moore 执行相关源文件与实测字节一致。共享参考的末尾空行差异单独记录。", + "repository_reference": { + "path": "03_hadamard_tc/a962695448-rgb/include/reference.hpp", + "git_blob_sha": "e06ce3fef99a5b431ef0a0c23e0e59bbb5a582a7", + "sha256": "5be2751c9ebf3729bf6a59e19c9b91ad52a6b3a7d86720de4a23880f76deb3c4", + "gpu_tested_sha256": "1ff2c70cc8849020bd413a1d174718683bd89d5b6d8f25ee0a01d1c9b8876888", + "difference": "实测副本比仓库文件多一个末尾空行；代码内容一致，仓库参考文件未修改。" + }, + "publication_record": "实际提交及分支状态以 GitHub 历史为准；此文件记录实机验证证据。", + "public_reproduction": { + "directory": "repro", + "manifest": "repro/MANIFEST.json", + "recorded_text_files": 14, + "gpu_binaries_included": false, + "fixture_regenerable": true, + "new_gpu_run_for_publication": false + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/analyze_results.py b/03_hadamard_tc/a962695448-rgb/platforms/moore/analyze_results.py new file mode 100644 index 00000000..c9be695d --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/analyze_results.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Independently verify the three original MUSA timing CSVs and summarize them.""" + +import argparse +import collections +import csv +import hashlib +import json +import math +from pathlib import Path +import statistics + + +def analyze(source): + report = json.loads((source / "run_summary.json").read_text()) + if report["status"] != "PASS" or not report["validation"]["full_matrix"]: + raise ValueError("A successful full validation report is required") + medians = {} + expected_keys = None + total = 0 + for round_index in (1, 2, 3): + path = source / f"benchmark_round{round_index}.csv" + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != report["artifacts"][path.name]["sha256"]: + raise ValueError("CSV checksum mismatch: " + path.name) + with path.open(newline="") as stream: + records = list(csv.DictReader(stream)) + if len(records) != 1350: + raise ValueError("Expected 1,350 observations in each round") + groups = collections.defaultdict(list) + seen = set() + for record in records: + rows, dim = int(record["rows"]), int(record["dim"]) + key = (record["dtype"], rows, dim, record["method"]) + group = int(record["group"]) + if (*key, group) in seen or group not in range(5): + raise ValueError("Duplicate or invalid timing group") + seen.add((*key, group)) + if int(record["batch"]) * int(record["seq"]) * int(record["heads"]) != rows: + raise ValueError("Invalid four-dimensional shape") + if int(record["repeats"]) != 100 or int(record["seed"]) != 2909 or float(record["scale"]) != 1: + raise ValueError("Unexpected measurement protocol") + us, ms = float(record["kernel_us"]), float(record["kernel_ms"]) + if not math.isfinite(us) or us <= 0 or not math.isclose(ms, us / 1000, rel_tol=1e-9): + raise ValueError("Invalid event time or unit conversion") + groups[key].append(us) + if len(groups) != 270 or any(len(values) != 5 for values in groups.values()): + raise ValueError("Incomplete configuration matrix") + if expected_keys is None: + expected_keys = set(groups) + if set(groups) != expected_keys: + raise ValueError("Configuration sets differ across rounds") + medians[round_index] = {key: statistics.median(values) for key, values in groups.items()} + total += len(records) + shapes = sorted({key[:3] for key in expected_keys}) + comparisons, summary = [], [] + for candidate in ("optimized", "shuffle32"): + for operation in ("transform", "split", "fused"): + ratios, stable_faster, stable_five_percent = [], 0, 0 + for shape in shapes: + reductions = [] + for round_index in (1, 2, 3): + baseline = medians[round_index][(*shape, "baseline_" + operation)] + value = medians[round_index][(*shape, candidate + "_" + operation)] + ratio, reduction = baseline / value, 100 * (1 - value / baseline) + ratios.append(ratio) + reductions.append(reduction) + comparisons.append(dict(round=round_index, dtype=shape[0], rows=shape[1], + dim=shape[2], operation=operation, candidate=candidate, + baseline_us=baseline, candidate_us=value, + baseline_over_candidate=ratio, time_reduction_percent=reduction)) + stable_faster += all(value > 0 for value in reductions) + stable_five_percent += all(value >= 5 for value in reductions) + summary.append(dict(candidate=candidate, operation=operation, configurations=len(shapes), + minimum_speedup=min(ratios), maximum_speedup=max(ratios), + faster_in_every_round=stable_faster, + at_least_five_percent_in_every_round=stable_five_percent)) + representatives = [] + for shape in (("fp16", 16384, 256), ("bf16", 16384, 256), ("fp16", 1, 64), ("bf16", 17, 256)): + times = {} + for method in ("baseline_transform", "optimized_transform", "shuffle32_transform", + "baseline_fused", "optimized_fused", "shuffle32_fused"): + values = [medians[round_index][(*shape, method)] for round_index in (1, 2, 3)] + times[method] = {"round_medians_us": values, "median_of_round_medians_us": statistics.median(values)} + representatives.append({"dtype": shape[0], "rows": shape[1], "dim": shape[2], "times": times}) + return {"observations": total, "rounds": 3, "configurations_per_round": 270, + "method": "Median of five group-average event timings per configuration, in each of three independent benchmark processes", + "limits": "Warm-input MUSA event intervals; no CPU, cross-device, application-latency or statistical-significance claim", + "summary": summary, "representatives": representatives, "comparisons": comparisons} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + if args.output.exists(): + parser.error("Use a new output directory") + result = analyze(args.source) + args.output.mkdir(parents=True) + (args.output / "analysis.json").write_text(json.dumps(result, indent=2) + "\n") + with (args.output / "paired_comparisons.csv").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(result["comparisons"][0])) + writer.writeheader() + writer.writerows(result["comparisons"]) + print(json.dumps({"verified_observations": result["observations"], "output": str(args.output)})) + + +if __name__ == "__main__": + main() diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/exact_int4.hpp b/03_hadamard_tc/a962695448-rgb/platforms/moore/exact_int4.hpp new file mode 100644 index 00000000..2a1cd404 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/exact_int4.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include + +#ifndef HADAMARD_QUANT_HD +#define HADAMARD_QUANT_HD +#define HADAMARD_QUANT_HD_LOCAL +#endif + +namespace hadamard::moore::detail { + +// Compute clamp(RNE_integer(RNE_float32(value / scale)), -7, 7). +// The inputs are IEEE-754 binary32 bit patterns, finite value and finite scale>0. +// No floating arithmetic or division is used in this function. +HADAMARD_QUANT_HD inline int exact_int4_from_bits(std::uint32_t value_bits, + std::uint32_t scale_bits) { + const std::uint32_t magnitude_bits = value_bits & 0x7fffffffu; + if (magnitude_bits == 0) return 0; + const int value_exponent = static_cast((magnitude_bits >> 23) & 255u); + const int scale_exponent = static_cast((scale_bits >> 23) & 255u); + const std::uint32_t value_significand = (magnitude_bits & 0x7fffffu) + | (value_exponent ? 0x800000u : 0u); + const std::uint32_t scale_significand = (scale_bits & 0x7fffffu) + | (scale_exponent ? 0x800000u : 0u); + const int shift = (value_exponent ? value_exponent - 150 : -149) + - (scale_exponent ? scale_exponent - 150 : -149) + 25; + int quantized = 0; + if (shift >= 30) { + quantized = 7; + } else if (shift >= 0) { + // At most 24+29 bits, so this shift cannot overflow uint64_t. + const std::uint64_t numerator = static_cast(value_significand) << shift; + int low = 0, high = 7; + // Seven increasing transition boundaries require exactly three decisions. + for (int iteration = 0; iteration < 3; ++iteration) { + const int middle = (low + high) / 2; + const int half_ulp_units = middle == 0 ? 1 : (middle == 1 ? 2 : (middle <= 3 ? 4 : 8)); + // A half-integer has an even binary32 significand. Include its rounding + // interval on the side selected by the subsequent ties-to-even integer step. + const std::uint32_t boundary = static_cast( + ((2 * middle + 1) << 24) + ((middle & 1) ? -half_ulp_units : half_ulp_units)); + const std::uint64_t denominator = static_cast(scale_significand) * boundary; + const bool above = (middle & 1) ? numerator >= denominator : numerator > denominator; + low = above ? middle + 1 : low; + high = above ? high : middle; + } + quantized = low; + } + return (value_bits >> 31) ? -quantized : quantized; +} + +} // namespace hadamard::moore::detail + +#ifdef HADAMARD_QUANT_HD_LOCAL +#undef HADAMARD_QUANT_HD +#undef HADAMARD_QUANT_HD_LOCAL +#endif diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/hadamard_api.h b/03_hadamard_tc/a962695448-rgb/platforms/moore/hadamard_api.h new file mode 100644 index 00000000..1c437ef4 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/hadamard_api.h @@ -0,0 +1,56 @@ +// Ported from the verified MetaX implementation at cb82ea6. +// MUSA results require independent validation on Moore Threads hardware. +#pragma once + +#include +#include +#include +#include +#include + +namespace hadamard::moore { + +enum class Method { Baseline, Optimized, Shuffle32 }; + +// Shuffle32 is an explicit MUSA candidate enabled by HADAMARD_MOORE_SHUFFLE32. +// Select only after the native 32-lane shuffle probe passes. On the tested S4000, +// host device properties report 128, but device-code warpSize is 32. +// The asynchronous API does not query device properties on each launch. +// Nonempty Shuffle32 calls return musaErrorNotSupported when it is not compiled. + +// 连续设备内存：[rows, n]，n 为 1..256 的二次幂。scale 必须有限且为正数。 +// 支持 FP16/BF16 存储，内部 FP32；输入应有限，变换后的值应在输出类型范围内。 +// 所有操作仅在调用方的 stream 上发射，不分配内存、不复制、不等待。 +// rows=0 时只检查 n、scale、method，允许空指针，不发射内核。 +// 输入和变换输出只要求 2 字节对齐；scales 要求 float 对齐。 +// transform 允许 input==output 原位变换，其余重叠均返回 musaErrorInvalidValue。 +// 不检查分配容量/设备归属；调用方须提供当前设备有效且足够大的设备缓冲区。 +// 返回参数/发射错误；异步执行错误由调用方同步 stream 时检查。 +musaError_t transform(const __half* input, __half* output, std::size_t rows, + int n, float scale, musaStream_t stream, + Method method = Method::Optimized); +musaError_t transform(const __mt_bfloat16* input, __mt_bfloat16* output, + std::size_t rows, int n, float scale, musaStream_t stream, + Method method = Method::Optimized); + +// 每行 ceil(n/2) 字节；偶数元素在低 4 位，奇数在高 4 位，n=1 的高位为 0。 +// q=clamp(round_to_nearest_even(x/s), -7, 7)，s=max(abs(x))/7；全零行 s=1。 +// scales 为 rows 个 float。所有输入/输出缓冲区必须互不重叠。 +musaError_t quantize_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, musaStream_t stream, + Method method = Method::Optimized); +musaError_t quantize_int4(const __mt_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, + musaStream_t stream, Method method = Method::Optimized); + +// 融合路径先以最近偶数规则舍入到公开的 FP16/BF16 输出类型，再计算 INT4， +// 因而与 transform 后调用 quantize_int4 的字节和 scales 语义一致。 +musaError_t transform_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, + musaStream_t stream, Method method = Method::Optimized); +musaError_t transform_int4(const __mt_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale, + musaStream_t stream, Method method = Method::Optimized); + +} // namespace hadamard::moore + diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/hadamard_api.mu b/03_hadamard_tc/a962695448-rgb/platforms/moore/hadamard_api.mu new file mode 100644 index 00000000..28a29c63 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/hadamard_api.mu @@ -0,0 +1,353 @@ +// Ported from the verified MetaX implementation at cb82ea6. +// MUSA results require independent validation on Moore Threads hardware. +#include "hadamard_api.h" +#define HADAMARD_QUANT_HD __host__ __device__ +#include "exact_int4.hpp" +#undef HADAMARD_QUANT_HD + +#include +#include + +namespace hadamard::moore { +namespace { + +template __device__ float read_value(T value); +template<> __device__ float read_value(__half value) { return __half2float(value); } +template<> __device__ float read_value(__mt_bfloat16 value) { + return __bfloat162float(value); +} +template __device__ T store_value(float value); +template<> __device__ __half store_value(float value) { return __float2half_rn(value); } +template<> __device__ __mt_bfloat16 store_value(float value) { + return __float2bfloat16_rn(value); +} + +// The baseline retains native RNE division. The optimized paths compare exact +// integer significands at the composite float32/integer rounding boundaries. +template +__device__ std::uint8_t quantize_pair(const float* values, int pair, int n, + float row_scale) { + std::uint8_t byte = 0; + for (int k = 0; k < 2 && pair * 2 + k < n; ++k) { + int q; + if constexpr (IntegerCompare) { + q = detail::exact_int4_from_bits(__float_as_uint(values[pair * 2 + k]), + __float_as_uint(row_scale)); + } else { + const float x = __fdiv_rn(values[pair * 2 + k], row_scale); + const float lower = floorf(x); + const float fraction = x - lower; + q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + } + byte |= static_cast((q & 15) << (4 * k)); + } + return byte; +} + +// 沿用固定来源版本的基线算法：一元素一线程、每层两次屏障、线程 0 串行归约。 +// 与 optimized 独立保留蝶形计算，供同一运行环境下公平比较。 +template +__global__ void baseline_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float row_scale; + const int i = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + if (i < n) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + float next = 0; + if (i < n) { + const float a = values[i], b = values[i ^ stride]; + next = (i & stride) ? b - a : a + b; + } + __syncthreads(); + if (i < n) values[i] = next; + __syncthreads(); + } + if (i < n) values[i] = read_value(store_value(values[i] * scale)); + __syncthreads(); + } + if constexpr (!Quantize) { + if (i < n) output[offset + i] = store_value(values[i]); + } else { + if (i == 0) { + float magnitude = 0; + for (int j = 0; j < n; ++j) + magnitude = fmaxf(magnitude, fabsf(values[j])); + row_scale = magnitude == 0 ? 1.0f : __fdiv_rn(magnitude, 7.0f); + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + if (i < bytes) packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + // 多行复用同一 block 时，防止下行加载覆盖仍在打包/写出的上一行。 + if (rows - row > gridDim.x) __syncthreads(); + } +} + +// 每个线程独占一个蝶形的两个输入/输出，因此单层没有线程间读写冲突， +// 只需在本层结束后同步。此算法不假定 warp 宽度，不使用 warp 级隐式同步。 +template +__global__ void optimized_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float maxima[128]; + __shared__ float row_scale; + const int tid = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + for (int i = tid; i < n; i += blockDim.x) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + if (tid < n / 2) { + const int low = ((tid & ~(stride - 1)) << 1) | (tid & (stride - 1)); + const float a = values[low], b = values[low + stride]; + values[low] = a + b; + values[low + stride] = a - b; + } + __syncthreads(); + } + } + if constexpr (!Quantize) { + for (int i = tid; i < n; i += blockDim.x) + output[offset + i] = store_value(values[i] * scale); + } else { + float magnitude = 0; + for (int i = tid; i < n; i += blockDim.x) { + float value = values[i]; + if constexpr (Transform) { + value = read_value(store_value(value * scale)); + values[i] = value; + } + magnitude = fmaxf(magnitude, fabsf(value)); + } + maxima[tid] = magnitude; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 1; stride /= 2) { + if (tid < stride) maxima[tid] = fmaxf(maxima[tid], maxima[tid + stride]); + __syncthreads(); + } + if (tid == 0) { + // 最后两个值只由线程 0 消费，与 scale 写入合并，少一次屏障。 + const float maximum = fmaxf(maxima[0], maxima[1]); + row_scale = maximum == 0 ? 1.0f : __fdiv_rn(maximum, 7.0f); + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + for (int i = tid; i < bytes; i += blockDim.x) + packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + if (rows - row > gridDim.x) __syncthreads(); + } +} + +#if defined(HADAMARD_MOORE_SHUFFLE32) + +// Only select this candidate after the native MUSA width=32 shuffle probe passes. +// A CTA has eight logical 32-lane row groups. Tail groups execute with zero values; +// all 256 threads participate in each exchange, including the two reported 128-lane units. +__device__ int shuffle32_quantized_nibble(float value, float scale) { + return detail::exact_int4_from_bits(__float_as_uint(value), __float_as_uint(scale)) & 15; +} + +template +__global__ void shuffle32_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float scale) { + constexpr int width = 32; + constexpr int rows_per_block = 8; + constexpr int registers = N > width ? N / width : 1; + const int lane = threadIdx.x % width; + const int warp = threadIdx.x / width; + const std::size_t first_row = static_cast(blockIdx.x) * rows_per_block; + const std::size_t row_stride = static_cast(gridDim.x) * rows_per_block; + // Every thread in a CTA executes the same number of row-batch iterations. + for (std::size_t row_base = first_row; row_base < rows; row_base += row_stride) { + const std::size_t row = row_base + warp; + const bool active_row = row < rows; + const std::size_t offset = row * N; + float values[registers]; + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + values[r] = active_row && i < N ? read_value(input[offset + i]) : 0.0f; + } + if constexpr (Transform) { + #pragma unroll + for (int stride = 1; stride < (N < width ? N : width); stride *= 2) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const float current = values[r]; + const float peer = __shfl_xor_sync(0xffffffffu, current, stride, width); + values[r] = (lane & stride) ? peer - current : current + peer; + } + } + // Higher butterfly stages exchange registers owned by the same lane. + #pragma unroll + for (int stride = 1; stride < registers; stride *= 2) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + if (!(r & stride)) { + const float x = values[r], y = values[r + stride]; + values[r] = x + y; + values[r + stride] = x - y; + } + } + } + } + if constexpr (!Quantize) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + if (active_row && i < N) output[offset + i] = store_value(values[r] * scale); + } + } else { + float magnitude = 0.0f; + #pragma unroll + for (int r = 0; r < registers; ++r) { + if constexpr (Transform) values[r] = read_value(store_value(values[r] * scale)); + magnitude = fmaxf(magnitude, fabsf(values[r])); + } + #pragma unroll + for (int stride = width / 2; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor_sync(0xffffffffu, magnitude, stride, width)); + const float row_scale = magnitude == 0.0f ? 1.0f : __fdiv_rn(magnitude, 7.0f); + if (active_row && lane == 0) scales[row] = row_scale; + #pragma unroll + for (int r = 0; r < registers; ++r) { + // shuffle 在分支之前：奇数 lane 同样必须提供其相邻元素。 + const float peer = __shfl_xor_sync(0xffffffffu, values[r], 1, width); + const int i = lane + r * width; + if (active_row && (lane & 1) == 0 && i < N) { + const int low = shuffle32_quantized_nibble(values[r], row_scale); + const int high = i + 1 < N ? shuffle32_quantized_nibble(peer, row_scale) : 0; + packed[row * ((N + 1) / 2) + i / 2] + = static_cast(low | (high << 4)); + } + } + } + } +} + +template +musaError_t launch_shuffle32(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, musaStream_t stream) { + // launch() checks rows <= SIZE_MAX / sizeof(float), so rows+7 cannot overflow. + const std::size_t requested = (rows + 7) / 8; + const unsigned int blocks = static_cast(requested < 65535 ? requested : 65535); + #define LAUNCH_SHUFFLE32(N) case N: \ + shuffle32_kernel<<>>( \ + input, output, packed, scales, rows, scale); \ + break + switch (n) { + LAUNCH_SHUFFLE32(1); + LAUNCH_SHUFFLE32(2); + LAUNCH_SHUFFLE32(4); + LAUNCH_SHUFFLE32(8); + LAUNCH_SHUFFLE32(16); + LAUNCH_SHUFFLE32(32); + LAUNCH_SHUFFLE32(64); + LAUNCH_SHUFFLE32(128); + LAUNCH_SHUFFLE32(256); + default: return musaErrorInvalidValue; + } + #undef LAUNCH_SHUFFLE32 + return musaGetLastError(); +} + +#endif // HADAMARD_MOORE_SHUFFLE32 + +bool valid_range(const void* pointer, std::size_t bytes, std::size_t alignment) { + const auto address = reinterpret_cast(pointer); + return pointer != nullptr && address % alignment == 0 + && bytes <= std::numeric_limits::max() - address; +} + +bool overlaps(const void* left, std::size_t left_size, + const void* right, std::size_t right_size) { + // valid_range 已确保两个区间端点的加法不会溢出。 + const auto a = reinterpret_cast(left); + const auto b = reinterpret_cast(right); + return a < b + right_size && b < a + left_size; +} + +template +musaError_t launch(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, musaStream_t stream, Method method) { + if (n < 1 || n > 256 || (n & (n - 1)) != 0 + || !std::isfinite(scale) || scale <= 0 + || (method != Method::Baseline && method != Method::Optimized && method != Method::Shuffle32)) + return musaErrorInvalidValue; + if (rows == 0) return musaSuccess; + + constexpr auto maximum = std::numeric_limits::max(); + const std::size_t row_bytes = static_cast(n) * sizeof(T); + if (rows > maximum / row_bytes || rows > maximum / sizeof(float)) + return musaErrorInvalidValue; + const std::size_t input_bytes = rows * row_bytes; + if (!valid_range(input, input_bytes, alignof(T))) return musaErrorInvalidValue; + if constexpr (Quantize) { + const std::size_t packed_bytes = rows * static_cast((n + 1) / 2); + const std::size_t scale_bytes = rows * sizeof(float); + if (!valid_range(packed, packed_bytes, alignof(std::uint8_t)) + || !valid_range(scales, scale_bytes, alignof(float)) + || overlaps(input, input_bytes, packed, packed_bytes) + || overlaps(input, input_bytes, scales, scale_bytes) + || overlaps(packed, packed_bytes, scales, scale_bytes)) return musaErrorInvalidValue; + } else { + if (!valid_range(output, input_bytes, alignof(T)) + || (input != output && overlaps(input, input_bytes, output, input_bytes))) + return musaErrorInvalidValue; + } + + if (method == Method::Shuffle32) { + #if defined(HADAMARD_MOORE_SHUFFLE32) + return launch_shuffle32(input, output, packed, scales, rows, n, scale, stream); + #else + return musaErrorNotSupported; + #endif + } + + // 保守上限兼容不同设备的 grid.x 限制，超过上限由 block 顺序处理多行。 + const unsigned int blocks = static_cast(rows < 65535 ? rows : 65535); + if (method == Method::Baseline) { + const int threads = n < 128 ? 128 : n; + baseline_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } else { + const int threads = 128; + optimized_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } + return musaGetLastError(); +} + +} // namespace + +#define DEFINE_TYPED_API(T) \ +musaError_t transform(const T* input, T* output, std::size_t rows, int n, float scale, \ + musaStream_t stream, Method method) { \ + return launch(input, output, nullptr, nullptr, rows, n, scale, stream, method); \ +} \ +musaError_t quantize_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, musaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, 1.0f, stream, method); \ +} \ +musaError_t transform_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, float scale, musaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, scale, stream, method); \ +} + +DEFINE_TYPED_API(__half) +DEFINE_TYPED_API(__mt_bfloat16) +#undef DEFINE_TYPED_API + +} // namespace hadamard::moore + diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/probe.mu b/03_hadamard_tc/a962695448-rgb/platforms/moore/probe.mu new file mode 100644 index 00000000..da10eb74 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/probe.mu @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include +#include + +#define CHECK(call) do { musaError_t e = (call); if (e != musaSuccess) { \ + std::fprintf(stderr, "%s: %s\n", #call, musaGetErrorString(e)); return 2; } } while (0) + +__global__ void probe(int* out, float* numeric) { + const int tid = threadIdx.x; + out[tid * 22] = warpSize; + int slot = 1; + for (int width = 32; width <= 32; width *= 2) { + for (int step = 1; step < width; step *= 2) { + out[tid * 22 + slot++] = __shfl_xor_sync(0xffffffffu, tid, step, width); + } + } + float x = (tid - 128) * 0.125f; + numeric[tid * 3] = __half2float(__float2half_rn(x)); + numeric[tid * 3 + 1] = __bfloat162float(__float2bfloat16_rn(x)); + const float scale = 1.0f / 7.0f; + numeric[tid * 3 + 2] = 0.5f / scale; +} + +int main() { + musaDeviceProp p{}; + CHECK(musaSetDevice(0)); + CHECK(musaGetDeviceProperties(&p, 0)); + std::printf("DEVICE name=%s arch=%d.%d warp=%d memory=%zu max_threads=%d sm=%d\n", + p.name, p.major, p.minor, p.warpSize, p.totalGlobalMem, + p.maxThreadsPerBlock, p.multiProcessorCount); + int runtime = 0, driver = 0; + CHECK(musaRuntimeGetVersion(&runtime)); CHECK(musaDriverGetVersion(&driver)); + std::printf("VERSIONS runtime=%d driver=%d half_bytes=%zu bf16_bytes=%zu\n", + runtime, driver, sizeof(__half), sizeof(__mt_bfloat16)); + if (p.warpSize != 128) { std::puts("UNEXPECTED_WARP_SIZE"); return 3; } + int* d = nullptr; float* f = nullptr; + CHECK(musaMalloc(reinterpret_cast(&d), 256 * 22 * sizeof(int))); + CHECK(musaMalloc(reinterpret_cast(&f), 256 * 3 * sizeof(float))); + probe<<<1, 256>>>(d, f); + CHECK(musaGetLastError()); CHECK(musaDeviceSynchronize()); + std::vector out(256 * 22); + std::vector numeric(256 * 3); + CHECK(musaMemcpy(out.data(), d, out.size()*sizeof(int), musaMemcpyDeviceToHost)); + CHECK(musaMemcpy(numeric.data(), f, numeric.size()*sizeof(float), musaMemcpyDeviceToHost)); + int bad = 0, checks = 0, numeric_bad = 0; + int width_bad[3]{}; + std::printf("BUILTIN_WARP first=%d last=%d\n",out[0],out[255*22]); + for (int tid = 0; tid < 256; ++tid) { + if (out[tid * 22] != 32) ++bad; + int slot = 1; + for (int width = 32; width <= 32; width *= 2) { + for (int step = 1; step < width; step *= 2) { + const int expected = (tid / width) * width + ((tid % width) ^ step); + const int actual = out[tid * 22 + slot++]; + if (actual != expected) { + if (bad < 8) std::printf("SHUFFLE_FAIL tid=%d width=%d step=%d got=%d expected=%d\n", tid,width,step,actual,expected); + ++bad; + ++width_bad[width == 32 ? 0 : (width == 64 ? 1 : 2)]; + } + ++checks; + } + } + const float expected = (tid - 128) * 0.125f; + if (numeric[tid*3] != expected || numeric[tid*3+1] != expected) { ++bad; ++numeric_bad; } + } + std::printf("RESULT shuffle_checks=%d failures=%d division_probe=%.9g\n",checks,bad,numeric[2]); + std::printf("SUPPORTED_COUNTS width32_failures=%d numeric_failures=%d\n",width_bad[0],numeric_bad); + CHECK(musaFree(d)); CHECK(musaFree(f)); + return bad ? 1 : 0; +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/MANIFEST.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/MANIFEST.json new file mode 100644 index 00000000..576d8682 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/MANIFEST.json @@ -0,0 +1,111 @@ +{ + "scope": "Public reproduction package for the recorded 2026-09-16 S4000 experiment", + "implementation_commit": "9f4515491a1d8302693e311d4746cf72f2361f98", + "recorded_files": { + "optimization.json": { + "server_relative_path": "runs/optimization.json", + "bytes": 15655, + "sha256": "e1056f72c16310c39837f5ab330c911bbf92ee3ba96735e68dd4f4e289225f27" + }, + "quant-gpu.log": { + "server_relative_path": "runs/quant-gpu.log", + "bytes": 112, + "sha256": "10f4b76d5a433cce1626a0d3146b354a47550363ec36688d7e990e61e61531f4" + }, + "full-run-summary.json": { + "server_relative_path": "runs/full/run_summary.json", + "bytes": 18467, + "sha256": "f6a50f6e1defee88fa752b69bf63544850c20d53e5bacb9e42152a94501608e5" + }, + "full-validation.json": { + "server_relative_path": "runs/full/validation.json", + "bytes": 1046, + "sha256": "f7c7af273bc74c40d602227ef0014eb23ce07b07380bbeb893622f294cbee19e" + }, + "disabled-run-summary.json": { + "server_relative_path": "runs/disabled/run_summary.json", + "bytes": 18442, + "sha256": "a505ad08186225ad344b0ce825384436daf1a0e71a629aee111f2d8e25ac6cc5" + }, + "disabled-validation.json": { + "server_relative_path": "runs/disabled/validation.json", + "bytes": 970, + "sha256": "9e3cf608dc9880c4922d9ef8978b7c16f9e0f75b3051c3d2904c73e19179e368" + }, + "cpu-equivalence-linux.json": { + "server_relative_path": "cpu-equivalence.json", + "bytes": 530, + "sha256": "1b2d4104c5d4bc3da97a6c25f5374474c5d849945d0695091c25448cb3d8a61a" + }, + "paired-round1.csv": { + "server_relative_path": "runs/paired-round1.csv", + "bytes": 265111, + "sha256": "90a10dcfe163bb44425ada09f28e84c6a5cf0159872728b2b0a2975ceadd0cdb" + }, + "paired-round1.log": { + "server_relative_path": "runs/paired-round1.log", + "bytes": 1455, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + "paired-round2.csv": { + "server_relative_path": "runs/paired-round2.csv", + "bytes": 265090, + "sha256": "a90fc97aee1ede505c0c83b86d5369bba8931ead53149a3a4e5c36a506dbb363" + }, + "paired-round2.log": { + "server_relative_path": "runs/paired-round2.log", + "bytes": 1455, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + "paired-round3.csv": { + "server_relative_path": "runs/paired-round3.csv", + "bytes": 265132, + "sha256": "a9846fc8bb1d8efba317b3aba1929a5cf5102335833c2e2799475653a6eee033" + }, + "paired-round3.log": { + "server_relative_path": "runs/paired-round3.log", + "bytes": 1455, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + "cpu-equivalence-macos.json": { + "local_origin": "macOS CPU validation before S4000 run", + "bytes": 528, + "sha256": "0cab7e46a5057ff23ba7b1742539be76f0275950eee42e60ca8d67b8992a3132" + } + }, + "frozen_source_sha256": { + "check_quant_cpu.cpp": "8d4c6702c36a81f1cdef27df8e4b6bb413b92585d5169f21053d97ac1a450d1a", + "check_quant_gpu.mu": "e51771809fb54db5cdb9e63c142f5266f2105ce0777d24577e428941f89f3b5d", + "paired_benchmark.mu": "138e6dd6ec9f9fd959162ae5edc0d27727e355ba5354cd1bd70bf702cfc69043", + "control/control_api.h": "b40c05f707e806381b3b7440537a435fdb4137cf27b78d403b336b32bf722014", + "control/control_api.mu": "d3ebe8f53f076a66db0cdc5d8cca5092fa821c729336949c3e2913d45e5ad56c" + }, + "adapted_entry_points": { + "check_quant_cpu.py": { + "sha256": "50d2a8115079c039541818765317bb6efb0efac24ed85b4586bef79e14c49227", + "change": "Portable paths and explicit output directory; offline input validation for the analyzer." + }, + "analyze_optimization.py": { + "sha256": "8d6f11ca3fd7b8c709a0885502272d472d8169027fe29b652afd2d8c70f758d0", + "change": "Portable paths and explicit output directory; offline input validation for the analyzer." + } + }, + "test_script_sha256": "d05f1426a406c21aa37efccd5f0d09e3856f281ad84aa6706c13a6806216156e", + "fixture": { + "records": 2074568, + "bytes": 24894816, + "sha256": "b404de04128361ed75347666113cfe2545f7c6e52aa828b070c1812c967ce230", + "binary_committed": false + }, + "publication_validation": { + "cpu_cases": 17527208, + "fixture_matches_original_bytes": true, + "analysis_matches_original_values": true, + "analysis_csv_matches_original_bytes": true, + "negative_and_positive_analysis_tests": 6, + "python": "3.12.14", + "numpy": "2.3.5", + "platform": "Darwin", + "gpu_rerun": false + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/README.zh-CN.md b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/README.zh-CN.md new file mode 100644 index 00000000..b0799e89 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/README.zh-CN.md @@ -0,0 +1,113 @@ +# 摩尔 S4000：公开复现材料 + +本目录公开 2026-09-16 已完成实验的原始计时、验证清单、冻结对照源码和复算工具。 +整理阶段只执行本地 CPU 检查，没有重新开启 S4000 或生成新的 GPU 性能结果。 + +## 文件与口径 + +| 文件 | 用途 | +|---|---| +| recorded/paired-round1.csv 至 paired-round3.csv | 三个独立进程的 6,750 条原始计时 | +| recorded/optimization.json | 原始实机阶段、返回码、命令、输入和输出哈希 | +| recorded/full-run-summary.json、full-validation.json | 1,504 组完整输入、192 项 API、14 项 CLI 检查 | +| recorded/disabled-run-summary.json、disabled-validation.json | 关闭 Shuffle32 的兼容性结果 | +| recorded/quant-gpu.log | 2,074,568 项 GPU 量化对照，三方零差异 | +| recorded/cpu-equivalence-*.json | 原始 macOS/Linux CPU 检查，包含夹具 SHA-256 | +| control/control_api.h、control_api.mu | 冻结旧版，仅调整命名空间与头文件名以同程序链接 | +| paired_benchmark.mu、check_quant_gpu.mu、check_quant_cpu.cpp | 与实测版本字节一致的计时及对照程序 | +| check_quant_cpu.py | 可指定库文件、输出目录的 CPU 夹具生成器 | +| analyze_optimization.py | 离线核验 CSV 完整性、哈希、单位及全部配对结果 | +| MANIFEST.json | 公布文件的来源、哈希及整理阶段的检查范围 | + +原始记录中的 /data 路径属于当时服务器的实验现场，用于追溯。下面的复现命令使用仓库相对路径。 +账户连接信息、关机截图、构建二进制和 25 MB 夹具不在此目录中；夹具可从源码精确再生成。 + +## 1. 离线重算原始结果 + +从 03_hadamard_tc/a962695448-rgb 项目目录运行，只需要 Python 标准库： + +~~~bash +moore_work=$(mktemp -d) +python3 platforms/moore/repro/analyze_optimization.py --output "$moore_work/analysis" +python3 -m unittest discover -s platforms/moore/repro -p 'test_*.py' -v +~~~ + +输出目录必须尚不存在。analysis.json 保留全部正负例，paired-comparisons.csv 给出各轮、 +各配置旧版/新版的耗时比。脚本检查每轮 2,250 条、每个配置/方法组合的五个分组、 +固定的 30 个形状/精度配置、15 个方法/操作组合以及微秒和毫秒换算； +篡改、缺行、重复或未知配置都会失败。 + +默认 Optimized 分步与融合在 30 个配置的每轮都减少至少 5% 耗时，原始验收通过。 +融合旧版/新版比值为 2.012～2.898×；它是同设备、热输入、MUSA event 区间的比较， +不代表应用端到端收益。完整解释见 [实测报告](../REPORT.zh-CN.md)。 + +## 2. 在 CPU 上重新生成测试夹具 + +需要支持 C++17 的编译器及 NumPy。原始 macOS/Linux 分别使用 NumPy 2.3.5/1.23.5， +两者生成了相同字节。沿用上节新建的 moore_work： + +~~~bash +c++ -std=c++17 -O2 -fno-fast-math -ffp-contract=off -shared -fPIC \ + -Iplatforms/moore -Iinclude platforms/moore/repro/check_quant_cpu.cpp \ + -o "$moore_work/libcheck_quant.so" +python3 platforms/moore/repro/check_quant_cpu.py \ + --library "$moore_work/libcheck_quant.so" --output "$moore_work/cpu" +~~~ + +该动态库使用既有 CPU 除法/舍入参考检查实际整数实现，共 17,527,208 项； +随后生成 2,074,568 条小端记录，每条为 float32 value、float32 scale、int32 expected。 +GPU 夹具与 CPU 检查重叠，不将两类数量相加当作独立输入。 + +预期夹具大小为 24,894,816 字节，SHA-256： + +~~~text +b404de04128361ed75347666113cfe2545f7c6e52aa828b070c1812c967ce230 +~~~ + +原服务器参考头文件比 GitHub 中的共享参考多一个末尾空行。计算代码一致；公开版本已在 +CPU 上重建，并检查生成夹具的字节哈希与原实机输入完全相同。 + +## 3. 在另行准备的 MUSA 设备上重建 + +以下命令需要实际 MUSA SDK 和设备；不属于离线复算步骤。原实测环境为 S4000、 +MUSA 5.1.0、mp_22。CPU 夹具先按上一节生成。 + +~~~bash +musa_root=/usr/local/musa +"$musa_root/bin/mcc" -std=c++17 -O2 -fno-fast-math -ffp-contract=off \ + --offload-arch=mp_22 -Iplatforms/moore \ + platforms/moore/repro/check_quant_gpu.mu \ + -L"$musa_root/lib" -lmusart -o "$moore_work/quant_gpu" +LD_LIBRARY_PATH="$musa_root/lib:${LD_LIBRARY_PATH:-}" \ + "$moore_work/quant_gpu" "$moore_work/cpu/quant_fixture.bin" + +"$musa_root/bin/mcc" -std=c++17 -O2 -fno-fast-math -ffp-contract=off \ + --offload-arch=mp_22 -DHADAMARD_MOORE_SHUFFLE32 \ + -Iplatforms/moore -Iplatforms/moore/repro/control -Iinclude \ + platforms/moore/hadamard_api.mu platforms/moore/repro/control/control_api.mu \ + platforms/moore/repro/paired_benchmark.mu \ + -L"$musa_root/lib" -lmusart -o "$moore_work/paired_benchmark" + +for round in 1 2 3; do + LD_LIBRARY_PATH="$musa_root/lib:${LD_LIBRARY_PATH:-}" \ + "$moore_work/paired_benchmark" --benchmark --groups 5 --repeats 100 \ + --csv "$moore_work/paired-round${round}.csv" \ + > "$moore_work/paired-round${round}.log" 2>&1 +done + +python3 platforms/moore/run_platform.py --shuffle32 --no-benchmark \ + --output "$moore_work/full" +python3 platforms/moore/run_platform.py --quick --no-benchmark \ + --output "$moore_work/disabled" +~~~ + +新运行会产生自己的 CSV 和日志，不能覆盖 recorded 目录或套用原始哈希清单。 +analyze_optimization.py 默认针对已归档的实验；分析新数据时需提供相应的 +optimization.json，至少记录通过状态、三份 CSV 的实际 SHA-256，并另外保存 +环境、编译参数、测试与源码来源。原始清单提供完整格式示例。 + +## 整理阶段的验证范围 + +本次公开整理保留 GPU 计算源码和冻结对照的原字节；调整的是 CPU 生成器的路径参数、 +离线复算入口和文档。原始三轮数据可以重新得到与此前完全相同的分析结果。 +GPU 源码已有实机证据，本轮没有再编译或运行 MUSA 程序。 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/analyze_optimization.py b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/analyze_optimization.py new file mode 100644 index 00000000..0536e7d6 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/analyze_optimization.py @@ -0,0 +1,214 @@ +"""Verify the recorded paired experiment and recompute every old/new comparison.""" + +import argparse +import collections +import csv +import hashlib +import json +import math +import statistics +from pathlib import Path + +DTYPES = ("fp16", "bf16") +ROWS = (1, 17, 257, 4096, 16384) +DIMS = (64, 128, 256) +OPERATIONS = ("transform", "split", "fused") +METHODS = ( + "baseline", + "control_optimized", + "optimized", + "control_shuffle32", + "shuffle32", +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def read_medians(recorded): + state = json.loads((recorded / "optimization.json").read_text()) + require(state["status"] == "PASS", "The original experiment did not pass.") + expected = { + (dtype, rows, dim, method + "_" + operation) + for dtype in DTYPES + for rows in ROWS + for dim in DIMS + for method in METHODS + for operation in OPERATIONS + } + medians, count = {}, 0 + for round_index in (1, 2, 3): + path = recorded / f"paired-round{round_index}.csv" + require( + hashlib.sha256(path.read_bytes()).hexdigest() + == state["artifacts"][path.name]["sha256"], + f"Checksum mismatch: {path.name}", + ) + with path.open(newline="") as stream: + records = list(csv.DictReader(stream)) + require(len(records) == 2250, f"Expected 2250 observations: {path.name}") + groups, seen = collections.defaultdict(list), set() + for record in records: + key = ( + record["dtype"], + int(record["rows"]), + int(record["dim"]), + record["method"], + ) + group = int(record["group"]) + require( + (*key, group) not in seen and group in range(5), + f"Duplicate or invalid timing group: {path.name}", + ) + seen.add((*key, group)) + require( + int(record["batch"]) * int(record["seq"]) * int(record["heads"]) + == key[1], + "Inconsistent row count.", + ) + require( + int(record["repeats"]) == 100 + and int(record["seed"]) == 2909 + and float(record["scale"]) == 1, + "Unexpected experiment parameters.", + ) + value, ms = float(record["kernel_us"]), float(record["kernel_ms"]) + require( + value > 0 + and math.isfinite(value) + and math.isclose(ms, value / 1000, rel_tol=1e-9), + "Invalid time or inconsistent microsecond/millisecond units.", + ) + groups[key].append(value) + require( + set(groups) == expected, "Incomplete or unexpected configuration matrix." + ) + require(all(len(values) == 5 for values in groups.values()), "Missing groups.") + medians[round_index] = { + key: statistics.median(values) for key, values in groups.items() + } + count += len(records) + return medians, count + + +def analyze(recorded): + medians, count = read_medians(recorded) + shapes = sorted( + (dtype, rows, dim) for dtype in DTYPES for rows in ROWS for dim in DIMS + ) + comparisons, summary, representatives = [], [], [] + for method in ("optimized", "shuffle32"): + for operation in OPERATIONS: + ratios, stable, stable_five = [], 0, 0 + for shape in shapes: + reductions = [] + for round_index in (1, 2, 3): + old = medians[round_index][ + (*shape, "control_" + method + "_" + operation) + ] + new = medians[round_index][(*shape, method + "_" + operation)] + ratio, reduction = old / new, 100 * (1 - new / old) + ratios.append(ratio) + reductions.append(reduction) + comparisons.append( + { + "round": round_index, + "dtype": shape[0], + "rows": shape[1], + "dim": shape[2], + "method": method, + "operation": operation, + "old_us": old, + "new_us": new, + "old_over_new": ratio, + "time_reduction_percent": reduction, + } + ) + stable += all(value > 0 for value in reductions) + stable_five += all(value >= 5 for value in reductions) + summary.append( + { + "method": method, + "operation": operation, + "configurations": len(shapes), + "minimum_speedup": min(ratios), + "maximum_speedup": max(ratios), + "faster_every_round": stable, + "at_least_five_percent_every_round": stable_five, + } + ) + for shape in ( + ("fp16", 16384, 256), + ("bf16", 16384, 256), + ("fp16", 1, 64), + ("bf16", 17, 256), + ): + item = {"dtype": shape[0], "rows": shape[1], "dim": shape[2], "timings": {}} + for method in ("optimized", "shuffle32"): + for operation in OPERATIONS: + old = [ + medians[n][(*shape, "control_" + method + "_" + operation)] + for n in (1, 2, 3) + ] + new = [ + medians[n][(*shape, method + "_" + operation)] for n in (1, 2, 3) + ] + item["timings"][method + "_" + operation] = { + "old_round_medians_us": old, + "new_round_medians_us": new, + "old_us": statistics.median(old), + "new_us": statistics.median(new), + "old_over_new": statistics.median(old) / statistics.median(new), + } + representatives.append(item) + accepted = all( + item["at_least_five_percent_every_round"] == 30 + for item in summary + if item["method"] == "optimized" and item["operation"] in ("split", "fused") + ) + return { + "status": "VERIFIED", + "accepted": accepted, + "observations": count, + "rounds": 3, + "configurations_per_round": 450, + "summary": summary, + "representatives": representatives, + "measurement": "Median of 5 group-average event intervals; 100 calls per group; paired old/new methods", + "scope": "Same S4000, warm inputs, scalar scale=1; no application-level or cross-device inference", + "comparisons": comparisons, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--recorded", type=Path, default=Path(__file__).resolve().parent / "recorded" + ) + parser.add_argument( + "--output", type=Path, required=True, help="New output directory." + ) + args = parser.parse_args() + result = analyze(args.recorded) + args.output.mkdir(parents=True, exist_ok=False) + (args.output / "analysis.json").write_text(json.dumps(result, indent=2) + "\n") + with (args.output / "paired-comparisons.csv").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(result["comparisons"][0])) + writer.writeheader() + writer.writerows(result["comparisons"]) + print( + json.dumps( + { + key: result[key] + for key in ("status", "accepted", "observations", "summary") + }, + indent=2, + ) + ) + return 0 if result["accepted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/check_quant_cpu.cpp b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/check_quant_cpu.cpp new file mode 100644 index 00000000..77c32413 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/check_quant_cpu.cpp @@ -0,0 +1,29 @@ +#include "exact_int4.hpp" +#include "reference.hpp" +#include +#include +#include +#include + +extern "C" std::size_t scan_quant(const float* values, const float* scales, + std::size_t count, std::uint32_t* first) { + std::size_t errors = 0; + for (std::size_t i = 0; i < count; ++i) { + std::uint32_t vb = 0, sb = 0; + std::memcpy(&vb, values + i, sizeof(float)); + std::memcpy(&sb, scales + i, sizeof(float)); + const float quotient = values[i] / scales[i]; + const int expected = std::clamp(hadamard::nearest_even(quotient), -7, 7); + const int actual = hadamard::moore::detail::exact_int4_from_bits(vb, sb); + if (actual != expected) { + if (!errors) { + first[0] = static_cast(i); + first[1] = vb; first[2] = sb; + first[3] = static_cast(expected); + first[4] = static_cast(actual); + } + ++errors; + } + } + return errors; +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/check_quant_cpu.py b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/check_quant_cpu.py new file mode 100644 index 00000000..cb236b18 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/check_quant_cpu.py @@ -0,0 +1,128 @@ +"""Regenerate the exact INT4 fixture using the independently checked CPU reference.""" + +import argparse +import ctypes +import hashlib +import json +import time +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--library", + type=Path, + required=True, + help="Path to the compiled CPU comparison library.", +) +parser.add_argument( + "--output", + type=Path, + required=True, + help="New output directory for the fixture and report.", +) +args = parser.parse_args() +lib = ctypes.CDLL(str(args.library.resolve())) +OUTPUT = args.output.resolve() +OUTPUT.mkdir(parents=True, exist_ok=False) +scan = lib.scan_quant +scan.argtypes = [ + ctypes.POINTER(ctypes.c_float), + ctypes.POINTER(ctypes.c_float), + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint32), +] +scan.restype = ctypes.c_size_t +count = 0 +gpu_values, gpu_scales = [], [] + + +def check(values, scales, keep=False): + global count + values = np.ascontiguousarray(values, dtype=np.float32) + scales = np.ascontiguousarray( + np.broadcast_to(scales, values.shape), dtype=np.float32 + ) + first = np.zeros(5, dtype=np.uint32) + errors = scan( + values.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + scales.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + values.size, + first.ctypes.data_as(ctypes.POINTER(ctypes.c_uint32)), + ) + if errors: + raise AssertionError( + { + "errors": errors, + "first": first.tolist(), + "value": float(values[first[0]]), + "scale": float(scales[first[0]]), + } + ) + count += values.size + if keep: + gpu_values.append(values.copy()) + gpu_scales.append(scales.copy()) + + +started = time.monotonic() +per_dtype = {} +for name, maximum in [("fp16", 0x7BFF), ("bf16", 0x7F7F)]: + bits = np.arange(maximum + 1, dtype=np.uint16) + pool = ( + bits.view(np.float16).astype(np.float32) + if name == "fp16" + else (bits.astype(np.uint32) << 16).view(np.float32) + ) + indices = np.unique(np.linspace(1, maximum, 257).astype(int)) + gpu_indices = set(np.unique(np.linspace(1, maximum, 17).astype(int))) + before = count + for limit in sorted(set(indices) | gpu_indices): + scale = np.float32(pool[limit] / np.float32(7)) + values = pool[: limit + 1] + check(values, scale, limit in gpu_indices) + check(-values, scale, limit in gpu_indices) + per_dtype[name] = count - before + +# Probe the composite binary32 quotient/integer rounding boundary from both sides. +rng = np.random.default_rng(20260916) +scales = rng.integers(1, 0x7E000000, size=10000, dtype=np.uint32).view(np.float32) +for integer in range(7): + middle = np.float32(scales * np.float32(integer + 0.5)) + values = middle.copy() + for offset in range(4): + check(values, scales, True) + check(-values, scales, True) + values = np.nextafter(values, np.float32(np.inf)) + values = middle.copy() + for offset in range(3): + values = np.nextafter(values, np.float32(0)) + check(values, scales, True) + check(-values, scales, True) + +values = np.concatenate(gpu_values) +scales = np.concatenate(gpu_scales) +expected = np.clip(np.rint(np.float32(values / scales)), -7, 7).astype(np.int32) +dtype = np.dtype([("value", " +#define HADAMARD_QUANT_HD __host__ __device__ +#include "exact_int4.hpp" +#undef HADAMARD_QUANT_HD +#include +#include +#include +#include +#include + +struct Record { float value, scale; std::int32_t expected; }; +static_assert(sizeof(Record) == 12); +#define CHECK(call) do { auto status=(call); if(status!=musaSuccess) { \ + std::fprintf(stderr,"%s: %s\n",#call,musaGetErrorString(status)); return 2; } } while(0) + +__global__ void compare_quant(const Record* input, int* legacy, int* candidate, std::size_t count) { + const std::size_t index = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= count) return; + const auto item = input[index]; + const float quotient = __fdiv_rn(item.value, item.scale); + const float lower = floorf(quotient), fraction = quotient - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + legacy[index] = q < -7 ? -7 : (q > 7 ? 7 : q); + candidate[index] = hadamard::moore::detail::exact_int4_from_bits( + __float_as_uint(item.value), __float_as_uint(item.scale)); +} + +int main(int argc, char** argv) { + if (argc != 2) { std::fprintf(stderr,"Expected fixture path\n"); return 2; } + std::ifstream file(argv[1],std::ios::binary|std::ios::ate); + const auto bytes = file.tellg(); + if (!file || bytes <= 0 || bytes % sizeof(Record) || bytes > 120000000) return 2; + const std::size_t count = static_cast(bytes) / sizeof(Record); + std::vector records(count); + file.seekg(0); file.read(reinterpret_cast(records.data()),bytes); + if (!file) return 2; + for (const auto& r : records) + if (!std::isfinite(r.value) || !std::isfinite(r.scale) || r.scale <= 0 || r.expected < -7 || r.expected > 7) return 2; + CHECK(musaSetDevice(0)); + musaDeviceProp prop{}; CHECK(musaGetDeviceProperties(&prop,0)); + std::printf("DEVICE %s arch=%d.%d reported_warp=%d\n",prop.name,prop.major,prop.minor,prop.warpSize); + Record* input=nullptr; int *legacy=nullptr,*candidate=nullptr; + CHECK(musaMalloc(reinterpret_cast(&input),count*sizeof(Record))); + CHECK(musaMalloc(reinterpret_cast(&legacy),count*sizeof(int))); + CHECK(musaMalloc(reinterpret_cast(&candidate),count*sizeof(int))); + CHECK(musaMemcpy(input,records.data(),count*sizeof(Record),musaMemcpyHostToDevice)); + compare_quant<<((count+255)/256),256>>>(input,legacy,candidate,count); + CHECK(musaGetLastError()); CHECK(musaDeviceSynchronize()); + std::vector a(count),b(count); + CHECK(musaMemcpy(a.data(),legacy,count*sizeof(int),musaMemcpyDeviceToHost)); + CHECK(musaMemcpy(b.data(),candidate,count*sizeof(int),musaMemcpyDeviceToHost)); + std::size_t old_errors=0,new_errors=0,different=0; + for (std::size_t i=0;i +#include +#include +#include +#include + +namespace hadamard::moore_control { + +enum class Method { Baseline, Optimized, Shuffle32 }; + +// Shuffle32 is an explicit MUSA candidate enabled by HADAMARD_MOORE_SHUFFLE32. +// Select only after the native 32-lane shuffle probe passes. On the tested S4000, +// host device properties report 128, but device-code warpSize is 32. +// The asynchronous API does not query device properties on each launch. +// Nonempty Shuffle32 calls return musaErrorNotSupported when it is not compiled. + +// 连续设备内存：[rows, n]，n 为 1..256 的二次幂。scale 必须有限且为正数。 +// 支持 FP16/BF16 存储，内部 FP32；输入应有限，变换后的值应在输出类型范围内。 +// 所有操作仅在调用方的 stream 上发射，不分配内存、不复制、不等待。 +// rows=0 时只检查 n、scale、method，允许空指针，不发射内核。 +// 输入和变换输出只要求 2 字节对齐；scales 要求 float 对齐。 +// transform 允许 input==output 原位变换，其余重叠均返回 musaErrorInvalidValue。 +// 不检查分配容量/设备归属；调用方须提供当前设备有效且足够大的设备缓冲区。 +// 返回参数/发射错误；异步执行错误由调用方同步 stream 时检查。 +musaError_t transform(const __half* input, __half* output, std::size_t rows, + int n, float scale, musaStream_t stream, + Method method = Method::Optimized); +musaError_t transform(const __mt_bfloat16* input, __mt_bfloat16* output, + std::size_t rows, int n, float scale, musaStream_t stream, + Method method = Method::Optimized); + +// 每行 ceil(n/2) 字节；偶数元素在低 4 位，奇数在高 4 位，n=1 的高位为 0。 +// q=clamp(round_to_nearest_even(x/s), -7, 7)，s=max(abs(x))/7；全零行 s=1。 +// scales 为 rows 个 float。所有输入/输出缓冲区必须互不重叠。 +musaError_t quantize_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, musaStream_t stream, + Method method = Method::Optimized); +musaError_t quantize_int4(const __mt_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, + musaStream_t stream, Method method = Method::Optimized); + +// 融合路径先以最近偶数规则舍入到公开的 FP16/BF16 输出类型，再计算 INT4， +// 因而与 transform 后调用 quantize_int4 的字节和 scales 语义一致。 +musaError_t transform_int4(const __half* input, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, + musaStream_t stream, Method method = Method::Optimized); +musaError_t transform_int4(const __mt_bfloat16* input, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale, + musaStream_t stream, Method method = Method::Optimized); + +} // namespace hadamard::moore_control + diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/control/control_api.mu b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/control/control_api.mu new file mode 100644 index 00000000..bda8354d --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/control/control_api.mu @@ -0,0 +1,350 @@ +// Ported from the verified MetaX implementation at cb82ea6. +// MUSA results require independent validation on Moore Threads hardware. +#include "control_api.h" + +#include +#include + +namespace hadamard::moore_control { +namespace { + +template __device__ float read_value(T value); +template<> __device__ float read_value(__half value) { return __half2float(value); } +template<> __device__ float read_value(__mt_bfloat16 value) { + return __bfloat162float(value); +} +template __device__ T store_value(float value); +template<> __device__ __half store_value(float value) { return __float2half_rn(value); } +template<> __device__ __mt_bfloat16 store_value(float value) { + return __float2bfloat16_rn(value); +} + +// Explicit RNE division preserves CPU/fused scale equality on MUSA 5.1. +// Ordinary division differed by one ULP for FP16 value 0.5009765625 divided by 7. +__device__ std::uint8_t quantize_pair(const float* values, int pair, int n, + float row_scale) { + std::uint8_t byte = 0; + for (int k = 0; k < 2 && pair * 2 + k < n; ++k) { + const float x = __fdiv_rn(values[pair * 2 + k], row_scale); + // 明确实现最近偶数舍入，不依赖编译器默认取整模式。 + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + byte |= static_cast((q & 15) << (4 * k)); + } + return byte; +} + +// 沿用固定来源版本的基线算法：一元素一线程、每层两次屏障、线程 0 串行归约。 +// 与 optimized 独立保留蝶形计算，供同一运行环境下公平比较。 +template +__global__ void baseline_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float row_scale; + const int i = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + if (i < n) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + float next = 0; + if (i < n) { + const float a = values[i], b = values[i ^ stride]; + next = (i & stride) ? b - a : a + b; + } + __syncthreads(); + if (i < n) values[i] = next; + __syncthreads(); + } + if (i < n) values[i] = read_value(store_value(values[i] * scale)); + __syncthreads(); + } + if constexpr (!Quantize) { + if (i < n) output[offset + i] = store_value(values[i]); + } else { + if (i == 0) { + float magnitude = 0; + for (int j = 0; j < n; ++j) + magnitude = fmaxf(magnitude, fabsf(values[j])); + row_scale = magnitude == 0 ? 1.0f : __fdiv_rn(magnitude, 7.0f); + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + if (i < bytes) packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + // 多行复用同一 block 时，防止下行加载覆盖仍在打包/写出的上一行。 + if (rows - row > gridDim.x) __syncthreads(); + } +} + +// 每个线程独占一个蝶形的两个输入/输出，因此单层没有线程间读写冲突， +// 只需在本层结束后同步。此算法不假定 warp 宽度，不使用 warp 级隐式同步。 +template +__global__ void optimized_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, int n, float scale) { + __shared__ float values[256]; + __shared__ float maxima[128]; + __shared__ float row_scale; + const int tid = threadIdx.x; + for (std::size_t row = blockIdx.x; row < rows; row += gridDim.x) { + const std::size_t offset = row * static_cast(n); + for (int i = tid; i < n; i += blockDim.x) values[i] = read_value(input[offset + i]); + __syncthreads(); + if constexpr (Transform) { + for (int stride = 1; stride < n; stride *= 2) { + if (tid < n / 2) { + const int low = ((tid & ~(stride - 1)) << 1) | (tid & (stride - 1)); + const float a = values[low], b = values[low + stride]; + values[low] = a + b; + values[low + stride] = a - b; + } + __syncthreads(); + } + } + if constexpr (!Quantize) { + for (int i = tid; i < n; i += blockDim.x) + output[offset + i] = store_value(values[i] * scale); + } else { + float magnitude = 0; + for (int i = tid; i < n; i += blockDim.x) { + float value = values[i]; + if constexpr (Transform) { + value = read_value(store_value(value * scale)); + values[i] = value; + } + magnitude = fmaxf(magnitude, fabsf(value)); + } + maxima[tid] = magnitude; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 1; stride /= 2) { + if (tid < stride) maxima[tid] = fmaxf(maxima[tid], maxima[tid + stride]); + __syncthreads(); + } + if (tid == 0) { + // 最后两个值只由线程 0 消费，与 scale 写入合并，少一次屏障。 + const float maximum = fmaxf(maxima[0], maxima[1]); + row_scale = maximum == 0 ? 1.0f : __fdiv_rn(maximum, 7.0f); + scales[row] = row_scale; + } + __syncthreads(); + const int bytes = (n + 1) / 2; + for (int i = tid; i < bytes; i += blockDim.x) + packed[row * bytes + i] = quantize_pair(values, i, n, row_scale); + } + if (rows - row > gridDim.x) __syncthreads(); + } +} + +#if defined(HADAMARD_MOORE_SHUFFLE32) + +// Only select this candidate after the native MUSA width=32 shuffle probe passes. +// A CTA has eight logical 32-lane row groups. Tail groups execute with zero values; +// all 256 threads participate in each exchange, including the two reported 128-lane units. +__device__ int shuffle32_quantized_nibble(float value, float scale) { + const float x = __fdiv_rn(value, scale); + const float lower = floorf(x); + const float fraction = x - lower; + int q = static_cast(lower); + if (fraction > 0.5f || (fraction == 0.5f && q % 2 != 0)) ++q; + q = q < -7 ? -7 : (q > 7 ? 7 : q); + return q & 15; +} + +template +__global__ void shuffle32_kernel(const T* input, T* output, std::uint8_t* packed, + float* scales, std::size_t rows, float scale) { + constexpr int width = 32; + constexpr int rows_per_block = 8; + constexpr int registers = N > width ? N / width : 1; + const int lane = threadIdx.x % width; + const int warp = threadIdx.x / width; + const std::size_t first_row = static_cast(blockIdx.x) * rows_per_block; + const std::size_t row_stride = static_cast(gridDim.x) * rows_per_block; + // Every thread in a CTA executes the same number of row-batch iterations. + for (std::size_t row_base = first_row; row_base < rows; row_base += row_stride) { + const std::size_t row = row_base + warp; + const bool active_row = row < rows; + const std::size_t offset = row * N; + float values[registers]; + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + values[r] = active_row && i < N ? read_value(input[offset + i]) : 0.0f; + } + if constexpr (Transform) { + #pragma unroll + for (int stride = 1; stride < (N < width ? N : width); stride *= 2) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const float current = values[r]; + const float peer = __shfl_xor_sync(0xffffffffu, current, stride, width); + values[r] = (lane & stride) ? peer - current : current + peer; + } + } + // Higher butterfly stages exchange registers owned by the same lane. + #pragma unroll + for (int stride = 1; stride < registers; stride *= 2) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + if (!(r & stride)) { + const float x = values[r], y = values[r + stride]; + values[r] = x + y; + values[r + stride] = x - y; + } + } + } + } + if constexpr (!Quantize) { + #pragma unroll + for (int r = 0; r < registers; ++r) { + const int i = lane + r * width; + if (active_row && i < N) output[offset + i] = store_value(values[r] * scale); + } + } else { + float magnitude = 0.0f; + #pragma unroll + for (int r = 0; r < registers; ++r) { + if constexpr (Transform) values[r] = read_value(store_value(values[r] * scale)); + magnitude = fmaxf(magnitude, fabsf(values[r])); + } + #pragma unroll + for (int stride = width / 2; stride > 0; stride /= 2) + magnitude = fmaxf(magnitude, __shfl_xor_sync(0xffffffffu, magnitude, stride, width)); + const float row_scale = magnitude == 0.0f ? 1.0f : __fdiv_rn(magnitude, 7.0f); + if (active_row && lane == 0) scales[row] = row_scale; + #pragma unroll + for (int r = 0; r < registers; ++r) { + // shuffle 在分支之前：奇数 lane 同样必须提供其相邻元素。 + const float peer = __shfl_xor_sync(0xffffffffu, values[r], 1, width); + const int i = lane + r * width; + if (active_row && (lane & 1) == 0 && i < N) { + const int low = shuffle32_quantized_nibble(values[r], row_scale); + const int high = i + 1 < N ? shuffle32_quantized_nibble(peer, row_scale) : 0; + packed[row * ((N + 1) / 2) + i / 2] + = static_cast(low | (high << 4)); + } + } + } + } +} + +template +musaError_t launch_shuffle32(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, musaStream_t stream) { + // launch() checks rows <= SIZE_MAX / sizeof(float), so rows+7 cannot overflow. + const std::size_t requested = (rows + 7) / 8; + const unsigned int blocks = static_cast(requested < 65535 ? requested : 65535); + #define LAUNCH_SHUFFLE32(N) case N: \ + shuffle32_kernel<<>>( \ + input, output, packed, scales, rows, scale); \ + break + switch (n) { + LAUNCH_SHUFFLE32(1); + LAUNCH_SHUFFLE32(2); + LAUNCH_SHUFFLE32(4); + LAUNCH_SHUFFLE32(8); + LAUNCH_SHUFFLE32(16); + LAUNCH_SHUFFLE32(32); + LAUNCH_SHUFFLE32(64); + LAUNCH_SHUFFLE32(128); + LAUNCH_SHUFFLE32(256); + default: return musaErrorInvalidValue; + } + #undef LAUNCH_SHUFFLE32 + return musaGetLastError(); +} + +#endif // HADAMARD_MOORE_SHUFFLE32 + +bool valid_range(const void* pointer, std::size_t bytes, std::size_t alignment) { + const auto address = reinterpret_cast(pointer); + return pointer != nullptr && address % alignment == 0 + && bytes <= std::numeric_limits::max() - address; +} + +bool overlaps(const void* left, std::size_t left_size, + const void* right, std::size_t right_size) { + // valid_range 已确保两个区间端点的加法不会溢出。 + const auto a = reinterpret_cast(left); + const auto b = reinterpret_cast(right); + return a < b + right_size && b < a + left_size; +} + +template +musaError_t launch(const T* input, T* output, std::uint8_t* packed, float* scales, + std::size_t rows, int n, float scale, musaStream_t stream, Method method) { + if (n < 1 || n > 256 || (n & (n - 1)) != 0 + || !std::isfinite(scale) || scale <= 0 + || (method != Method::Baseline && method != Method::Optimized && method != Method::Shuffle32)) + return musaErrorInvalidValue; + if (rows == 0) return musaSuccess; + + constexpr auto maximum = std::numeric_limits::max(); + const std::size_t row_bytes = static_cast(n) * sizeof(T); + if (rows > maximum / row_bytes || rows > maximum / sizeof(float)) + return musaErrorInvalidValue; + const std::size_t input_bytes = rows * row_bytes; + if (!valid_range(input, input_bytes, alignof(T))) return musaErrorInvalidValue; + if constexpr (Quantize) { + const std::size_t packed_bytes = rows * static_cast((n + 1) / 2); + const std::size_t scale_bytes = rows * sizeof(float); + if (!valid_range(packed, packed_bytes, alignof(std::uint8_t)) + || !valid_range(scales, scale_bytes, alignof(float)) + || overlaps(input, input_bytes, packed, packed_bytes) + || overlaps(input, input_bytes, scales, scale_bytes) + || overlaps(packed, packed_bytes, scales, scale_bytes)) return musaErrorInvalidValue; + } else { + if (!valid_range(output, input_bytes, alignof(T)) + || (input != output && overlaps(input, input_bytes, output, input_bytes))) + return musaErrorInvalidValue; + } + + if (method == Method::Shuffle32) { + #if defined(HADAMARD_MOORE_SHUFFLE32) + return launch_shuffle32(input, output, packed, scales, rows, n, scale, stream); + #else + return musaErrorNotSupported; + #endif + } + + // 保守上限兼容不同设备的 grid.x 限制，超过上限由 block 顺序处理多行。 + const unsigned int blocks = static_cast(rows < 65535 ? rows : 65535); + if (method == Method::Baseline) { + const int threads = n < 128 ? 128 : n; + baseline_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } else { + const int threads = 128; + optimized_kernel<<>>( + input, output, packed, scales, rows, n, scale); + } + return musaGetLastError(); +} + +} // namespace + +#define DEFINE_TYPED_API(T) \ +musaError_t transform(const T* input, T* output, std::size_t rows, int n, float scale, \ + musaStream_t stream, Method method) { \ + return launch(input, output, nullptr, nullptr, rows, n, scale, stream, method); \ +} \ +musaError_t quantize_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, musaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, 1.0f, stream, method); \ +} \ +musaError_t transform_int4(const T* input, std::uint8_t* packed, float* scales, \ + std::size_t rows, int n, float scale, musaStream_t stream, Method method) { \ + return launch(input, nullptr, packed, scales, rows, n, scale, stream, method); \ +} + +DEFINE_TYPED_API(__half) +DEFINE_TYPED_API(__mt_bfloat16) +#undef DEFINE_TYPED_API + +} // namespace hadamard::moore_control + diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/paired_benchmark.mu b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/paired_benchmark.mu new file mode 100644 index 00000000..7cc7fde3 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/paired_benchmark.mu @@ -0,0 +1,601 @@ +// Ported from the verified MetaX implementation at cb82ea6. +// MUSA results require independent validation on Moore Threads hardware. +#include "hadamard_api.h" +#include "control_api.h" +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace api = hadamard::moore; +#define CHECK(call) do { musaError_t e_ = (call); if (e_ != musaSuccess) \ + throw std::runtime_error(std::string(#call) + ": " + musaGetErrorString(e_)); } while (0) + +template float read(T); +template<> float read(__half x) { return __half2float(x); } +template<> float read(__mt_bfloat16 x) { return __bfloat162float(x); } +template T rounded(float); +template<> __half rounded(float x) { return __float2half_rn(x); } +template<> __mt_bfloat16 rounded(float x) { return __float2bfloat16_rn(x); } + +std::vector active_methods() { + std::vector methods{api::Method::Baseline, api::Method::Optimized}; +#if defined(HADAMARD_MOORE_SHUFFLE32) + methods.push_back(api::Method::Shuffle32); +#endif + return methods; +} + +const char* method_name(api::Method method) { + if (method == api::Method::Baseline) return "baseline"; + if (method == api::Method::Optimized) return "optimized"; + return "shuffle32"; +} + +// 每个区间都有前后哨兵。17 个元素的偏移同时覆盖仅 2 字节对齐的 FP16/BF16 指针。 +template class Guarded { + static constexpr std::size_t guard = 17; + T* raw_ = nullptr; + std::size_t count_; + std::vector initial_; + std::string name_; +public: + explicit Guarded(std::size_t count, musaStream_t stream, const char* name = "unnamed") + : count_(count), initial_((count + 2 * guard) * sizeof(T), 0xa5), name_(name) { + CHECK(musaMalloc(reinterpret_cast(&raw_), initial_.size())); + // 初始化、上传、kernel、回读使用同一个非阻塞 stream。默认 stream 上的 + // pageable H2D 即使通过 musaMemcpy 调用，也不保证返回时最终 DMA 已结束。 + // 按字节 memset 避免异步初始化读取随后被 upload 更新的主机 shadow。 + CHECK(musaMemsetAsync(raw_, 0xa5, initial_.size(), stream)); + } + ~Guarded() { if (raw_) musaFree(raw_); } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + T* data() { return raw_ + guard; } + void upload(const std::vector& values, musaStream_t stream) { + if (values.size() != count_) throw std::runtime_error("upload size mismatch"); + std::memcpy(initial_.data() + guard * sizeof(T), values.data(), count_ * sizeof(T)); + CHECK(musaMemcpyAsync(raw_, initial_.data(), initial_.size(), musaMemcpyHostToDevice, stream)); + } + std::vector download(musaStream_t stream, bool unchanged = false, const char* phase = "readback") { + std::vector bytes(initial_.size()); + CHECK(musaMemcpyAsync(bytes.data(), raw_, bytes.size(), musaMemcpyDeviceToHost, stream)); + CHECK(musaStreamSynchronize(stream)); + const std::size_t prefix = guard * sizeof(T), end = prefix + count_ * sizeof(T); + for (std::size_t i = 0; i < bytes.size(); ++i) { + if (bytes[i] == initial_[i] || (!unchanged && i >= prefix && i < end)) continue; + const bool is_guard = i < prefix || i >= end; + throw std::runtime_error(std::string(is_guard ? "device buffer guard overwritten" : "read-only input modified") + + " buffer=" + name_ + " phase=" + phase + " region=" + (i < prefix ? "prefix" : (i >= end ? "suffix" : "payload")) + + " byte_from_payload=" + std::to_string(static_cast(i) - static_cast(prefix)) + + " expected=" + std::to_string(static_cast(initial_[i])) + + " actual=" + std::to_string(static_cast(bytes[i])) + + " elements=" + std::to_string(count_) + " element_bytes=" + std::to_string(sizeof(T))); + } + std::vector result(count_); + std::memcpy(result.data(), bytes.data() + prefix, count_ * sizeof(T)); + return result; + } +}; + +struct Options { + bool validate = false, benchmark = false, custom_shape = false, quick = false; + std::size_t batch = 1, seq = 1, heads = 1; + int dim = 128, repeats = 100, groups = 5; + std::string dtype = "both", csv = "moore_benchmark.csv", json = "moore_validation.json"; +}; + +std::size_t positive(const std::string& text, const char* name) { + if (text.empty() || text.find_first_not_of("0123456789") != std::string::npos) + throw std::invalid_argument(std::string(name) + " must be a positive integer"); + std::size_t used = 0; + const auto value = std::stoull(text, &used); + if (!value || value > std::numeric_limits::max() || used != text.size()) + throw std::invalid_argument(std::string(name) + " is outside its supported range"); + return static_cast(value); +} + +std::size_t multiply(std::size_t a, std::size_t b) { + if (b && a > std::numeric_limits::max() / b) + throw std::invalid_argument("shape product overflows size_t"); + return a * b; +} + +std::size_t checked_shape(std::size_t b, std::size_t s, std::size_t h, int n) { + if (n < 1 || n > 256 || !hadamard::power_of_two(n)) + throw std::invalid_argument("dim must be a power of two in [1,256]"); + const auto rows = multiply(multiply(b, s), h); + if (!rows || rows > static_cast(std::numeric_limits::max())) + throw std::invalid_argument("rows outside supported grid range"); + multiply(multiply(rows, static_cast(n)), sizeof(__half)); + return rows; +} + +Options parse(int argc, char** argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string key = argv[i]; + if (key == "--validate") o.validate = true; + else if (key == "--benchmark") o.benchmark = true; + else if (key == "--quick") o.quick = true; + else { + if (i + 1 == argc) throw std::invalid_argument("missing value for " + key); + const std::string value = argv[++i]; + if (key == "--dtype") o.dtype = value; + else if (key == "--csv") o.csv = value; + else if (key == "--json") o.json = value; + else if (key == "--batch" || key == "--seq" || key == "--heads" || key == "--dim") { + const auto v = positive(value, key.c_str()); + o.custom_shape = true; + if (key == "--batch") o.batch = v; + else if (key == "--seq") o.seq = v; + else if (key == "--heads") o.heads = v; + else { + if (v > 256) throw std::invalid_argument("dim is greater than 256"); + o.dim = static_cast(v); + } + } else if (key == "--repeats" || key == "--groups") { + const auto v = positive(value, key.c_str()); + if (v > 10000) throw std::invalid_argument("repeats/groups exceed 10000"); + if (key == "--repeats") o.repeats = static_cast(v); + else o.groups = static_cast(v); + } else throw std::invalid_argument("unknown argument " + key); + } + } + if (!o.validate && !o.benchmark) throw std::invalid_argument("specify --validate and/or --benchmark"); + if (o.dtype != "both" && o.dtype != "fp16" && o.dtype != "bf16") + throw std::invalid_argument("dtype must be fp16, bf16 or both"); + checked_shape(o.batch, o.seq, o.heads, o.dim); + return o; +} + +template std::vector make_input(std::size_t rows, int n, int pattern, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::normal_distribution normal(0.0f, 0.5f); + std::vector result(rows * n); + for (std::size_t i = 0; i < result.size(); ++i) { + float x = 0; + if (pattern == 0) x = uniform(rng); + else if (pattern == 1) x = normal(rng); + else if (pattern == 2) x = i % n == (i / n + seed) % n ? 8.0f : uniform(rng); + else if (pattern == 4) x = i % n == (i / n) % n ? 1.0f : 0.0f; + result[i] = rounded(x); + } + return result; +} + +struct Summary { + std::size_t cases = 0, elements = 0, exact_transform_elements = 0, contract_checks = 0; + std::size_t shuffle32_cases = 0, exact_shuffle32_elements = 0; + std::size_t shuffle32_grid_stride_cases = 0, shuffle32_grid_stride_elements = 0, unsupported_shuffle32_checks = 0; + double max_rounded_error = 0, max_unrounded_error = 0; +}; + +template void one_case(musaStream_t stream, std::size_t rows, int n, float scale, + int pattern, unsigned seed, const char* dtype, Summary& summary) { + const auto input = make_input(rows, n, pattern, seed); + const std::size_t size = input.size(), bytes = rows * ((n + 1) / 2); + Guarded x(size, stream), baseline(size, stream), optimized(size, stream), inplace(size, stream); + Guarded base_split(bytes, stream), opt_split(bytes, stream), base_fused(bytes, stream), opt_fused(bytes, stream); + Guarded bs(rows, stream), os(rows, stream), bfs(rows, stream), ofs(rows, stream); + x.upload(input, stream); + inplace.upload(input, stream); + CHECK(api::transform(x.data(), baseline.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform(x.data(), optimized.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::transform(inplace.data(), inplace.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::quantize_int4(baseline.data(), base_split.data(), bs.data(), rows, n, stream, api::Method::Baseline)); + CHECK(api::quantize_int4(optimized.data(), opt_split.data(), os.data(), rows, n, stream, api::Method::Optimized)); + CHECK(api::transform_int4(x.data(), base_fused.data(), bfs.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform_int4(x.data(), opt_fused.data(), ofs.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(musaStreamSynchronize(stream)); + x.download(stream, true); + const auto a = baseline.download(stream), b = optimized.download(stream), ip = inplace.download(stream); + const std::string context = std::string(dtype) + " rows=" + std::to_string(rows) + " n=" + std::to_string(n) + + " pattern=" + std::to_string(pattern) + " seed=" + std::to_string(seed) + " scale=" + std::to_string(scale); + if (std::memcmp(a.data(), b.data(), size * sizeof(T)) || std::memcmp(b.data(), ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/optimized/in-place transform not bitwise identical: " + context); + std::vector fx(size), actual(size); + for (std::size_t i = 0; i < size; ++i) { fx[i] = read(input[i]); actual[i] = read(b[i]); } + // 全部元素使用 O(N^2) FP64 稠密矩阵公式，不调用任何 FWHT 作为正确性 oracle。 + const auto dense = hadamard::dense_reference(fx, n, static_cast(scale)); + const double tolerance = std::string(dtype) == "fp16" ? 1e-2 : 5e-2; + for (std::size_t i = 0; i < size; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + const double error = std::abs(static_cast(actual[i]) - expected); + if (!std::isfinite(actual[i]) || !(error < tolerance)) + throw std::runtime_error("rounded FP64 dense oracle mismatch: " + context + " index=" + std::to_string(i) + + " error=" + std::to_string(error)); + summary.max_rounded_error = std::max(summary.max_rounded_error, error); + summary.max_unrounded_error = std::max(summary.max_unrounded_error, std::abs(actual[i] - dense[i])); + } + const auto expected_q = hadamard::quantize_int4(actual, n); + auto compare_packed = [&](const char* name, const std::vector& got) { + for (std::size_t i = 0; i < got.size(); ++i) { + if (got[i] != expected_q.packed[i]) { + throw std::runtime_error(std::string(name) + " packed mismatch: " + context + + " index=" + std::to_string(i) + " got=" + std::to_string(got[i]) + + " expected=" + std::to_string(expected_q.packed[i])); + } + } + }; + auto compare_scales = [&](const char* name, const std::vector& got) { + for (std::size_t i = 0; i < got.size(); ++i) { + if (got[i] != expected_q.scales[i]) { + std::uint32_t actual_bits = 0, expected_bits = 0; + std::memcpy(&actual_bits, &got[i], sizeof(float)); + std::memcpy(&expected_bits, &expected_q.scales[i], sizeof(float)); + std::ostringstream message; + message << name << " scale mismatch: " << context << " index=" << i + << std::setprecision(15) << " got=" << got[i] << " expected=" << expected_q.scales[i] + << " got_bits=" << actual_bits << " expected_bits=" << expected_bits; + throw std::runtime_error(message.str()); + } + } + }; + compare_packed("baseline.split", base_split.download(stream)); + compare_packed("optimized.split", opt_split.download(stream)); + compare_packed("baseline.fused", base_fused.download(stream)); + compare_packed("optimized.fused", opt_fused.download(stream)); + compare_scales("baseline.split", bs.download(stream)); + compare_scales("optimized.split", os.download(stream)); + compare_scales("baseline.fused", bfs.download(stream)); + compare_scales("optimized.fused", ofs.download(stream)); +#if defined(HADAMARD_MOORE_SHUFFLE32) + Guarded warp_output(size, stream, "shuffle32.output"), warp_inplace(size, stream, "shuffle32.inplace"); + Guarded warp_split(bytes, stream, "shuffle32.split"), warp_fused(bytes, stream, "shuffle32.fused"); + Guarded warp_split_scales(rows, stream, "shuffle32.split-scales"), warp_fused_scales(rows, stream, "shuffle32.fused-scales"); + warp_inplace.upload(input, stream); + CHECK(api::transform(x.data(), warp_output.data(), rows, n, scale, stream, api::Method::Shuffle32)); + CHECK(api::transform(warp_inplace.data(), warp_inplace.data(), rows, n, scale, stream, api::Method::Shuffle32)); + CHECK(api::quantize_int4(warp_output.data(), warp_split.data(), warp_split_scales.data(), rows, n, stream, api::Method::Shuffle32)); + CHECK(api::transform_int4(x.data(), warp_fused.data(), warp_fused_scales.data(), rows, n, scale, stream, api::Method::Shuffle32)); + const auto warp = warp_output.download(stream), warp_ip = warp_inplace.download(stream); + if (std::memcmp(a.data(), warp.data(), size * sizeof(T)) || std::memcmp(a.data(), warp_ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/Shuffle32/in-place transform not bitwise identical: " + context); + if (warp_split.download(stream) != expected_q.packed || warp_fused.download(stream) != expected_q.packed || + warp_split_scales.download(stream) != expected_q.scales || warp_fused_scales.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/Shuffle32 split/fused INT4 bytes or scales mismatch: " + context); + x.download(stream, true, "after-shuffle32"); + ++summary.shuffle32_cases; + summary.exact_shuffle32_elements += size; +#endif + ++summary.cases; + summary.elements += size; + summary.exact_transform_elements += size; +} + +template void contract_tests(musaStream_t stream, Summary& summary) { + Guarded input(64, stream, "contract.input"), output(64, stream, "contract.output"); + Guarded packed(32, stream, "contract.packed"); + Guarded scales(8, stream, "contract.scales"); + input.download(stream, true, "initialized"); + output.download(stream, true, "initialized"); + packed.download(stream, true, "initialized"); + scales.download(stream, true, "initialized"); + auto reject = [&](musaError_t status) { + if (status != musaErrorInvalidValue) throw std::runtime_error("invalid API input did not return musaErrorInvalidValue"); + ++summary.contract_checks; + }; + auto success = [&](musaError_t status) { CHECK(status); ++summary.contract_checks; }; + for (const auto method : active_methods()) { + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=invalid-parameters" << std::endl; + for (int n : {0, 3, 512}) { + reject(api::transform(input.data(), output.data(), 1, n, 1, stream, method)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, n, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, n, 1, stream, method)); + } + for (float scale : {0.0f, -1.0f, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { + reject(api::transform(input.data(), output.data(), 1, 8, scale, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, scale, stream, method)); + } + reject(api::transform(static_cast(nullptr), output.data(), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), static_cast(nullptr), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), input.data() + 1, 1, 8, 1, stream, method)); + reject(api::transform(input.data(), output.data(), std::numeric_limits::max(), 256, 1, stream, method)); + reject(api::quantize_int4(input.data(), reinterpret_cast(input.data()), scales.data(), 1, 8, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), nullptr, 1, 8, 1, stream, method)); + auto* odd = reinterpret_cast(reinterpret_cast(input.data()) + 1); + reject(api::transform(odd, output.data(), 1, 8, 1, stream, method)); + auto* bad_scale = reinterpret_cast(reinterpret_cast(scales.data()) + 2); + reject(api::quantize_int4(input.data(), packed.data(), bad_scale, 1, 8, stream, method)); + success(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, method)); + success(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, method)); + success(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, method)); + input.download(stream, true, "after-invalid-and-zero-rows"); + output.download(stream, true, "after-invalid-and-zero-rows"); + packed.download(stream, false, "after-invalid-and-zero-rows"); + scales.download(stream, false, "after-invalid-and-zero-rows"); + // 正负半整数：预期手写，避免舍入测试仅复用 CPU 参考实现。 + const std::vector ties{7, -7, .5f, 1.5f, 2.5f, -.5f, -1.5f, -2.5f}; + std::vector t(64, rounded(0)); + for (std::size_t i = 0; i < ties.size(); ++i) t[i] = rounded(ties[i]); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-upload" << std::endl; + input.upload(t, stream); + input.download(stream, true, "after-ties-upload"); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-quantize" << std::endl; + CHECK(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, method)); + const auto q = packed.download(stream, false, "after-ties-quantize"); + const auto s = scales.download(stream, false, "after-ties-quantize"); + const std::array expected{{0x97, 0x20, 0x02, 0xee}}; + if (!std::equal(expected.begin(), expected.end(), q.begin()) || s[0] != 1.0f) + throw std::runtime_error("positive/negative ties-to-even test failed"); + input.download(stream, true, "after-ties-quantize"); + ++summary.contract_checks; + // A rounded FP32 division lies just below the half-integer boundary. + // This checks the quantization contract independently of the CPU helper. + std::vector boundary(64, rounded(0)); + boundary[0] = rounded(0.75f); + boundary[1] = rounded(0.25f); + input.upload(boundary, stream); + CHECK(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 2, 1.0f, stream, method)); + const auto boundary_q = packed.download(stream); + const auto boundary_s = scales.download(stream); + if (boundary_q[0] != 0x37 || boundary_s[0] != 1.0f / 7.0f) + throw std::runtime_error("FP32 division midpoint regression failed"); + ++summary.contract_checks; + // Regression for the observed one-ULP row-scale difference on S4000. + boundary.assign(64, rounded(0)); + boundary[0] = rounded(0.5009765625f); + input.upload(boundary, stream); + CHECK(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 1, stream, method)); + const auto scale_witness = scales.download(stream); + const auto nibble_witness = packed.download(stream); + if (scale_witness[0] != read(boundary[0]) / 7.0f || nibble_witness[0] != 0x07) + throw std::runtime_error("single-value scale RNE regression failed"); + ++summary.contract_checks; + } + const auto invalid = static_cast(-1); + reject(api::transform(input.data(), output.data(), 1, 8, 1, stream, invalid)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, invalid)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, invalid)); +#if !defined(HADAMARD_MOORE_SHUFFLE32) + auto unsupported = [&](musaError_t status) { + if (status != musaErrorNotSupported) throw std::runtime_error("uncompiled Shuffle32 path did not return musaErrorNotSupported"); + ++summary.unsupported_shuffle32_checks; + }; + unsupported(api::transform(input.data(), output.data(), 1, 8, 1, stream, api::Method::Shuffle32)); + unsupported(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, api::Method::Shuffle32)); + unsupported(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, api::Method::Shuffle32)); + CHECK(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, api::Method::Shuffle32)); + CHECK(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, api::Method::Shuffle32)); + CHECK(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, api::Method::Shuffle32)); + summary.unsupported_shuffle32_checks += 3; +#endif + CHECK(musaStreamSynchronize(stream)); + output.download(stream, true); +} + + +template Summary validate(musaStream_t stream, const char* dtype, const Options& options) { + Summary result; + const std::vector dims = options.custom_shape ? std::vector{options.dim} + : (options.quick ? std::vector{1, 64, 256} : std::vector{1, 2, 4, 8, 16, 32, 64, 128, 256}); + const std::vector rows = options.custom_shape + ? std::vector{checked_shape(options.batch, options.seq, options.heads, options.dim)} + : (options.quick ? std::vector{3} : std::vector{1, 3, 17, 257}); + for (int n : dims) { + for (auto r : rows) { + for (int normalized = 0; normalized < (n == 1 ? 1 : 2); ++normalized) { + const float scale = normalized ? 1.0f / std::sqrt(static_cast(n)) : 1.0f; + for (int pattern = 0; pattern < 5; ++pattern) { + // 零值/脉冲无随机性，只计一次；N=1 的两种 scale 相同，也只计一次。 + const int seeds = pattern < 3 && !options.quick ? 3 : 1; + for (int seed = 0; seed < seeds; ++seed) + one_case(stream, r, n, scale, pattern, 123 + 7919 * seed, dtype, result); + } + } + } + std::cout << "VALIDATION_PROGRESS dtype=" << dtype << " n=" << n << " cases=" << result.cases << std::endl; + } + if (!options.quick && !options.custom_shape) { + // 超过 65535 个 block 的网格上限，让同一个 block 必须处理下一行。 + for (int n : {1, 2}) one_case(stream, 65537, n, 1.0f, 0, 1847, dtype, result); +#if defined(HADAMARD_MOORE_SHUFFLE32) + // Eight row groups per CTA: 524281 covers grid reuse and an incomplete CTA. + for (int n : {1, 2}) { + one_case(stream, 524281, n, 1.0f, 0, 4541, dtype, result); + ++result.shuffle32_grid_stride_cases; + result.shuffle32_grid_stride_elements += 524281u * n; + } +#endif + } + contract_tests(stream, result); + std::cout << "VALIDATION_PASS dtype=" << dtype << " cases=" << result.cases << " elements=" << result.elements + << " max_rounded_error=" << std::setprecision(12) << result.max_rounded_error + << " max_unrounded_error=" << result.max_unrounded_error << " contract_checks=" << result.contract_checks + << " shuffle32_cases=" << result.shuffle32_cases << " exact_shuffle32_elements=" << result.exact_shuffle32_elements + << " shuffle32_grid_stride_cases=" << result.shuffle32_grid_stride_cases + << " unsupported_shuffle32_checks=" << result.unsupported_shuffle32_checks << std::endl; + return result; +} + +template void benchmark(musaStream_t stream, const char* dtype, const Options& o, std::ofstream& csv) { + struct Shape { std::size_t b, s, h; int n; }; + std::vector shapes; + if (o.custom_shape) shapes.push_back({o.batch, o.seq, o.heads, o.dim}); + else for (int n : {64, 128, 256}) for (auto rows : {1, 17, 257, 4096, 16384}) + shapes.push_back({rows >= 4096 ? static_cast(rows / 1024) : 1, + rows >= 4096 ? 64 : static_cast(rows), rows >= 4096 ? 16u : 1u, n}); + struct Configuration { api::Method method; int operation; const char* name; bool control = false; }; + // 无 Shuffle32 宏时保留原来的 6 路顺序；启用后追加 3 路，独立输出新的原始样本。 + std::vector configurations; + for (int operation = 0; operation < 3; ++operation) { + const char* baseline[] = {"baseline_transform", "baseline_split", "baseline_fused"}; + const char* old_shared[] = {"control_optimized_transform", "control_optimized_split", "control_optimized_fused"}; + const char* new_shared[] = {"optimized_transform", "optimized_split", "optimized_fused"}; + const char* old_shuffle[] = {"control_shuffle32_transform", "control_shuffle32_split", "control_shuffle32_fused"}; + const char* new_shuffle[] = {"shuffle32_transform", "shuffle32_split", "shuffle32_fused"}; + configurations.push_back({api::Method::Baseline, operation, baseline[operation], false}); + configurations.push_back({api::Method::Optimized, operation, old_shared[operation], true}); + configurations.push_back({api::Method::Optimized, operation, new_shared[operation], false}); + configurations.push_back({api::Method::Shuffle32, operation, old_shuffle[operation], true}); + configurations.push_back({api::Method::Shuffle32, operation, new_shuffle[operation], false}); + } + const int configuration_count = static_cast(configurations.size()); + for (const auto shape : shapes) { + const auto rows = checked_shape(shape.b, shape.s, shape.h, shape.n), count = rows * shape.n; + const auto input = make_input(rows, shape.n, 0, 2909); + Guarded x(count, stream), y(count, stream); + Guarded q(rows * ((shape.n + 1) / 2), stream); + Guarded s(rows, stream); + x.upload(input, stream); + const float scale = 1.0f; + auto launch = [&](int which) { + const auto cfg = configurations[which]; + if (cfg.control) { + const auto method = static_cast(static_cast(cfg.method)); + if (cfg.operation < 2) CHECK(hadamard::moore_control::transform(x.data(), y.data(), rows, shape.n, scale, stream, method)); + if (cfg.operation == 1) CHECK(hadamard::moore_control::quantize_int4(y.data(), q.data(), s.data(), rows, shape.n, stream, method)); + if (cfg.operation == 2) CHECK(hadamard::moore_control::transform_int4(x.data(), q.data(), s.data(), rows, shape.n, scale, stream, method)); + } else { + if (cfg.operation < 2) CHECK(api::transform(x.data(), y.data(), rows, shape.n, scale, stream, cfg.method)); + if (cfg.operation == 1) CHECK(api::quantize_int4(y.data(), q.data(), s.data(), rows, shape.n, stream, cfg.method)); + if (cfg.operation == 2) CHECK(api::transform_int4(x.data(), q.data(), s.data(), rows, shape.n, scale, stream, cfg.method)); + } + }; + // Freeze an output reference before timing and verify every method separately. + CHECK(api::transform(x.data(), y.data(), rows, shape.n, scale, stream, api::Method::Baseline)); + const auto baseline_values = y.download(stream); + std::vector reference_values(count); + for (std::size_t i = 0; i < count; ++i) reference_values[i] = read(baseline_values[i]); + const auto expected_quantized = hadamard::quantize_int4(reference_values, shape.n); + // Full matrices use the all-element dense oracle; large timing shapes use + // three independently checked rows plus full-array method/INT4 comparisons. + for (std::size_t row : {std::size_t(0), rows / 2, rows - 1}) { + std::vector sample(shape.n); + for (int i = 0; i < shape.n; ++i) sample[i] = read(input[row * shape.n + i]); + const auto dense = hadamard::dense_reference(sample, shape.n, scale); + for (int i = 0; i < shape.n; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + if (!(std::abs(reference_values[row * shape.n + i] - expected) + < (std::string(dtype) == "fp16" ? .01 : .05))) + throw std::runtime_error("benchmark reference dense oracle mismatch"); + } + } + auto verify_output = [&](int which) { + const auto cfg = configurations[which]; + if (cfg.operation != 2) { + const auto values = y.download(stream); + if (std::memcmp(values.data(), baseline_values.data(), count * sizeof(T))) + throw std::runtime_error(std::string("benchmark transform mismatch: ") + cfg.name); + } + if (cfg.operation != 0) { + if (q.download(stream) != expected_quantized.packed || s.download(stream) != expected_quantized.scales) + throw std::runtime_error(std::string("benchmark INT4 mismatch: ") + cfg.name); + } + x.download(stream, true); + }; + for (int which = 0; which < configuration_count; ++which) { + launch(which); + verify_output(which); + } + for (int which = 0; which < configuration_count; ++which) for (int i = 0; i < 10; ++i) launch(which); + CHECK(musaStreamSynchronize(stream)); + musaEvent_t begin, end; + CHECK(musaEventCreate(&begin)); CHECK(musaEventCreate(&end)); + for (int group = 0; group < o.groups; ++group) { + // 各组轮换方法顺序；两端事件之间无分配、CPU 参考或主机设备复制。 + for (int order = 0; order < configuration_count; ++order) { + const int which = (order + group) % configuration_count; + const auto cfg = configurations[which]; + CHECK(musaEventRecord(begin, stream)); + for (int i = 0; i < o.repeats; ++i) launch(which); + CHECK(musaEventRecord(end, stream)); + CHECK(musaEventSynchronize(end)); + float elapsed = 0; + CHECK(musaEventElapsedTime(&elapsed, begin, end)); + const double us = static_cast(elapsed) * 1000.0 / o.repeats; + if (!(us > 0) || !std::isfinite(us)) throw std::runtime_error("invalid event timing"); + const std::size_t logical_bytes = cfg.operation == 0 ? count * sizeof(T) * 2 + : (cfg.operation == 1 ? count * sizeof(T) * 3 : count * sizeof(T)) + rows * ((shape.n + 1) / 2) + rows * sizeof(float); + csv << dtype << ',' << shape.b << ',' << shape.s << ',' << shape.h << ',' << shape.n << ',' << rows + << ',' << cfg.name << ',' << group << ',' << order << ',' << o.repeats << ',' << std::setprecision(12) << us + << ',' << logical_bytes << ',' << logical_bytes / us / 1000.0 << ',' << count * sizeof(T) + << ",2909,true,1," << us / 1000.0 << '\n'; + csv.flush(); + if (group + 1 == o.groups) verify_output(which); + } + } + CHECK(musaEventDestroy(begin)); CHECK(musaEventDestroy(end)); + std::cout << "BENCHMARK_PROGRESS dtype=" << dtype << " rows=" << rows << " n=" << shape.n << std::endl; + } +} + +void write_summary(std::ostream& f, const char* dtype, const Summary& s) { + f << '"' << dtype << "\":{\"cases\":" << s.cases << ",\"elements\":" << s.elements + << ",\"exact_baseline_optimized_elements\":" << s.exact_transform_elements + << ",\"shuffle32_cases\":" << s.shuffle32_cases << ",\"exact_baseline_shuffle32_elements\":" << s.exact_shuffle32_elements + << ",\"shuffle32_grid_stride_cases\":" << s.shuffle32_grid_stride_cases + << ",\"shuffle32_grid_stride_elements\":" << s.shuffle32_grid_stride_elements + << ",\"unsupported_shuffle32_checks\":" << s.unsupported_shuffle32_checks + << ",\"max_abs_error_rounded_fp64\":" << std::setprecision(15) << s.max_rounded_error + << ",\"max_abs_error_unrounded_fp64\":" << s.max_unrounded_error + << ",\"api_contract_checks\":" << s.contract_checks << '}'; +} + +int main(int argc, char** argv) { + Options options; + try { options = parse(argc, argv); } + catch (const std::exception& e) { std::cerr << "INVALID_ARGUMENT " << e.what() << '\n'; return 2; } + try { + CHECK(musaSetDevice(0)); + musaDeviceProp prop{}; + CHECK(musaGetDeviceProperties(&prop, 0)); +#if defined(HADAMARD_MOORE_SHUFFLE32) + if (prop.warpSize != 128) throw std::runtime_error("This validated S4000 build requires host-reported warpSize=128"); +#endif + int runtime = 0, driver = 0; + CHECK(musaRuntimeGetVersion(&runtime)); CHECK(musaDriverGetVersion(&driver)); + std::cout << "DEVICE name=" << prop.name << " warp=" << prop.warpSize << " runtime=" << runtime << " driver=" << driver << std::endl; + musaStream_t stream; + CHECK(musaStreamCreateWithFlags(&stream, musaStreamNonBlocking)); + Summary fp16, bf16; + if (options.validate) { + if (options.dtype != "bf16") fp16 = validate<__half>(stream, "fp16", options); + if (options.dtype != "fp16") bf16 = validate<__mt_bfloat16>(stream, "bf16", options); + std::ofstream json(options.json); + if (!json) throw std::runtime_error("cannot create validation JSON " + options.json); + json << "{\"status\":\"PASS\",\"platform\":\"moore\",\"warp_size\":" << prop.warpSize << ",\"full_matrix\":" << ((!options.quick && !options.custom_shape && options.dtype == "both") ? "true" : "false") +#if defined(HADAMARD_MOORE_SHUFFLE32) + << ",\"shuffle32_enabled\":true,\"methods\":[\"baseline\",\"optimized\",\"shuffle32\"]" +#else + << ",\"shuffle32_enabled\":false,\"methods\":[\"baseline\",\"optimized\"]" +#endif + << ",\"oracle\":\"all-element FP64 dense, rounded to output dtype\",\"fp16_tolerance_strict\":0.01,\"bf16_tolerance_strict\":0.05," + << "\"warmup_not_counted\":true,\"dtypes\":{"; + bool comma = false; + if (options.dtype != "bf16") { write_summary(json, "fp16", fp16); comma = true; } + if (options.dtype != "fp16") { if (comma) json << ','; write_summary(json, "bf16", bf16); } + json << "}}\n"; + if (!json) throw std::runtime_error("failed writing validation JSON"); + } + if (options.benchmark) { + std::ofstream csv(options.csv); + if (!csv) throw std::runtime_error("cannot create benchmark CSV " + options.csv); + csv << "dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale,kernel_ms\n"; + if (options.dtype != "bf16") benchmark<__half>(stream, "fp16", options, csv); + if (options.dtype != "fp16") benchmark<__mt_bfloat16>(stream, "bf16", options, csv); + if (!csv) throw std::runtime_error("failed writing benchmark CSV"); + } + CHECK(musaStreamDestroy(stream)); + std::cout << "PASS requested validation/benchmark operations completed" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "FAIL " << e.what() << std::endl; + return 1; + } +} + diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/cpu-equivalence-linux.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/cpu-equivalence-linux.json new file mode 100644 index 00000000..d0235dfa --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/cpu-equivalence-linux.json @@ -0,0 +1,15 @@ +{ + "status": "PASS", + "cpu_cases": 17527208, + "per_dtype": { + "fp16": 8158468, + "bf16": 8388740 + }, + "boundary_cases": 980000, + "gpu_fixture_cases": 2074568, + "fixture_sha256": "b404de04128361ed75347666113cfe2545f7c6e52aa828b070c1812c967ce230", + "header_sha256": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a", + "seconds": 0.2998413380119018, + "numpy_version": "1.23.5", + "scope": "Actual C++ bit-comparison implementation versus existing CPU divide/round reference; this is not a GPU test" +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/cpu-equivalence-macos.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/cpu-equivalence-macos.json new file mode 100644 index 00000000..282623ea --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/cpu-equivalence-macos.json @@ -0,0 +1,15 @@ +{ + "status": "PASS", + "cpu_cases": 17527208, + "per_dtype": { + "fp16": 8158468, + "bf16": 8388740 + }, + "boundary_cases": 980000, + "gpu_fixture_cases": 2074568, + "fixture_sha256": "b404de04128361ed75347666113cfe2545f7c6e52aa828b070c1812c967ce230", + "header_sha256": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a", + "seconds": 7.429811832960695, + "numpy_version": "2.3.5", + "scope": "Actual C++ bit-comparison implementation versus existing CPU divide/round reference; this is not a GPU test" +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/disabled-run-summary.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/disabled-run-summary.json new file mode 100644 index 00000000..4be4b9c1 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/disabled-run-summary.json @@ -0,0 +1,532 @@ +{ + "status": "PASS", + "platform": "moore", + "target_model": "MTT S4000", + "base_commit": "cb82ea6e1d5c8f78b48b6922b0f7af279696cc44", + "source_sha256": { + "platforms/moore/hadamard_api.h": "cba4d5c187defec8f70083a02ba252da8d73f5fba5b1b7692efb91dd1a0c4d93", + "platforms/moore/hadamard_api.mu": "f9c7fb325cbdc19e57e5c02ce58ec973197b75039fff38eb4b1fbac4daec4a80", + "platforms/moore/validate_and_benchmark.mu": "2a23a7938bd3140742b3c45f6b85db08d539bf03d4cb44ee9a48be41a268459e", + "platforms/moore/probe.mu": "2e777447c5153b55a7eee8bd7729170517637c4e04cb3cdbe89c6c51ec58bccd", + "platforms/moore/run_platform.py": "d92b236ca4bcaff910921e2d0ef21f452968bb66f1bd98a1a157c0a7c8a88773", + "platforms/moore/exact_int4.hpp": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a", + "include/reference.hpp": "1ff2c70cc8849020bd413a1d174718683bd89d5b6d8f25ee0a01d1c9b8876888" + }, + "started_utc": "2026-09-15T16:43:15Z", + "python": "3.11.0rc1 (main, Aug 12 2022, 10:02:14) [GCC 11.2.0]", + "arch": "mp_22", + "quick": true, + "shuffle32_enabled": false, + "scope": "validation", + "stages": [ + { + "command": [ + "/usr/local/musa-5.1.0/bin/mcc", + "-std=c++17", + "-O2", + "-fno-fast-math", + "-ffp-contract=off", + "--offload-arch=mp_22", + "-I/data/moore-opt-20260916/project/include", + "/data/moore-opt-20260916/project/platforms/moore/probe.mu", + "-L/usr/local/musa-5.1.0/lib", + "-lmusart", + "-o", + "/data/moore-opt-20260916/runs/disabled/probe" + ], + "log": "probe_build.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 1.6696049720048904, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/probe" + ], + "log": "probe.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 0.06294345698552206, + "sha256": "185e3405158ba1df91d2b5337045438c522c08d26b06ade0e0226ad075c6a4a1" + }, + { + "command": [ + "/usr/local/musa-5.1.0/bin/mcc", + "-std=c++17", + "-O2", + "-fno-fast-math", + "-ffp-contract=off", + "--offload-arch=mp_22", + "-I/data/moore-opt-20260916/project/include", + "/data/moore-opt-20260916/project/platforms/moore/hadamard_api.mu", + "/data/moore-opt-20260916/project/platforms/moore/validate_and_benchmark.mu", + "-L/usr/local/musa-5.1.0/lib", + "-lmusart", + "-o", + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark" + ], + "log": "build.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 7.21005147905089, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark" + ], + "log": "invalid_00.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0017190529615618289, + "sha256": "57a02a5ddbdd1c62c445fa1feac13e27585b3837bd9d9ed0d3cd869a33e3c26e" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--dim", + "0" + ], + "log": "invalid_01.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.001599579001776874, + "sha256": "db18b9f8d2040efced14ec6027b078290f358387206be6945f703d8bd4b5d7b7" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--dim", + "3" + ], + "log": "invalid_02.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0015864320448599756, + "sha256": "fe9b4ea505976f30aab4a65234e33296fa2f61a7068429b2611ff99644178709" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--dim", + "512" + ], + "log": "invalid_03.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014391890144906938, + "sha256": "699f1f70340c77149d3bae4063f00207638e87be51bb3bb02bb2a99a650e68a1" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--dtype", + "fp32" + ], + "log": "invalid_04.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014366969699040055, + "sha256": "5d1c70e05b6561f86a8c8c5bec7d168cbfc043fdfb9e1e7af32b5a87c1b1ff90" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--batch", + "-1" + ], + "log": "invalid_05.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0016181209939531982, + "sha256": "3bac927d06f4ce7258f9c037afb2b3a5dfdc51f7504c16e4b1dae94bcc91cf1b" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--seq", + "0" + ], + "log": "invalid_06.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014105800073593855, + "sha256": "49316fe0d2d4c9660cf5218d6ee6bc3708e737c38cfc3b53775f0636dc194997" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--heads", + "x" + ], + "log": "invalid_07.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014274860150180757, + "sha256": "6d69299a2872ffc0a140db9fc171769b85e3f61111a5e73399436d32cb0fc27b" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--batch", + "18446744073709551615", + "--seq", + "2" + ], + "log": "invalid_08.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.001469318987801671, + "sha256": "439fd2b48f724414272ea4a69fae98cf1809e904afe56e2e8edd677e50993d56" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--heads", + "999999999999999999999999999999" + ], + "log": "invalid_09.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014070890028961003, + "sha256": "4b8c91252c32d2f7cbcebc21561c29defe89809c97c23be7ab6a3577d4b74a3d" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--dim" + ], + "log": "invalid_10.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.001410211028996855, + "sha256": "76dd633500e539227532274081e41835e98a9cb82d73ba955f92c00ee1606cd0" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--unknown", + "1" + ], + "log": "invalid_11.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0013924940139986575, + "sha256": "be700711fd27b9719a4f6d490ef528894b371271344313d04298e25d9879ba54" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--benchmark", + "--repeats", + "0" + ], + "log": "invalid_12.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.001423505018465221, + "sha256": "eaddd4706d0da47a6fccbd2f81b6ace09f4f708d23cfc9b844f35d94f39d6241" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--benchmark", + "--groups", + "10001" + ], + "log": "invalid_13.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014513840433210135, + "sha256": "2116fe98f2998004c6367f99717fa82db3c3f77374b5c86cbeab03ca8922256e" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/disabled/validate_and_benchmark", + "--validate", + "--json", + "/data/moore-opt-20260916/runs/disabled/validation.json", + "--quick" + ], + "log": "validation.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 0.12584167602472007, + "sha256": "8b096594841bfc815468cf38d8385acd4316f1d13201217a987f287fb7c8d86c" + } + ], + "compiler": { + "command": [ + "/usr/local/musa-5.1.0/bin/mcc", + "--version" + ], + "returncode": 0, + "output": "clang version 14.0.0 (git@sh-code.mthreads.com:sw/mtcc.git 61c54cd8e5c72ef146656c7beee1889a837ed7bc)\nmcc version 5.1.0\nTarget: x86_64-unknown-linux-gnu\nThread model: posix\nInstalledDir: /usr/local/musa-5.1.0/bin\n" + }, + "device_before": { + "command": [ + "mthreads-gmi" + ], + "returncode": 0, + "output": "Wed Sep 16 00:43:15 2026\n---------------------------------------------------------------------\n mthreads-gmi:2.4.1 Driver Version:5.1.0-server\n---------------------------------------------------------------------\nID Name |PCIe |%GPU Mem \n Device Type Perf |Pcie Lane Width |Temp MPC Capable \n | ECC Mode \n+-------------------------------------------------------------------+\n0 MTT S4000 |00000000:08:00.0 |5% 0MiB(49152MiB)\n Physical N/A |16x(16x) |51C YES \n | N/A \n---------------------------------------------------------------------\n\n---------------------------------------------------------------------\nProcesses:\nID PID Process name GPU Memory\n Usage\n+-------------------------------------------------------------------+\n No running processes found\n---------------------------------------------------------------------\n" + }, + "sdk_version": { + "musa_toolkits": { + "version": "5.1.0", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "e1481406739bd545c45c4b852450a81ac5feee6c", + "commit date": "2026-03-06 20:38:51 +0800" + }, + "CUB": { + "version": "1.17.2", + "git branch": "develop-1.17", + "git tag": "No tag", + "commit id": "2aa7e2b6a2e82ca339ca132d345377d6faef6024", + "commit date": "2026-04-22 21:48:03 +0800" + }, + "mcc": { + "version": "5.1.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "61c54cd8e5c72ef146656c7beee1889a837ed7bc", + "commit date": "2026-04-28 15:32:37 +0800" + }, + "mtjpeg": { + "version": "1.0.5", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "070f3ff97088cfdef12da04b09af22bfc862874c", + "commit date": "2026-03-23 13:40:41 +0800" + }, + "mublasLt": { + "version": "1.12.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "b99c7919a587a92ec1f24b75a633d18da754377b", + "commit date": "2026-04-07 14:08:36 +0800" + }, + "mublas": { + "version": "1.12.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "4bced69bfb393863c43c23e9d21fbce6e2ff9ffd", + "commit date": "2026-04-23 19:16:27 +0800" + }, + "mufft": { + "version": "1.12.0", + "build archs": "22;31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "0a2e9ac103e2d54c384947561076c682c6811c1e", + "commit date": "2026-03-11 19:24:16 +0800" + }, + "muPP": { + "version": "1.13.0", + "build archs": "22;31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "0a8b4f2b1c8f9928130fa882a97705be2fc693e2", + "commit date": "2026-03-12 14:48:07 +0800" + }, + "MUPTI": { + "version": "1.2.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "453615f9e030b4c0bd101ce44d60346a1ffee11b", + "commit date": "2026-04-28 12:40:08 +0800" + }, + "murand": { + "version": "1.3.0", + "build archs": "22;31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "525224ed521b4cb0af25ada59b84583241e57761", + "commit date": "2026-03-27 19:58:21 +0800" + }, + "musa_runtime": { + "version": "5.1.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "73921cd20f278bd905e50dbd920c976d93cc18b4", + "commit date": "2026-04-24 13:42:11 +0800" + }, + "musify": { + "version": "1.2.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "a5f79eb9e507dd7bfa1b5bd8b6b833f671c6b8c1", + "commit date": "2026-04-08 14:58:40 +0800" + }, + "musolver": { + "version": "1.6.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "f6062eb2da90ad3b5091a91d3348423649275d12", + "commit date": "2026-04-25 19:06:04 +0800" + }, + "musparse": { + "version": "1.7.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "3b5df2ca75c504c98821bc2cd71a9160ddf4eb55", + "commit date": "2026-04-27 17:08:43 +0800" + }, + "Thrust": { + "version": "1.17.2", + "git branch": "develop-1.17", + "git tag": "No tag", + "commit id": "bffd03bde9af75266cc1a51d46cde48f4bcd9280", + "commit date": "2026-04-22 21:51:23 +0800" + } + }, + "probe_sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0", + "binary_sha256": "593faf17d6b34c45ff826b103ceb2957d71cd9befdaa51ecb0cfa8312c6e141a", + "cli_rejection_cases": 14, + "validation": { + "status": "PASS", + "platform": "moore", + "warp_size": 128, + "full_matrix": false, + "shuffle32_enabled": false, + "methods": [ + "baseline", + "optimized" + ], + "oracle": "all-element FP64 dense, rounded to output dtype", + "fp16_tolerance_strict": 0.01, + "bf16_tolerance_strict": 0.05, + "warmup_not_counted": true, + "dtypes": { + "fp16": { + "cases": 25, + "elements": 9615, + "exact_baseline_optimized_elements": 9615, + "shuffle32_cases": 0, + "exact_baseline_shuffle32_elements": 0, + "shuffle32_grid_stride_cases": 0, + "shuffle32_grid_stride_elements": 0, + "unsupported_shuffle32_checks": 6, + "max_abs_error_rounded_fp64": 0, + "max_abs_error_unrounded_fp64": 0.00782299041748047, + "api_contract_checks": 65 + }, + "bf16": { + "cases": 25, + "elements": 9615, + "exact_baseline_optimized_elements": 9615, + "shuffle32_cases": 0, + "exact_baseline_shuffle32_elements": 0, + "shuffle32_grid_stride_cases": 0, + "shuffle32_grid_stride_elements": 0, + "unsupported_shuffle32_checks": 6, + "max_abs_error_rounded_fp64": 0, + "max_abs_error_unrounded_fp64": 0.062225341796875, + "api_contract_checks": 65 + } + } + }, + "finished_utc": "2026-09-15T16:43:24Z", + "artifacts": { + "validation.log": { + "bytes": 1512, + "sha256": "8b096594841bfc815468cf38d8385acd4316f1d13201217a987f287fb7c8d86c" + }, + "invalid_10.log": { + "bytes": 41, + "sha256": "76dd633500e539227532274081e41835e98a9cb82d73ba955f92c00ee1606cd0" + }, + "invalid_00.log": { + "bytes": 55, + "sha256": "57a02a5ddbdd1c62c445fa1feac13e27585b3837bd9d9ed0d3cd869a33e3c26e" + }, + "validate_and_benchmark": { + "bytes": 331272, + "sha256": "593faf17d6b34c45ff826b103ceb2957d71cd9befdaa51ecb0cfa8312c6e141a" + }, + "invalid_08.log": { + "bytes": 48, + "sha256": "439fd2b48f724414272ea4a69fae98cf1809e904afe56e2e8edd677e50993d56" + }, + "invalid_11.log": { + "bytes": 44, + "sha256": "be700711fd27b9719a4f6d490ef528894b371271344313d04298e25d9879ba54" + }, + "probe.log": { + "bytes": 293, + "sha256": "185e3405158ba1df91d2b5337045438c522c08d26b06ade0e0226ad075c6a4a1" + }, + "invalid_04.log": { + "bytes": 50, + "sha256": "5d1c70e05b6561f86a8c8c5bec7d168cbfc043fdfb9e1e7af32b5a87c1b1ff90" + }, + "probe_build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "invalid_03.log": { + "bytes": 41, + "sha256": "699f1f70340c77149d3bae4063f00207638e87be51bb3bb02bb2a99a650e68a1" + }, + "invalid_12.log": { + "bytes": 58, + "sha256": "eaddd4706d0da47a6fccbd2f81b6ace09f4f708d23cfc9b844f35d94f39d6241" + }, + "invalid_06.log": { + "bytes": 54, + "sha256": "49316fe0d2d4c9660cf5218d6ee6bc3708e737c38cfc3b53775f0636dc194997" + }, + "build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "invalid_13.log": { + "bytes": 45, + "sha256": "2116fe98f2998004c6367f99717fa82db3c3f77374b5c86cbeab03ca8922256e" + }, + "invalid_02.log": { + "bytes": 55, + "sha256": "fe9b4ea505976f30aab4a65234e33296fa2f61a7068429b2611ff99644178709" + }, + "validation.json": { + "bytes": 970, + "sha256": "9e3cf608dc9880c4922d9ef8978b7c16f9e0f75b3051c3d2904c73e19179e368" + }, + "invalid_09.log": { + "bytes": 24, + "sha256": "4b8c91252c32d2f7cbcebc21561c29defe89809c97c23be7ab6a3577d4b74a3d" + }, + "invalid_07.log": { + "bytes": 52, + "sha256": "6d69299a2872ffc0a140db9fc171769b85e3f61111a5e73399436d32cb0fc27b" + }, + "probe": { + "bytes": 25872, + "sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0" + }, + "invalid_01.log": { + "bytes": 54, + "sha256": "db18b9f8d2040efced14ec6027b078290f358387206be6945f703d8bd4b5d7b7" + }, + "invalid_05.log": { + "bytes": 52, + "sha256": "3bac927d06f4ce7258f9c037afb2b3a5dfdc51f7504c16e4b1dae94bcc91cf1b" + } + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/disabled-validation.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/disabled-validation.json new file mode 100644 index 00000000..532bebca --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/disabled-validation.json @@ -0,0 +1 @@ +{"status":"PASS","platform":"moore","warp_size":128,"full_matrix":false,"shuffle32_enabled":false,"methods":["baseline","optimized"],"oracle":"all-element FP64 dense, rounded to output dtype","fp16_tolerance_strict":0.01,"bf16_tolerance_strict":0.05,"warmup_not_counted":true,"dtypes":{"fp16":{"cases":25,"elements":9615,"exact_baseline_optimized_elements":9615,"shuffle32_cases":0,"exact_baseline_shuffle32_elements":0,"shuffle32_grid_stride_cases":0,"shuffle32_grid_stride_elements":0,"unsupported_shuffle32_checks":6,"max_abs_error_rounded_fp64":0,"max_abs_error_unrounded_fp64":0.00782299041748047,"api_contract_checks":65},"bf16":{"cases":25,"elements":9615,"exact_baseline_optimized_elements":9615,"shuffle32_cases":0,"exact_baseline_shuffle32_elements":0,"shuffle32_grid_stride_cases":0,"shuffle32_grid_stride_elements":0,"unsupported_shuffle32_checks":6,"max_abs_error_rounded_fp64":0,"max_abs_error_unrounded_fp64":0.062225341796875,"api_contract_checks":65}}} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/full-run-summary.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/full-run-summary.json new file mode 100644 index 00000000..47f192f0 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/full-run-summary.json @@ -0,0 +1,533 @@ +{ + "status": "PASS", + "platform": "moore", + "target_model": "MTT S4000", + "base_commit": "cb82ea6e1d5c8f78b48b6922b0f7af279696cc44", + "source_sha256": { + "platforms/moore/hadamard_api.h": "cba4d5c187defec8f70083a02ba252da8d73f5fba5b1b7692efb91dd1a0c4d93", + "platforms/moore/hadamard_api.mu": "f9c7fb325cbdc19e57e5c02ce58ec973197b75039fff38eb4b1fbac4daec4a80", + "platforms/moore/validate_and_benchmark.mu": "2a23a7938bd3140742b3c45f6b85db08d539bf03d4cb44ee9a48be41a268459e", + "platforms/moore/probe.mu": "2e777447c5153b55a7eee8bd7729170517637c4e04cb3cdbe89c6c51ec58bccd", + "platforms/moore/run_platform.py": "d92b236ca4bcaff910921e2d0ef21f452968bb66f1bd98a1a157c0a7c8a88773", + "platforms/moore/exact_int4.hpp": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a", + "include/reference.hpp": "1ff2c70cc8849020bd413a1d174718683bd89d5b6d8f25ee0a01d1c9b8876888" + }, + "started_utc": "2026-09-15T16:42:54Z", + "python": "3.11.0rc1 (main, Aug 12 2022, 10:02:14) [GCC 11.2.0]", + "arch": "mp_22", + "quick": false, + "shuffle32_enabled": true, + "scope": "validation", + "stages": [ + { + "command": [ + "/usr/local/musa-5.1.0/bin/mcc", + "-std=c++17", + "-O2", + "-fno-fast-math", + "-ffp-contract=off", + "--offload-arch=mp_22", + "-I/data/moore-opt-20260916/project/include", + "/data/moore-opt-20260916/project/platforms/moore/probe.mu", + "-L/usr/local/musa-5.1.0/lib", + "-lmusart", + "-o", + "/data/moore-opt-20260916/runs/full/probe" + ], + "log": "probe_build.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 1.679499724006746, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/probe" + ], + "log": "probe.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 0.06223989499267191, + "sha256": "185e3405158ba1df91d2b5337045438c522c08d26b06ade0e0226ad075c6a4a1" + }, + { + "command": [ + "/usr/local/musa-5.1.0/bin/mcc", + "-std=c++17", + "-O2", + "-fno-fast-math", + "-ffp-contract=off", + "--offload-arch=mp_22", + "-I/data/moore-opt-20260916/project/include", + "-DHADAMARD_MOORE_SHUFFLE32", + "/data/moore-opt-20260916/project/platforms/moore/hadamard_api.mu", + "/data/moore-opt-20260916/project/platforms/moore/validate_and_benchmark.mu", + "-L/usr/local/musa-5.1.0/lib", + "-lmusart", + "-o", + "/data/moore-opt-20260916/runs/full/validate_and_benchmark" + ], + "log": "build.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 12.295903087011538, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark" + ], + "log": "invalid_00.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0017629349604249, + "sha256": "57a02a5ddbdd1c62c445fa1feac13e27585b3837bd9d9ed0d3cd869a33e3c26e" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--dim", + "0" + ], + "log": "invalid_01.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0016300660208798945, + "sha256": "db18b9f8d2040efced14ec6027b078290f358387206be6945f703d8bd4b5d7b7" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--dim", + "3" + ], + "log": "invalid_02.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0015659669879823923, + "sha256": "fe9b4ea505976f30aab4a65234e33296fa2f61a7068429b2611ff99644178709" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--dim", + "512" + ], + "log": "invalid_03.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0015901500009931624, + "sha256": "699f1f70340c77149d3bae4063f00207638e87be51bb3bb02bb2a99a650e68a1" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--dtype", + "fp32" + ], + "log": "invalid_04.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014455250347964466, + "sha256": "5d1c70e05b6561f86a8c8c5bec7d168cbfc043fdfb9e1e7af32b5a87c1b1ff90" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--batch", + "-1" + ], + "log": "invalid_05.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014556659734807909, + "sha256": "3bac927d06f4ce7258f9c037afb2b3a5dfdc51f7504c16e4b1dae94bcc91cf1b" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--seq", + "0" + ], + "log": "invalid_06.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.001422279980033636, + "sha256": "49316fe0d2d4c9660cf5218d6ee6bc3708e737c38cfc3b53775f0636dc194997" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--heads", + "x" + ], + "log": "invalid_07.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014643610338680446, + "sha256": "6d69299a2872ffc0a140db9fc171769b85e3f61111a5e73399436d32cb0fc27b" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--batch", + "18446744073709551615", + "--seq", + "2" + ], + "log": "invalid_08.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.001547265041153878, + "sha256": "439fd2b48f724414272ea4a69fae98cf1809e904afe56e2e8edd677e50993d56" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--heads", + "999999999999999999999999999999" + ], + "log": "invalid_09.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014633550308644772, + "sha256": "4b8c91252c32d2f7cbcebc21561c29defe89809c97c23be7ab6a3577d4b74a3d" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--dim" + ], + "log": "invalid_10.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014155349927023053, + "sha256": "76dd633500e539227532274081e41835e98a9cb82d73ba955f92c00ee1606cd0" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--unknown", + "1" + ], + "log": "invalid_11.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014265060308389366, + "sha256": "be700711fd27b9719a4f6d490ef528894b371271344313d04298e25d9879ba54" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--benchmark", + "--repeats", + "0" + ], + "log": "invalid_12.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.0014229940134100616, + "sha256": "eaddd4706d0da47a6fccbd2f81b6ace09f4f708d23cfc9b844f35d94f39d6241" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--benchmark", + "--groups", + "10001" + ], + "log": "invalid_13.log", + "returncode": 2, + "expected_returncode": 2, + "wall_seconds": 0.001408968004398048, + "sha256": "2116fe98f2998004c6367f99717fa82db3c3f77374b5c86cbeab03ca8922256e" + }, + { + "command": [ + "/data/moore-opt-20260916/runs/full/validate_and_benchmark", + "--validate", + "--json", + "/data/moore-opt-20260916/runs/full/validation.json" + ], + "log": "validation.log", + "returncode": 0, + "expected_returncode": 0, + "wall_seconds": 7.062892174988519, + "sha256": "70bd5a3305631837b480b31abea36dc67277bb48c89ccc73090dd4062357153b" + } + ], + "compiler": { + "command": [ + "/usr/local/musa-5.1.0/bin/mcc", + "--version" + ], + "returncode": 0, + "output": "clang version 14.0.0 (git@sh-code.mthreads.com:sw/mtcc.git 61c54cd8e5c72ef146656c7beee1889a837ed7bc)\nmcc version 5.1.0\nTarget: x86_64-unknown-linux-gnu\nThread model: posix\nInstalledDir: /usr/local/musa-5.1.0/bin\n" + }, + "device_before": { + "command": [ + "mthreads-gmi" + ], + "returncode": 0, + "output": "Wed Sep 16 00:42:54 2026\n---------------------------------------------------------------------\n mthreads-gmi:2.4.1 Driver Version:5.1.0-server\n---------------------------------------------------------------------\nID Name |PCIe |%GPU Mem \n Device Type Perf |Pcie Lane Width |Temp MPC Capable \n | ECC Mode \n+-------------------------------------------------------------------+\n0 MTT S4000 |00000000:08:00.0 |0% 0MiB(49152MiB)\n Physical N/A |16x(16x) |50C YES \n | N/A \n---------------------------------------------------------------------\n\n---------------------------------------------------------------------\nProcesses:\nID PID Process name GPU Memory\n Usage\n+-------------------------------------------------------------------+\n No running processes found\n---------------------------------------------------------------------\n" + }, + "sdk_version": { + "musa_toolkits": { + "version": "5.1.0", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "e1481406739bd545c45c4b852450a81ac5feee6c", + "commit date": "2026-03-06 20:38:51 +0800" + }, + "CUB": { + "version": "1.17.2", + "git branch": "develop-1.17", + "git tag": "No tag", + "commit id": "2aa7e2b6a2e82ca339ca132d345377d6faef6024", + "commit date": "2026-04-22 21:48:03 +0800" + }, + "mcc": { + "version": "5.1.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "61c54cd8e5c72ef146656c7beee1889a837ed7bc", + "commit date": "2026-04-28 15:32:37 +0800" + }, + "mtjpeg": { + "version": "1.0.5", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "070f3ff97088cfdef12da04b09af22bfc862874c", + "commit date": "2026-03-23 13:40:41 +0800" + }, + "mublasLt": { + "version": "1.12.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "b99c7919a587a92ec1f24b75a633d18da754377b", + "commit date": "2026-04-07 14:08:36 +0800" + }, + "mublas": { + "version": "1.12.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "4bced69bfb393863c43c23e9d21fbce6e2ff9ffd", + "commit date": "2026-04-23 19:16:27 +0800" + }, + "mufft": { + "version": "1.12.0", + "build archs": "22;31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "0a2e9ac103e2d54c384947561076c682c6811c1e", + "commit date": "2026-03-11 19:24:16 +0800" + }, + "muPP": { + "version": "1.13.0", + "build archs": "22;31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "0a8b4f2b1c8f9928130fa882a97705be2fc693e2", + "commit date": "2026-03-12 14:48:07 +0800" + }, + "MUPTI": { + "version": "1.2.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "453615f9e030b4c0bd101ce44d60346a1ffee11b", + "commit date": "2026-04-28 12:40:08 +0800" + }, + "murand": { + "version": "1.3.0", + "build archs": "22;31", + "git branch": "release_musa_5.1.0", + "git tag": "AIMODULE_1.4.0", + "commit id": "525224ed521b4cb0af25ada59b84583241e57761", + "commit date": "2026-03-27 19:58:21 +0800" + }, + "musa_runtime": { + "version": "5.1.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "73921cd20f278bd905e50dbd920c976d93cc18b4", + "commit date": "2026-04-24 13:42:11 +0800" + }, + "musify": { + "version": "1.2.0", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "a5f79eb9e507dd7bfa1b5bd8b6b833f671c6b8c1", + "commit date": "2026-04-08 14:58:40 +0800" + }, + "musolver": { + "version": "1.6.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "f6062eb2da90ad3b5091a91d3348423649275d12", + "commit date": "2026-04-25 19:06:04 +0800" + }, + "musparse": { + "version": "1.7.0", + "build archs": "22,31", + "git branch": "release_musa_5.1.0", + "git tag": "No tag", + "commit id": "3b5df2ca75c504c98821bc2cd71a9160ddf4eb55", + "commit date": "2026-04-27 17:08:43 +0800" + }, + "Thrust": { + "version": "1.17.2", + "git branch": "develop-1.17", + "git tag": "No tag", + "commit id": "bffd03bde9af75266cc1a51d46cde48f4bcd9280", + "commit date": "2026-04-22 21:51:23 +0800" + } + }, + "probe_sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0", + "binary_sha256": "1155d721fe4e48c51bcde8c6fa0cf1f962be313e742ea102962c0a3353a91cd8", + "cli_rejection_cases": 14, + "validation": { + "status": "PASS", + "platform": "moore", + "warp_size": 128, + "full_matrix": true, + "shuffle32_enabled": true, + "methods": [ + "baseline", + "optimized", + "shuffle32" + ], + "oracle": "all-element FP64 dense, rounded to output dtype", + "fp16_tolerance_strict": 0.01, + "bf16_tolerance_strict": 0.05, + "warmup_not_counted": true, + "dtypes": { + "fp16": { + "cases": 752, + "elements": 4891672, + "exact_baseline_optimized_elements": 4891672, + "shuffle32_cases": 752, + "exact_baseline_shuffle32_elements": 4891672, + "shuffle32_grid_stride_cases": 2, + "shuffle32_grid_stride_elements": 1572843, + "unsupported_shuffle32_checks": 0, + "max_abs_error_rounded_fp64": 0.00390625, + "max_abs_error_unrounded_fp64": 0.015620231628418, + "api_contract_checks": 96 + }, + "bf16": { + "cases": 752, + "elements": 4891672, + "exact_baseline_optimized_elements": 4891672, + "shuffle32_cases": 752, + "exact_baseline_shuffle32_elements": 4891672, + "shuffle32_grid_stride_cases": 2, + "shuffle32_grid_stride_elements": 1572843, + "unsupported_shuffle32_checks": 0, + "max_abs_error_rounded_fp64": 3.814697265625e-06, + "max_abs_error_unrounded_fp64": 0.124984741210938, + "api_contract_checks": 96 + } + } + }, + "finished_utc": "2026-09-15T16:43:15Z", + "artifacts": { + "validation.log": { + "bytes": 2449, + "sha256": "70bd5a3305631837b480b31abea36dc67277bb48c89ccc73090dd4062357153b" + }, + "invalid_10.log": { + "bytes": 41, + "sha256": "76dd633500e539227532274081e41835e98a9cb82d73ba955f92c00ee1606cd0" + }, + "invalid_00.log": { + "bytes": 55, + "sha256": "57a02a5ddbdd1c62c445fa1feac13e27585b3837bd9d9ed0d3cd869a33e3c26e" + }, + "validate_and_benchmark": { + "bytes": 1010176, + "sha256": "1155d721fe4e48c51bcde8c6fa0cf1f962be313e742ea102962c0a3353a91cd8" + }, + "invalid_08.log": { + "bytes": 48, + "sha256": "439fd2b48f724414272ea4a69fae98cf1809e904afe56e2e8edd677e50993d56" + }, + "invalid_11.log": { + "bytes": 44, + "sha256": "be700711fd27b9719a4f6d490ef528894b371271344313d04298e25d9879ba54" + }, + "probe.log": { + "bytes": 293, + "sha256": "185e3405158ba1df91d2b5337045438c522c08d26b06ade0e0226ad075c6a4a1" + }, + "invalid_04.log": { + "bytes": 50, + "sha256": "5d1c70e05b6561f86a8c8c5bec7d168cbfc043fdfb9e1e7af32b5a87c1b1ff90" + }, + "probe_build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "invalid_03.log": { + "bytes": 41, + "sha256": "699f1f70340c77149d3bae4063f00207638e87be51bb3bb02bb2a99a650e68a1" + }, + "invalid_12.log": { + "bytes": 58, + "sha256": "eaddd4706d0da47a6fccbd2f81b6ace09f4f708d23cfc9b844f35d94f39d6241" + }, + "invalid_06.log": { + "bytes": 54, + "sha256": "49316fe0d2d4c9660cf5218d6ee6bc3708e737c38cfc3b53775f0636dc194997" + }, + "build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "invalid_13.log": { + "bytes": 45, + "sha256": "2116fe98f2998004c6367f99717fa82db3c3f77374b5c86cbeab03ca8922256e" + }, + "invalid_02.log": { + "bytes": 55, + "sha256": "fe9b4ea505976f30aab4a65234e33296fa2f61a7068429b2611ff99644178709" + }, + "validation.json": { + "bytes": 1046, + "sha256": "f7c7af273bc74c40d602227ef0014eb23ce07b07380bbeb893622f294cbee19e" + }, + "invalid_09.log": { + "bytes": 24, + "sha256": "4b8c91252c32d2f7cbcebc21561c29defe89809c97c23be7ab6a3577d4b74a3d" + }, + "invalid_07.log": { + "bytes": 52, + "sha256": "6d69299a2872ffc0a140db9fc171769b85e3f61111a5e73399436d32cb0fc27b" + }, + "probe": { + "bytes": 25872, + "sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0" + }, + "invalid_01.log": { + "bytes": 54, + "sha256": "db18b9f8d2040efced14ec6027b078290f358387206be6945f703d8bd4b5d7b7" + }, + "invalid_05.log": { + "bytes": 52, + "sha256": "3bac927d06f4ce7258f9c037afb2b3a5dfdc51f7504c16e4b1dae94bcc91cf1b" + } + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/full-validation.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/full-validation.json new file mode 100644 index 00000000..c977799e --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/full-validation.json @@ -0,0 +1 @@ +{"status":"PASS","platform":"moore","warp_size":128,"full_matrix":true,"shuffle32_enabled":true,"methods":["baseline","optimized","shuffle32"],"oracle":"all-element FP64 dense, rounded to output dtype","fp16_tolerance_strict":0.01,"bf16_tolerance_strict":0.05,"warmup_not_counted":true,"dtypes":{"fp16":{"cases":752,"elements":4891672,"exact_baseline_optimized_elements":4891672,"shuffle32_cases":752,"exact_baseline_shuffle32_elements":4891672,"shuffle32_grid_stride_cases":2,"shuffle32_grid_stride_elements":1572843,"unsupported_shuffle32_checks":0,"max_abs_error_rounded_fp64":0.00390625,"max_abs_error_unrounded_fp64":0.015620231628418,"api_contract_checks":96},"bf16":{"cases":752,"elements":4891672,"exact_baseline_optimized_elements":4891672,"shuffle32_cases":752,"exact_baseline_shuffle32_elements":4891672,"shuffle32_grid_stride_cases":2,"shuffle32_grid_stride_elements":1572843,"unsupported_shuffle32_checks":0,"max_abs_error_rounded_fp64":3.814697265625e-06,"max_abs_error_unrounded_fp64":0.124984741210938,"api_contract_checks":96}}} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/optimization.json b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/optimization.json new file mode 100644 index 00000000..32fa1a0e --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/optimization.json @@ -0,0 +1,442 @@ +{ + "status": "PASS", + "started_utc": "2026-09-15T16:42:52Z", + "scope": "Exact INT4 integer comparison, with original RNE-division controls", + "stages": [ + { + "name": "environment", + "command": [ + "mthreads-gmi" + ], + "returncode": 0, + "seconds": 0.058686012984253466, + "sha256": "15f9bf751e13a4453f961a3255504c3a6b77836575149fda1101e5784c811cf1" + }, + { + "name": "build-quant-gpu", + "command": [ + "/usr/local/musa/bin/mcc", + "-std=c++17", + "-O2", + "-fno-fast-math", + "-ffp-contract=off", + "--offload-arch=mp_22", + "-I/data/moore-opt-20260916", + "/data/moore-opt-20260916/check_quant_gpu.mu", + "-L/usr/local/musa/lib", + "-lmusart", + "-o", + "/data/moore-opt-20260916/runs/quant_gpu" + ], + "returncode": 0, + "seconds": 1.3358054279815406, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "name": "quant-gpu", + "command": [ + "/data/moore-opt-20260916/runs/quant_gpu", + "/data/moore-opt-20260916/quant_fixture.bin" + ], + "returncode": 0, + "seconds": 0.10466984799131751, + "sha256": "10f4b76d5a433cce1626a0d3146b354a47550363ec36688d7e990e61e61531f4" + }, + { + "name": "full", + "command": [ + "python3", + "/data/moore-opt-20260916/project/platforms/moore/run_platform.py", + "--shuffle32", + "--no-benchmark", + "--output", + "/data/moore-opt-20260916/runs/full" + ], + "returncode": 0, + "seconds": 21.211178090015892, + "sha256": "963a22694e02bfe8cfdaf3867280ed2c9832031189a659119a2c04414b761d67" + }, + { + "name": "disabled", + "command": [ + "python3", + "/data/moore-opt-20260916/project/platforms/moore/run_platform.py", + "--quick", + "--no-benchmark", + "--output", + "/data/moore-opt-20260916/runs/disabled" + ], + "returncode": 0, + "seconds": 9.177700225962326, + "sha256": "30ae1d93bc3ace9faf10c48545f942020a009a3853aca7544085d3b0fc3670f1" + }, + { + "name": "build-paired", + "command": [ + "/usr/local/musa/bin/mcc", + "-std=c++17", + "-O2", + "-fno-fast-math", + "-ffp-contract=off", + "--offload-arch=mp_22", + "-DHADAMARD_MOORE_SHUFFLE32", + "-I/data/moore-opt-20260916/project/platforms/moore", + "-I/data/moore-opt-20260916/control", + "-I/data/moore-opt-20260916/project/include", + "/data/moore-opt-20260916/project/platforms/moore/hadamard_api.mu", + "/data/moore-opt-20260916/control/control_api.mu", + "/data/moore-opt-20260916/paired_benchmark.mu", + "-L/usr/local/musa/lib", + "-lmusart", + "-o", + "/data/moore-opt-20260916/runs/paired_benchmark" + ], + "returncode": 0, + "seconds": 30.474890885001514, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "name": "paired-pilot", + "command": [ + "/data/moore-opt-20260916/runs/paired_benchmark", + "--benchmark", + "--batch", + "1", + "--seq", + "17", + "--heads", + "1", + "--dim", + "256", + "--groups", + "3", + "--repeats", + "20", + "--csv", + "/data/moore-opt-20260916/runs/paired-pilot.csv" + ], + "returncode": 0, + "seconds": 0.2844997219508514, + "sha256": "f8f6a0967bc953d16a27b92e55009dffee228a357fe67b906f7a14aace25ff81" + }, + { + "name": "paired-round1", + "command": [ + "/data/moore-opt-20260916/runs/paired_benchmark", + "--benchmark", + "--groups", + "5", + "--repeats", + "100", + "--csv", + "/data/moore-opt-20260916/runs/paired-round1.csv" + ], + "returncode": 0, + "seconds": 69.40715015301248, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + { + "name": "paired-round2", + "command": [ + "/data/moore-opt-20260916/runs/paired_benchmark", + "--benchmark", + "--groups", + "5", + "--repeats", + "100", + "--csv", + "/data/moore-opt-20260916/runs/paired-round2.csv" + ], + "returncode": 0, + "seconds": 69.41549559001578, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + { + "name": "paired-round3", + "command": [ + "/data/moore-opt-20260916/runs/paired_benchmark", + "--benchmark", + "--groups", + "5", + "--repeats", + "100", + "--csv", + "/data/moore-opt-20260916/runs/paired-round3.csv" + ], + "returncode": 0, + "seconds": 69.43805092398543, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + } + ], + "input_sha256": { + "exact_int4.hpp": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a", + "cpu-equivalence-linux.log": "1b2d4104c5d4bc3da97a6c25f5374474c5d849945d0695091c25448cb3d8a61a", + "cpu-equivalence.json": "1b2d4104c5d4bc3da97a6c25f5374474c5d849945d0695091c25448cb3d8a61a", + "check_quant_gpu.mu": "e51771809fb54db5cdb9e63c142f5266f2105ce0777d24577e428941f89f3b5d", + "METHOD.md": "b2e63c5b9b7de9328c84f5e8c6740c5b8d21fa0b4bae36780d5811884c639c79", + "paired_benchmark.mu": "138e6dd6ec9f9fd959162ae5edc0d27727e355ba5354cd1bd70bf702cfc69043", + "check_quant_cpu.py": "e1eba6f8b7cd0162b566889b5bc8ee66feb78c62b8d7b756b7ade6f7ebe84f43", + "check_quant_cpu.cpp": "8d4c6702c36a81f1cdef27df8e4b6bb413b92585d5169f21053d97ac1a450d1a", + "libcheck_quant.dylib": "560b68ba97d02f0076eba1f3b56a40232bf49fe6c76513a3eb6adab7ac64b805", + "run_optimization.py": "db94175d570dcf2214799d5e9a5a1983f3b381fd123d8c12b90a8d71faffec0a", + "quant_fixture.bin": "b404de04128361ed75347666113cfe2545f7c6e52aa828b070c1812c967ce230", + "project/platforms/moore/hadamard_api.mu": "f9c7fb325cbdc19e57e5c02ce58ec973197b75039fff38eb4b1fbac4daec4a80", + "project/platforms/moore/exact_int4.hpp": "784078d707dc6834ada435842a1a721b28acd953ecdb78f32e70b94ec6805f7a", + "project/platforms/moore/probe.mu": "2e777447c5153b55a7eee8bd7729170517637c4e04cb3cdbe89c6c51ec58bccd", + "project/platforms/moore/hadamard_api.h": "cba4d5c187defec8f70083a02ba252da8d73f5fba5b1b7692efb91dd1a0c4d93", + "project/platforms/moore/validate_and_benchmark.mu": "2a23a7938bd3140742b3c45f6b85db08d539bf03d4cb44ee9a48be41a268459e", + "project/platforms/moore/Makefile": "ccd363d6e1152d1fb3e47f856ecf05bbd431519329effb9f46924cc434e4917c", + "project/platforms/moore/run_platform.py": "d92b236ca4bcaff910921e2d0ef21f452968bb66f1bd98a1a157c0a7c8a88773", + "project/include/reference.hpp": "1ff2c70cc8849020bd413a1d174718683bd89d5b6d8f25ee0a01d1c9b8876888", + "control/control_api.mu": "d3ebe8f53f076a66db0cdc5d8cca5092fa821c729336949c3e2913d45e5ad56c", + "control/control_api.h": "b40c05f707e806381b3b7440537a435fdb4137cf27b78d403b336b32bf722014" + }, + "control_normalization_verified": true, + "finished_utc": "2026-09-15T16:47:23Z", + "artifacts": { + "paired-pilot.csv": { + "bytes": 10410, + "sha256": "420ea1f7b029e44c4564b2f9313df3c50bee595b0a01bcf60dd2e39f712fd6a6" + }, + "paired-round2.log": { + "bytes": 1455, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + "build-quant-gpu.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "paired-pilot.log": { + "bytes": 203, + "sha256": "f8f6a0967bc953d16a27b92e55009dffee228a357fe67b906f7a14aace25ff81" + }, + "paired-round3.log": { + "bytes": 1455, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + "paired-round1.csv": { + "bytes": 265111, + "sha256": "90a10dcfe163bb44425ada09f28e84c6a5cf0159872728b2b0a2975ceadd0cdb" + }, + "quant_gpu": { + "bytes": 34752, + "sha256": "77178f5e0a0a7cd41be4a2c25de63a578e23b48a1d9a3a1162482ed97994e3fb" + }, + "build-paired.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "paired-round2.csv": { + "bytes": 265090, + "sha256": "a90fc97aee1ede505c0c83b86d5369bba8931ead53149a3a4e5c36a506dbb363" + }, + "environment.log": { + "bytes": 1180, + "sha256": "15f9bf751e13a4453f961a3255504c3a6b77836575149fda1101e5784c811cf1" + }, + "paired-round1.log": { + "bytes": 1455, + "sha256": "cd2e2073bf6985304eb8ccb19fbd7e3799d62d3937d1b807804424ad5ed7f6c8" + }, + "quant-gpu.log": { + "bytes": 112, + "sha256": "10f4b76d5a433cce1626a0d3146b354a47550363ec36688d7e990e61e61531f4" + }, + "paired_benchmark": { + "bytes": 2720056, + "sha256": "151500968b461a024b62de40a0c80adf6d4d42f9b8b566335c170494cbb2e38f" + }, + "disabled.log": { + "bytes": 847, + "sha256": "30ae1d93bc3ace9faf10c48545f942020a009a3853aca7544085d3b0fc3670f1" + }, + "full.log": { + "bytes": 844, + "sha256": "963a22694e02bfe8cfdaf3867280ed2c9832031189a659119a2c04414b761d67" + }, + "paired-round3.csv": { + "bytes": 265132, + "sha256": "a9846fc8bb1d8efba317b3aba1929a5cf5102335833c2e2799475653a6eee033" + }, + "device-after.log": { + "bytes": 1180, + "sha256": "570261d2d4020e47a700b87f3cd10979c85632ee5e7f851bf4ec197b7c5f4e65" + }, + "full/run_summary.json": { + "bytes": 18467, + "sha256": "f6a50f6e1defee88fa752b69bf63544850c20d53e5bacb9e42152a94501608e5" + }, + "full/validation.log": { + "bytes": 2449, + "sha256": "70bd5a3305631837b480b31abea36dc67277bb48c89ccc73090dd4062357153b" + }, + "full/invalid_10.log": { + "bytes": 41, + "sha256": "76dd633500e539227532274081e41835e98a9cb82d73ba955f92c00ee1606cd0" + }, + "full/invalid_00.log": { + "bytes": 55, + "sha256": "57a02a5ddbdd1c62c445fa1feac13e27585b3837bd9d9ed0d3cd869a33e3c26e" + }, + "full/validate_and_benchmark": { + "bytes": 1010176, + "sha256": "1155d721fe4e48c51bcde8c6fa0cf1f962be313e742ea102962c0a3353a91cd8" + }, + "full/invalid_08.log": { + "bytes": 48, + "sha256": "439fd2b48f724414272ea4a69fae98cf1809e904afe56e2e8edd677e50993d56" + }, + "full/invalid_11.log": { + "bytes": 44, + "sha256": "be700711fd27b9719a4f6d490ef528894b371271344313d04298e25d9879ba54" + }, + "full/probe.log": { + "bytes": 293, + "sha256": "185e3405158ba1df91d2b5337045438c522c08d26b06ade0e0226ad075c6a4a1" + }, + "full/invalid_04.log": { + "bytes": 50, + "sha256": "5d1c70e05b6561f86a8c8c5bec7d168cbfc043fdfb9e1e7af32b5a87c1b1ff90" + }, + "full/probe_build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "full/invalid_03.log": { + "bytes": 41, + "sha256": "699f1f70340c77149d3bae4063f00207638e87be51bb3bb02bb2a99a650e68a1" + }, + "full/invalid_12.log": { + "bytes": 58, + "sha256": "eaddd4706d0da47a6fccbd2f81b6ace09f4f708d23cfc9b844f35d94f39d6241" + }, + "full/invalid_06.log": { + "bytes": 54, + "sha256": "49316fe0d2d4c9660cf5218d6ee6bc3708e737c38cfc3b53775f0636dc194997" + }, + "full/build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "full/invalid_13.log": { + "bytes": 45, + "sha256": "2116fe98f2998004c6367f99717fa82db3c3f77374b5c86cbeab03ca8922256e" + }, + "full/invalid_02.log": { + "bytes": 55, + "sha256": "fe9b4ea505976f30aab4a65234e33296fa2f61a7068429b2611ff99644178709" + }, + "full/validation.json": { + "bytes": 1046, + "sha256": "f7c7af273bc74c40d602227ef0014eb23ce07b07380bbeb893622f294cbee19e" + }, + "full/invalid_09.log": { + "bytes": 24, + "sha256": "4b8c91252c32d2f7cbcebc21561c29defe89809c97c23be7ab6a3577d4b74a3d" + }, + "full/invalid_07.log": { + "bytes": 52, + "sha256": "6d69299a2872ffc0a140db9fc171769b85e3f61111a5e73399436d32cb0fc27b" + }, + "full/probe": { + "bytes": 25872, + "sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0" + }, + "full/invalid_01.log": { + "bytes": 54, + "sha256": "db18b9f8d2040efced14ec6027b078290f358387206be6945f703d8bd4b5d7b7" + }, + "full/invalid_05.log": { + "bytes": 52, + "sha256": "3bac927d06f4ce7258f9c037afb2b3a5dfdc51f7504c16e4b1dae94bcc91cf1b" + }, + "disabled/run_summary.json": { + "bytes": 18442, + "sha256": "a505ad08186225ad344b0ce825384436daf1a0e71a629aee111f2d8e25ac6cc5" + }, + "disabled/validation.log": { + "bytes": 1512, + "sha256": "8b096594841bfc815468cf38d8385acd4316f1d13201217a987f287fb7c8d86c" + }, + "disabled/invalid_10.log": { + "bytes": 41, + "sha256": "76dd633500e539227532274081e41835e98a9cb82d73ba955f92c00ee1606cd0" + }, + "disabled/invalid_00.log": { + "bytes": 55, + "sha256": "57a02a5ddbdd1c62c445fa1feac13e27585b3837bd9d9ed0d3cd869a33e3c26e" + }, + "disabled/validate_and_benchmark": { + "bytes": 331272, + "sha256": "593faf17d6b34c45ff826b103ceb2957d71cd9befdaa51ecb0cfa8312c6e141a" + }, + "disabled/invalid_08.log": { + "bytes": 48, + "sha256": "439fd2b48f724414272ea4a69fae98cf1809e904afe56e2e8edd677e50993d56" + }, + "disabled/invalid_11.log": { + "bytes": 44, + "sha256": "be700711fd27b9719a4f6d490ef528894b371271344313d04298e25d9879ba54" + }, + "disabled/probe.log": { + "bytes": 293, + "sha256": "185e3405158ba1df91d2b5337045438c522c08d26b06ade0e0226ad075c6a4a1" + }, + "disabled/invalid_04.log": { + "bytes": 50, + "sha256": "5d1c70e05b6561f86a8c8c5bec7d168cbfc043fdfb9e1e7af32b5a87c1b1ff90" + }, + "disabled/probe_build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "disabled/invalid_03.log": { + "bytes": 41, + "sha256": "699f1f70340c77149d3bae4063f00207638e87be51bb3bb02bb2a99a650e68a1" + }, + "disabled/invalid_12.log": { + "bytes": 58, + "sha256": "eaddd4706d0da47a6fccbd2f81b6ace09f4f708d23cfc9b844f35d94f39d6241" + }, + "disabled/invalid_06.log": { + "bytes": 54, + "sha256": "49316fe0d2d4c9660cf5218d6ee6bc3708e737c38cfc3b53775f0636dc194997" + }, + "disabled/build.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "disabled/invalid_13.log": { + "bytes": 45, + "sha256": "2116fe98f2998004c6367f99717fa82db3c3f77374b5c86cbeab03ca8922256e" + }, + "disabled/invalid_02.log": { + "bytes": 55, + "sha256": "fe9b4ea505976f30aab4a65234e33296fa2f61a7068429b2611ff99644178709" + }, + "disabled/validation.json": { + "bytes": 970, + "sha256": "9e3cf608dc9880c4922d9ef8978b7c16f9e0f75b3051c3d2904c73e19179e368" + }, + "disabled/invalid_09.log": { + "bytes": 24, + "sha256": "4b8c91252c32d2f7cbcebc21561c29defe89809c97c23be7ab6a3577d4b74a3d" + }, + "disabled/invalid_07.log": { + "bytes": 52, + "sha256": "6d69299a2872ffc0a140db9fc171769b85e3f61111a5e73399436d32cb0fc27b" + }, + "disabled/probe": { + "bytes": 25872, + "sha256": "694c0d6f3e751e63976999ee0cd42c655835eb158c52cbfbef618ea6c20e1de0" + }, + "disabled/invalid_01.log": { + "bytes": 54, + "sha256": "db18b9f8d2040efced14ec6027b078290f358387206be6945f703d8bd4b5d7b7" + }, + "disabled/invalid_05.log": { + "bytes": 52, + "sha256": "3bac927d06f4ce7258f9c037afb2b3a5dfdc51f7504c16e4b1dae94bcc91cf1b" + } + } +} diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round1.csv b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round1.csv new file mode 100644 index 00000000..a3273235 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round1.csv @@ -0,0 +1,2251 @@ +dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale,kernel_ms +fp16,1,1,1,64,1,baseline_transform,0,0,100,8.9780575037,256,0.0285139630588,128,2909,true,1,0.0089780575037 +fp16,1,1,1,64,1,control_optimized_transform,0,1,100,9.23520028591,256,0.0277200268618,128,2909,true,1,0.00923520028591 +fp16,1,1,1,64,1,optimized_transform,0,2,100,9.32022869587,256,0.0274671371651,128,2909,true,1,0.00932022869587 +fp16,1,1,1,64,1,control_shuffle32_transform,0,3,100,7.87908554077,256,0.0324910801736,128,2909,true,1,0.00787908554077 +fp16,1,1,1,64,1,shuffle32_transform,0,4,100,7.89874315262,256,0.0324102195822,128,2909,true,1,0.00789874315262 +fp16,1,1,1,64,1,baseline_split,0,5,100,69.9092578888,420,0.00600778799094,128,2909,true,1,0.0699092578888 +fp16,1,1,1,64,1,control_optimized_split,0,6,100,64.604344368,420,0.00650111078611,128,2909,true,1,0.064604344368 +fp16,1,1,1,64,1,optimized_split,0,7,100,33.7078857422,420,0.012459992395,128,2909,true,1,0.0337078857422 +fp16,1,1,1,64,1,control_shuffle32_split,0,8,100,94.5702838898,420,0.00444114136836,128,2909,true,1,0.0945702838898 +fp16,1,1,1,64,1,shuffle32_split,0,9,100,34.929599762,420,0.0120241858728,128,2909,true,1,0.034929599762 +fp16,1,1,1,64,1,baseline_fused,0,10,100,65.4630851746,164,0.00250522870352,128,2909,true,1,0.0654630851746 +fp16,1,1,1,64,1,control_optimized_fused,0,11,100,58.6957740784,164,0.0027940682711,128,2909,true,1,0.0586957740784 +fp16,1,1,1,64,1,optimized_fused,0,12,100,28.5122299194,164,0.0057519177021,128,2909,true,1,0.0285122299194 +fp16,1,1,1,64,1,control_shuffle32_fused,0,13,100,89.2457199097,164,0.00183762313942,128,2909,true,1,0.0892457199097 +fp16,1,1,1,64,1,shuffle32_fused,0,14,100,29.7817134857,164,0.00550673486529,128,2909,true,1,0.0297817134857 +fp16,1,1,1,64,1,control_optimized_transform,1,0,100,9.18697178364,256,0.0278655476504,128,2909,true,1,0.00918697178364 +fp16,1,1,1,64,1,optimized_transform,1,1,100,9.32571411133,256,0.0274509809055,128,2909,true,1,0.00932571411133 +fp16,1,1,1,64,1,control_shuffle32_transform,1,2,100,8.09074282646,256,0.0316410996482,128,2909,true,1,0.00809074282646 +fp16,1,1,1,64,1,shuffle32_transform,1,3,100,8.07999968529,256,0.0316831695509,128,2909,true,1,0.00807999968529 +fp16,1,1,1,64,1,baseline_split,1,4,100,69.8902893066,420,0.00600941853535,128,2909,true,1,0.0698902893066 +fp16,1,1,1,64,1,control_optimized_split,1,5,100,64.1247987747,420,0.00654972815549,128,2909,true,1,0.0641247987747 +fp16,1,1,1,64,1,optimized_split,1,6,100,33.7154269218,420,0.01245720545,128,2909,true,1,0.0337154269218 +fp16,1,1,1,64,1,control_shuffle32_split,1,7,100,94.5785140991,420,0.00444075490084,128,2909,true,1,0.0945785140991 +fp16,1,1,1,64,1,shuffle32_split,1,8,100,34.8450279236,420,0.0120533695918,128,2909,true,1,0.0348450279236 +fp16,1,1,1,64,1,baseline_fused,1,9,100,65.4601144791,164,0.00250534239521,128,2909,true,1,0.0654601144791 +fp16,1,1,1,64,1,control_optimized_fused,1,10,100,58.6639976501,164,0.00279558172933,128,2909,true,1,0.0586639976501 +fp16,1,1,1,64,1,optimized_fused,1,11,100,28.569829464,164,0.00574032127867,128,2909,true,1,0.028569829464 +fp16,1,1,1,64,1,control_shuffle32_fused,1,12,100,89.2482280731,164,0.00183757149627,128,2909,true,1,0.0892482280731 +fp16,1,1,1,64,1,shuffle32_fused,1,13,100,29.7309708595,164,0.00551613335383,128,2909,true,1,0.0297309708595 +fp16,1,1,1,64,1,baseline_transform,1,14,100,9.02742862701,256,0.0283580198279,128,2909,true,1,0.00902742862701 +fp16,1,1,1,64,1,optimized_transform,2,0,100,9.2699432373,256,0.027616134581,128,2909,true,1,0.0092699432373 +fp16,1,1,1,64,1,control_shuffle32_transform,2,1,100,7.95588612556,256,0.0321774339099,128,2909,true,1,0.00795588612556 +fp16,1,1,1,64,1,shuffle32_transform,2,2,100,7.926171422,256,0.0322980650266,128,2909,true,1,0.007926171422 +fp16,1,1,1,64,1,baseline_split,2,3,100,70.0034284592,420,0.00599970614646,128,2909,true,1,0.0700034284592 +fp16,1,1,1,64,1,control_optimized_split,2,4,100,64.1126823425,420,0.00655096596577,128,2909,true,1,0.0641126823425 +fp16,1,1,1,64,1,optimized_split,2,5,100,33.6701726913,420,0.0124739484959,128,2909,true,1,0.0336701726913 +fp16,1,1,1,64,1,control_shuffle32_split,2,6,100,93.9197731018,420,0.00447190177456,128,2909,true,1,0.0939197731018 +fp16,1,1,1,64,1,shuffle32_split,2,7,100,34.8029708862,420,0.0120679352741,128,2909,true,1,0.0348029708862 +fp16,1,1,1,64,1,baseline_fused,2,8,100,65.4578304291,164,0.00250542981527,128,2909,true,1,0.0654578304291 +fp16,1,1,1,64,1,control_optimized_fused,2,9,100,58.6738300323,164,0.00279511325423,128,2909,true,1,0.0586738300323 +fp16,1,1,1,64,1,optimized_fused,2,10,100,28.5449123383,164,0.00574533205976,128,2909,true,1,0.0285449123383 +fp16,1,1,1,64,1,control_shuffle32_fused,2,11,100,89.251203537,164,0.00183751023516,128,2909,true,1,0.089251203537 +fp16,1,1,1,64,1,shuffle32_fused,2,12,100,29.7814869881,164,0.00550677674576,128,2909,true,1,0.0297814869881 +fp16,1,1,1,64,1,baseline_transform,2,13,100,9.13919985294,256,0.0280112049325,128,2909,true,1,0.00913919985294 +fp16,1,1,1,64,1,control_optimized_transform,2,14,100,9.330971241,256,0.0274355148449,128,2909,true,1,0.009330971241 +fp16,1,1,1,64,1,control_shuffle32_transform,3,0,100,8.00228536129,256,0.0319908611655,128,2909,true,1,0.00800228536129 +fp16,1,1,1,64,1,shuffle32_transform,3,1,100,8.05279970169,256,0.0317901859581,128,2909,true,1,0.00805279970169 +fp16,1,1,1,64,1,baseline_split,3,2,100,69.8838853836,420,0.00600996921815,128,2909,true,1,0.0698838853836 +fp16,1,1,1,64,1,control_optimized_split,3,3,100,64.0834283829,420,0.00655395646891,128,2909,true,1,0.0640834283829 +fp16,1,1,1,64,1,optimized_split,3,4,100,33.914513588,420,0.0123840785424,128,2909,true,1,0.033914513588 +fp16,1,1,1,64,1,control_shuffle32_split,3,5,100,93.91954422,420,0.00447191267258,128,2909,true,1,0.09391954422 +fp16,1,1,1,64,1,shuffle32_split,3,6,100,34.8639988899,420,0.0120468108471,128,2909,true,1,0.0348639988899 +fp16,1,1,1,64,1,baseline_fused,3,7,100,65.4493713379,164,0.00250575363289,128,2909,true,1,0.0654493713379 +fp16,1,1,1,64,1,control_optimized_fused,3,8,100,58.6681175232,164,0.00279538541415,128,2909,true,1,0.0586681175232 +fp16,1,1,1,64,1,optimized_fused,3,9,100,28.5229706764,164,0.0057497517303,128,2909,true,1,0.0285229706764 +fp16,1,1,1,64,1,control_shuffle32_fused,3,10,100,89.245262146,164,0.0018376325651,128,2909,true,1,0.089245262146 +fp16,1,1,1,64,1,shuffle32_fused,3,11,100,29.7506284714,164,0.00551248859021,128,2909,true,1,0.0297506284714 +fp16,1,1,1,64,1,baseline_transform,3,12,100,9.1325712204,256,0.0280315361164,128,2909,true,1,0.0091325712204 +fp16,1,1,1,64,1,control_optimized_transform,3,13,100,9.16502833366,256,0.0279322649838,128,2909,true,1,0.00916502833366 +fp16,1,1,1,64,1,optimized_transform,3,14,100,9.31817173958,256,0.0274732004469,128,2909,true,1,0.00931817173958 +fp16,1,1,1,64,1,shuffle32_transform,4,0,100,8.08777153492,256,0.031652723979,128,2909,true,1,0.00808777153492 +fp16,1,1,1,64,1,baseline_split,4,1,100,69.9517726898,420,0.00600413661942,128,2909,true,1,0.0699517726898 +fp16,1,1,1,64,1,control_optimized_split,4,2,100,64.090514183,420,0.0065532318683,128,2909,true,1,0.064090514183 +fp16,1,1,1,64,1,optimized_split,4,3,100,33.716571331,420,0.0124567826271,128,2909,true,1,0.033716571331 +fp16,1,1,1,64,1,control_shuffle32_split,4,4,100,93.9275455475,420,0.00447153172748,128,2909,true,1,0.0939275455475 +fp16,1,1,1,64,1,shuffle32_split,4,5,100,34.8130297661,420,0.0120644483638,128,2909,true,1,0.0348130297661 +fp16,1,1,1,64,1,baseline_fused,4,6,100,65.4571390152,164,0.00250545627975,128,2909,true,1,0.0654571390152 +fp16,1,1,1,64,1,control_optimized_fused,4,7,100,58.6761140823,164,0.00279500445053,128,2909,true,1,0.0586761140823 +fp16,1,1,1,64,1,optimized_fused,4,8,100,28.5378289223,164,0.00574675811698,128,2909,true,1,0.0285378289223 +fp16,1,1,1,64,1,control_shuffle32_fused,4,9,100,89.2470836639,164,0.00183759505932,128,2909,true,1,0.0892470836639 +fp16,1,1,1,64,1,shuffle32_fused,4,10,100,29.7325706482,164,0.00551583655314,128,2909,true,1,0.0297325706482 +fp16,1,1,1,64,1,baseline_transform,4,11,100,9.01165723801,256,0.0284076494743,128,2909,true,1,0.00901165723801 +fp16,1,1,1,64,1,control_optimized_transform,4,12,100,9.33668553829,256,0.0274187235877,128,2909,true,1,0.00933668553829 +fp16,1,1,1,64,1,optimized_transform,4,13,100,9.26605701447,256,0.0276277169027,128,2909,true,1,0.00926605701447 +fp16,1,1,1,64,1,control_shuffle32_transform,4,14,100,7.99748599529,256,0.0320100591799,128,2909,true,1,0.00799748599529 +fp16,1,17,1,64,17,baseline_transform,0,0,100,9.21302855015,4352,0.472374526608,2176,2909,true,1,0.00921302855015 +fp16,1,17,1,64,17,control_optimized_transform,0,1,100,9.49074268341,4352,0.45855210126,2176,2909,true,1,0.00949074268341 +fp16,1,17,1,64,17,optimized_transform,0,2,100,9.4047999382,4352,0.462742432438,2176,2909,true,1,0.0094047999382 +fp16,1,17,1,64,17,control_shuffle32_transform,0,3,100,8.09599995613,4352,0.537549410027,2176,2909,true,1,0.00809599995613 +fp16,1,17,1,64,17,shuffle32_transform,0,4,100,8.04159998894,4352,0.541185834409,2176,2909,true,1,0.00804159998894 +fp16,1,17,1,64,17,baseline_split,0,5,100,70.324344635,7140,0.101529563298,2176,2909,true,1,0.070324344635 +fp16,1,17,1,64,17,control_optimized_split,0,6,100,65.0098276138,7140,0.109829548271,2176,2909,true,1,0.0650098276138 +fp16,1,17,1,64,17,optimized_split,0,7,100,34.292113781,7140,0.208211136986,2176,2909,true,1,0.034292113781 +fp16,1,17,1,64,17,control_shuffle32_split,0,8,100,94.4102859497,7140,0.0756273527633,2176,2909,true,1,0.0944102859497 +fp16,1,17,1,64,17,shuffle32_split,0,9,100,34.7929143906,7140,0.205214197346,2176,2909,true,1,0.0347929143906 +fp16,1,17,1,64,17,baseline_fused,0,10,100,65.7371425629,2788,0.0424113353776,2176,2909,true,1,0.0657371425629 +fp16,1,17,1,64,17,control_optimized_fused,0,11,100,58.912229538,2788,0.0473246390752,2176,2909,true,1,0.058912229538 +fp16,1,17,1,64,17,optimized_fused,0,12,100,29.0461730957,2788,0.09598510588,2176,2909,true,1,0.0290461730957 +fp16,1,17,1,64,17,control_shuffle32_fused,0,13,100,89.091424942,2788,0.0312936963553,2176,2909,true,1,0.089091424942 +fp16,1,17,1,64,17,shuffle32_fused,0,14,100,29.5547413826,2788,0.0943334256899,2176,2909,true,1,0.0295547413826 +fp16,1,17,1,64,17,control_optimized_transform,1,0,100,9.49919998646,4352,0.458143844345,2176,2909,true,1,0.00949919998646 +fp16,1,17,1,64,17,optimized_transform,1,1,100,9.48822855949,4352,0.458673605164,2176,2909,true,1,0.00948822855949 +fp16,1,17,1,64,17,control_shuffle32_transform,1,2,100,8.06605756283,4352,0.539544872585,2176,2909,true,1,0.00806605756283 +fp16,1,17,1,64,17,shuffle32_transform,1,3,100,8.03542852402,4352,0.541601482359,2176,2909,true,1,0.00803542852402 +fp16,1,17,1,64,17,baseline_split,1,4,100,70.4137134552,7140,0.101400702358,2176,2909,true,1,0.0704137134552 +fp16,1,17,1,64,17,control_optimized_split,1,5,100,64.3641138077,7140,0.11093138051,2176,2909,true,1,0.0643641138077 +fp16,1,17,1,64,17,optimized_split,1,6,100,34.3524575233,7140,0.207845391997,2176,2909,true,1,0.0343524575233 +fp16,1,17,1,64,17,control_shuffle32_split,1,7,100,94.4745159149,7140,0.0755759363343,2176,2909,true,1,0.0944745159149 +fp16,1,17,1,64,17,shuffle32_split,1,8,100,34.7794294357,7140,0.205293764614,2176,2909,true,1,0.0347794294357 +fp16,1,17,1,64,17,baseline_fused,1,9,100,65.7366847992,2788,0.042411630713,2176,2909,true,1,0.0657366847992 +fp16,1,17,1,64,17,control_optimized_fused,1,10,100,58.9238834381,2788,0.0473152792607,2176,2909,true,1,0.0589238834381 +fp16,1,17,1,64,17,optimized_fused,1,11,100,29.0633130074,2788,0.095928499249,2176,2909,true,1,0.0290633130074 +fp16,1,17,1,64,17,control_shuffle32_fused,1,12,100,89.0850257874,2788,0.0312959442438,2176,2909,true,1,0.0890850257874 +fp16,1,17,1,64,17,shuffle32_fused,1,13,100,29.6262860298,2788,0.0941056194892,2176,2909,true,1,0.0296262860298 +fp16,1,17,1,64,17,baseline_transform,1,14,100,9.36251401901,4352,0.464832414794,2176,2909,true,1,0.00936251401901 +fp16,1,17,1,64,17,optimized_transform,2,0,100,9.48937118053,4352,0.458618375992,2176,2909,true,1,0.00948937118053 +fp16,1,17,1,64,17,control_shuffle32_transform,2,1,100,8.03839981556,4352,0.541401286308,2176,2909,true,1,0.00803839981556 +fp16,1,17,1,64,17,shuffle32_transform,2,2,100,8.07565689087,4352,0.538903529312,2176,2909,true,1,0.00807565689087 +fp16,1,17,1,64,17,baseline_split,2,3,100,70.3327989578,7140,0.101517358982,2176,2909,true,1,0.0703327989578 +fp16,1,17,1,64,17,control_optimized_split,2,4,100,64.4587469101,7140,0.110768520058,2176,2909,true,1,0.0644587469101 +fp16,1,17,1,64,17,optimized_split,2,5,100,34.3266296387,7140,0.20800177807,2176,2909,true,1,0.0343266296387 +fp16,1,17,1,64,17,control_shuffle32_split,2,6,100,93.7750816345,7140,0.0761396297987,2176,2909,true,1,0.0937750816345 +fp16,1,17,1,64,17,shuffle32_split,2,7,100,34.7462844849,7140,0.205489597114,2176,2909,true,1,0.0347462844849 +fp16,1,17,1,64,17,baseline_fused,2,8,100,65.7510852814,2788,0.0424023419244,2176,2909,true,1,0.0657510852814 +fp16,1,17,1,64,17,control_optimized_fused,2,9,100,58.9225149155,2788,0.047316378196,2176,2909,true,1,0.0589225149155 +fp16,1,17,1,64,17,optimized_fused,2,10,100,29.0317726135,2788,0.0960327168828,2176,2909,true,1,0.0290317726135 +fp16,1,17,1,64,17,control_shuffle32_fused,2,11,100,89.0861701965,2788,0.0312955422132,2176,2909,true,1,0.0890861701965 +fp16,1,17,1,64,17,shuffle32_fused,2,12,100,29.5533704758,2788,0.0943378015812,2176,2909,true,1,0.0295533704758 +fp16,1,17,1,64,17,baseline_transform,2,13,100,9.41005706787,4352,0.46248391148,2176,2909,true,1,0.00941005706787 +fp16,1,17,1,64,17,control_optimized_transform,2,14,100,9.46308553219,4352,0.459892281983,2176,2909,true,1,0.00946308553219 +fp16,1,17,1,64,17,control_shuffle32_transform,3,0,100,8.09737145901,4352,0.537458361893,2176,2909,true,1,0.00809737145901 +fp16,1,17,1,64,17,shuffle32_transform,3,1,100,8.04137170315,4352,0.541201198086,2176,2909,true,1,0.00804137170315 +fp16,1,17,1,64,17,baseline_split,3,2,100,70.3570270538,7140,0.101482400536,2176,2909,true,1,0.0703570270538 +fp16,1,17,1,64,17,control_optimized_split,3,3,100,64.4676589966,7140,0.110753207285,2176,2909,true,1,0.0644676589966 +fp16,1,17,1,64,17,optimized_split,3,4,100,34.4628572464,7140,0.207179571588,2176,2909,true,1,0.0344628572464 +fp16,1,17,1,64,17,control_shuffle32_split,3,5,100,94.4027423859,7140,0.0756333960174,2176,2909,true,1,0.0944027423859 +fp16,1,17,1,64,17,shuffle32_split,3,6,100,34.8235440254,7140,0.205033697742,2176,2909,true,1,0.0348235440254 +fp16,1,17,1,64,17,baseline_fused,3,7,100,65.7407999039,2788,0.0424089759187,2176,2909,true,1,0.0657407999039 +fp16,1,17,1,64,17,control_optimized_fused,3,8,100,58.915886879,2788,0.0473217012879,2176,2909,true,1,0.058915886879 +fp16,1,17,1,64,17,optimized_fused,3,9,100,29.0452575684,2788,0.0959881313994,2176,2909,true,1,0.0290452575684 +fp16,1,17,1,64,17,control_shuffle32_fused,3,10,100,89.0884590149,2788,0.031294738183,2176,2909,true,1,0.0890884590149 +fp16,1,17,1,64,17,shuffle32_fused,3,11,100,29.6265149117,2788,0.0941048924693,2176,2909,true,1,0.0296265149117 +fp16,1,17,1,64,17,baseline_transform,3,12,100,9.28251445293,4352,0.468838483588,2176,2909,true,1,0.00928251445293 +fp16,1,17,1,64,17,control_optimized_transform,3,13,100,9.40525710583,4352,0.462719939607,2176,2909,true,1,0.00940525710583 +fp16,1,17,1,64,17,optimized_transform,3,14,100,9.41760003567,4352,0.462113487886,2176,2909,true,1,0.00941760003567 +fp16,1,17,1,64,17,shuffle32_transform,4,0,100,7.94422864914,4352,0.547819076239,2176,2909,true,1,0.00794422864914 +fp16,1,17,1,64,17,baseline_split,4,1,100,70.3538322449,7140,0.101487008912,2176,2909,true,1,0.0703538322449 +fp16,1,17,1,64,17,control_optimized_split,4,2,100,64.4937133789,7140,0.11070846484,2176,2909,true,1,0.0644937133789 +fp16,1,17,1,64,17,optimized_split,4,3,100,34.3449139595,7140,0.207891043443,2176,2909,true,1,0.0343449139595 +fp16,1,17,1,64,17,control_shuffle32_split,4,4,100,93.8646888733,7140,0.076066943658,2176,2909,true,1,0.0938646888733 +fp16,1,17,1,64,17,shuffle32_split,4,5,100,34.7780585289,7140,0.205301857033,2176,2909,true,1,0.0347780585289 +fp16,1,17,1,64,17,baseline_fused,4,6,100,65.7433128357,2788,0.0424073549042,2176,2909,true,1,0.0657433128357 +fp16,1,17,1,64,17,control_optimized_fused,4,7,100,58.9200019836,2788,0.0473183962345,2176,2909,true,1,0.0589200019836 +fp16,1,17,1,64,17,optimized_fused,4,8,100,29.0397715569,2788,0.0960062648751,2176,2909,true,1,0.0290397715569 +fp16,1,17,1,64,17,control_shuffle32_fused,4,9,100,89.0404510498,2788,0.0313116113758,2176,2909,true,1,0.0890404510498 +fp16,1,17,1,64,17,shuffle32_fused,4,10,100,29.6114301682,2788,0.0941528316656,2176,2909,true,1,0.0296114301682 +fp16,1,17,1,64,17,baseline_transform,4,11,100,9.29988563061,4352,0.467962744152,2176,2909,true,1,0.00929988563061 +fp16,1,17,1,64,17,control_optimized_transform,4,12,100,9.4534856081,4352,0.460359298191,2176,2909,true,1,0.0094534856081 +fp16,1,17,1,64,17,optimized_transform,4,13,100,9.39360022545,4352,0.463294146605,2176,2909,true,1,0.00939360022545 +fp16,1,17,1,64,17,control_shuffle32_transform,4,14,100,8.0891430378,4352,0.538005074167,2176,2909,true,1,0.0080891430378 +fp16,1,257,1,64,257,baseline_transform,0,0,100,9.55611407757,65792,6.88480688551,32896,2909,true,1,0.00955611407757 +fp16,1,257,1,64,257,control_optimized_transform,0,1,100,9.5647996664,65792,6.87855494047,32896,2909,true,1,0.0095647996664 +fp16,1,257,1,64,257,optimized_transform,0,2,100,9.55428600311,65792,6.88612419375,32896,2909,true,1,0.00955428600311 +fp16,1,257,1,64,257,control_shuffle32_transform,0,3,100,8.7899428606,65792,7.48491782522,32896,2909,true,1,0.0087899428606 +fp16,1,257,1,64,257,shuffle32_transform,0,4,100,8.79360020161,65792,7.48180477752,32896,2909,true,1,0.00879360020161 +fp16,1,257,1,64,257,baseline_split,0,5,100,71.0925674438,107940,1.51830217815,32896,2909,true,1,0.0710925674438 +fp16,1,257,1,64,257,control_optimized_split,0,6,100,64.4413709641,107940,1.675010919,32896,2909,true,1,0.0644413709641 +fp16,1,257,1,64,257,optimized_split,0,7,100,34.7497153282,107940,3.10621249643,32896,2909,true,1,0.0347497153282 +fp16,1,257,1,64,257,control_shuffle32_split,0,8,100,102.086400986,107940,1.05733965502,32896,2909,true,1,0.102086400986 +fp16,1,257,1,64,257,shuffle32_split,0,9,100,39.1581702232,107940,2.75651286525,32896,2909,true,1,0.0391581702232 +fp16,1,257,1,64,257,baseline_fused,0,10,100,65.99588871,42148,0.638645843307,32896,2909,true,1,0.06599588871 +fp16,1,257,1,64,257,control_optimized_fused,0,11,100,59.3414878845,42148,0.710261934821,32896,2909,true,1,0.0593414878845 +fp16,1,257,1,64,257,optimized_fused,0,12,100,29.3602275848,42148,1.43554745542,32896,2909,true,1,0.0293602275848 +fp16,1,257,1,64,257,control_shuffle32_fused,0,13,100,96.660118103,42148,0.436043332319,32896,2909,true,1,0.096660118103 +fp16,1,257,1,64,257,shuffle32_fused,0,14,100,33.2130289078,42148,1.26902006189,32896,2909,true,1,0.0332130289078 +fp16,1,257,1,64,257,control_optimized_transform,1,0,100,9.54171419144,65792,6.89519709771,32896,2909,true,1,0.00954171419144 +fp16,1,257,1,64,257,optimized_transform,1,1,100,9.69417154789,65792,6.78675838105,32896,2909,true,1,0.00969417154789 +fp16,1,257,1,64,257,control_shuffle32_transform,1,2,100,8.94468545914,65792,7.3554291317,32896,2909,true,1,0.00894468545914 +fp16,1,257,1,64,257,shuffle32_transform,1,3,100,8.86468589306,65792,7.42180837468,32896,2909,true,1,0.00886468589306 +fp16,1,257,1,64,257,baseline_split,1,4,100,71.1081123352,107940,1.51797026324,32896,2909,true,1,0.0711081123352 +fp16,1,257,1,64,257,control_optimized_split,1,5,100,64.5478868484,107940,1.67224684293,32896,2909,true,1,0.0645478868484 +fp16,1,257,1,64,257,optimized_split,1,6,100,34.748570919,107940,3.10631479641,32896,2909,true,1,0.034748570919 +fp16,1,257,1,64,257,control_shuffle32_split,1,7,100,102.688684464,107940,1.05113821025,32896,2909,true,1,0.102688684464 +fp16,1,257,1,64,257,shuffle32_split,1,8,100,39.129140377,107940,2.7585579177,32896,2909,true,1,0.039129140377 +fp16,1,257,1,64,257,baseline_fused,1,9,100,65.9865140915,42148,0.638736574894,32896,2909,true,1,0.0659865140915 +fp16,1,257,1,64,257,control_optimized_fused,1,10,100,59.3474292755,42148,0.710190829064,32896,2909,true,1,0.0593474292755 +fp16,1,257,1,64,257,optimized_fused,1,11,100,29.3901729584,42148,1.43408478949,32896,2909,true,1,0.0293901729584 +fp16,1,257,1,64,257,control_shuffle32_fused,1,12,100,96.7149734497,42148,0.435796014791,32896,2909,true,1,0.0967149734497 +fp16,1,257,1,64,257,shuffle32_fused,1,13,100,33.2569146156,42148,1.26734546747,32896,2909,true,1,0.0332569146156 +fp16,1,257,1,64,257,baseline_transform,1,14,100,9.64479982853,65792,6.82149978949,32896,2909,true,1,0.00964479982853 +fp16,1,257,1,64,257,optimized_transform,2,0,100,9.78399991989,65792,6.72444813355,32896,2909,true,1,0.00978399991989 +fp16,1,257,1,64,257,control_shuffle32_transform,2,1,100,8.95291447639,65792,7.34866843344,32896,2909,true,1,0.00895291447639 +fp16,1,257,1,64,257,shuffle32_transform,2,2,100,9.016456604,65792,7.2968797932,32896,2909,true,1,0.009016456604 +fp16,1,257,1,64,257,baseline_split,2,3,100,71.0742855072,107940,1.5186927203,32896,2909,true,1,0.0710742855072 +fp16,1,257,1,64,257,control_optimized_split,2,4,100,64.5545101166,107940,1.67207527104,32896,2909,true,1,0.0645545101166 +fp16,1,257,1,64,257,optimized_split,2,5,100,34.7757720947,107940,3.10388507568,32896,2909,true,1,0.0347757720947 +fp16,1,257,1,64,257,control_shuffle32_split,2,6,100,102.142400742,107940,1.05675996664,32896,2909,true,1,0.102142400742 +fp16,1,257,1,64,257,shuffle32_split,2,7,100,39.0982866287,107940,2.7607347868,32896,2909,true,1,0.0390982866287 +fp16,1,257,1,64,257,baseline_fused,2,8,100,65.9901714325,42148,0.638701174509,32896,2909,true,1,0.0659901714325 +fp16,1,257,1,64,257,control_optimized_fused,2,9,100,59.3309688568,42148,0.710387860035,32896,2909,true,1,0.0593309688568 +fp16,1,257,1,64,257,optimized_fused,2,10,100,29.3401145935,42148,1.43653153997,32896,2909,true,1,0.0293401145935 +fp16,1,257,1,64,257,control_shuffle32_fused,2,11,100,96.7156600952,42148,0.4357929208,32896,2909,true,1,0.0967156600952 +fp16,1,257,1,64,257,shuffle32_fused,2,12,100,33.2107424736,42148,1.26910742912,32896,2909,true,1,0.0332107424736 +fp16,1,257,1,64,257,baseline_transform,2,13,100,9.79862868786,65792,6.71440893372,32896,2909,true,1,0.00979862868786 +fp16,1,257,1,64,257,control_optimized_transform,2,14,100,9.69714283943,65792,6.78467885741,32896,2909,true,1,0.00969714283943 +fp16,1,257,1,64,257,control_shuffle32_transform,3,0,100,9.05897140503,65792,7.262634692,32896,2909,true,1,0.00905897140503 +fp16,1,257,1,64,257,shuffle32_transform,3,1,100,9.0221709013,65792,7.29225822917,32896,2909,true,1,0.0090221709013 +fp16,1,257,1,64,257,baseline_split,3,2,100,71.1849164963,107940,1.51633246638,32896,2909,true,1,0.0711849164963 +fp16,1,257,1,64,257,control_optimized_split,3,3,100,64.5755434036,107940,1.67153064939,32896,2909,true,1,0.0645755434036 +fp16,1,257,1,64,257,optimized_split,3,4,100,34.9177145958,107940,3.09126760584,32896,2909,true,1,0.0349177145958 +fp16,1,257,1,64,257,control_shuffle32_split,3,5,100,102.11268425,107940,1.05706750139,32896,2909,true,1,0.10211268425 +fp16,1,257,1,64,257,shuffle32_split,3,6,100,39.1220569611,107940,2.75905738053,32896,2909,true,1,0.0391220569611 +fp16,1,257,1,64,257,baseline_fused,3,7,100,65.9915447235,42148,0.63868788307,32896,2909,true,1,0.0659915447235 +fp16,1,257,1,64,257,control_optimized_fused,3,8,100,59.352684021,42148,0.71012795285,32896,2909,true,1,0.059352684021 +fp16,1,257,1,64,257,optimized_fused,3,9,100,29.350399971,42148,1.43602813051,32896,2909,true,1,0.029350399971 +fp16,1,257,1,64,257,control_shuffle32_fused,3,10,100,96.6708564758,42148,0.435994895841,32896,2909,true,1,0.0966708564758 +fp16,1,257,1,64,257,shuffle32_fused,3,11,100,33.2164573669,42148,1.26888907912,32896,2909,true,1,0.0332164573669 +fp16,1,257,1,64,257,baseline_transform,3,12,100,9.65805709362,65792,6.81213616385,32896,2909,true,1,0.00965805709362 +fp16,1,257,1,64,257,control_optimized_transform,3,13,100,9.69645738602,65792,6.78515847395,32896,2909,true,1,0.00969645738602 +fp16,1,257,1,64,257,optimized_transform,3,14,100,9.60662841797,65792,6.84860464437,32896,2909,true,1,0.00960662841797 +fp16,1,257,1,64,257,shuffle32_transform,4,0,100,8.87771427631,65792,7.41091658869,32896,2909,true,1,0.00887771427631 +fp16,1,257,1,64,257,baseline_split,4,1,100,71.07817173,107940,1.51860968526,32896,2909,true,1,0.07107817173 +fp16,1,257,1,64,257,control_optimized_split,4,2,100,64.4502830505,107940,1.67477930105,32896,2909,true,1,0.0644502830505 +fp16,1,257,1,64,257,optimized_split,4,3,100,34.8052573204,107940,3.10125562372,32896,2909,true,1,0.0348052573204 +fp16,1,257,1,64,257,control_shuffle32_split,4,4,100,102.228345871,107940,1.05587153035,32896,2909,true,1,0.102228345871 +fp16,1,257,1,64,257,shuffle32_split,4,5,100,39.0537142754,107940,2.7638856381,32896,2909,true,1,0.0390537142754 +fp16,1,257,1,64,257,baseline_fused,4,6,100,65.9935998917,42148,0.638667993096,32896,2909,true,1,0.0659935998917 +fp16,1,257,1,64,257,control_optimized_fused,4,7,100,59.3385124207,42148,0.710297550117,32896,2909,true,1,0.0593385124207 +fp16,1,257,1,64,257,optimized_fused,4,8,100,29.3327999115,42148,1.43688976597,32896,2909,true,1,0.0293327999115 +fp16,1,257,1,64,257,control_shuffle32_fused,4,9,100,96.7138290405,42148,0.43580117154,32896,2909,true,1,0.0967138290405 +fp16,1,257,1,64,257,shuffle32_fused,4,10,100,33.2143998146,42148,1.26896768375,32896,2909,true,1,0.0332143998146 +fp16,1,257,1,64,257,baseline_transform,4,11,100,9.70674276352,65792,6.77796884113,32896,2909,true,1,0.00970674276352 +fp16,1,257,1,64,257,control_optimized_transform,4,12,100,9.60411429405,65792,6.85039744277,32896,2909,true,1,0.00960411429405 +fp16,1,257,1,64,257,optimized_transform,4,13,100,9.59725677967,65792,6.85529224761,32896,2909,true,1,0.00959725677967 +fp16,1,257,1,64,257,control_shuffle32_transform,4,14,100,8.79131436348,65792,7.4837501288,32896,2909,true,1,0.00879131436348 +fp16,4,64,16,64,4096,baseline_transform,0,0,100,23.1433153152,1048576,45.3079425189,524288,2909,true,1,0.0231433153152 +fp16,4,64,16,64,4096,control_optimized_transform,0,1,100,23.9798855782,1048576,43.7273145688,524288,2909,true,1,0.0239798855782 +fp16,4,64,16,64,4096,optimized_transform,0,2,100,24.0050268173,1048576,43.6815175413,524288,2909,true,1,0.0240050268173 +fp16,4,64,16,64,4096,control_shuffle32_transform,0,3,100,11.269942522,1048576,93.0418232345,524288,2909,true,1,0.011269942522 +fp16,4,64,16,64,4096,shuffle32_transform,0,4,100,11.3499426842,1048576,92.3860171966,524288,2909,true,1,0.0113499426842 +fp16,4,64,16,64,4096,baseline_split,0,5,100,437.937583923,1720320,3.92823101545,524288,2909,true,1,0.437937583923 +fp16,4,64,16,64,4096,control_optimized_split,0,6,100,380.525016785,1720320,4.52091169862,524288,2909,true,1,0.380525016785 +fp16,4,64,16,64,4096,optimized_split,0,7,100,162.120456696,1720320,10.6113690713,524288,2909,true,1,0.162120456696 +fp16,4,64,16,64,4096,control_shuffle32_split,0,8,100,248.871765137,1720320,6.91247558378,524288,2909,true,1,0.248871765137 +fp16,4,64,16,64,4096,shuffle32_split,0,9,100,72.6537132263,1720320,23.6783493039,524288,2909,true,1,0.0726537132263 +fp16,4,64,16,64,4096,baseline_fused,0,10,100,433.890304565,671744,1.54818854658,524288,2909,true,1,0.433890304565 +fp16,4,64,16,64,4096,control_optimized_fused,0,11,100,370.888671875,671744,1.81117421733,524288,2909,true,1,0.370888671875 +fp16,4,64,16,64,4096,optimized_fused,0,12,100,147.895307541,671744,4.5420237543,524288,2909,true,1,0.147895307541 +fp16,4,64,16,64,4096,control_shuffle32_fused,0,13,100,248.679084778,671744,2.70124848095,524288,2909,true,1,0.248679084778 +fp16,4,64,16,64,4096,shuffle32_fused,0,14,100,68.7590837479,671744,9.769530997,524288,2909,true,1,0.0687590837479 +fp16,4,64,16,64,4096,control_optimized_transform,1,0,100,23.9382839203,1048576,43.8033070161,524288,2909,true,1,0.0239382839203 +fp16,4,64,16,64,4096,optimized_transform,1,1,100,24.005715847,1048576,43.6802637623,524288,2909,true,1,0.024005715847 +fp16,4,64,16,64,4096,control_shuffle32_transform,1,2,100,11.1860573292,1048576,93.7395517601,524288,2909,true,1,0.0111860573292 +fp16,4,64,16,64,4096,shuffle32_transform,1,3,100,11.1062860489,1048576,94.4128393042,524288,2909,true,1,0.0111062860489 +fp16,4,64,16,64,4096,baseline_split,1,4,100,437.805938721,1720320,3.92941220721,524288,2909,true,1,0.437805938721 +fp16,4,64,16,64,4096,control_optimized_split,1,5,100,382.102851868,1720320,4.50224328762,524288,2909,true,1,0.382102851868 +fp16,4,64,16,64,4096,optimized_split,1,6,100,162.537384033,1720320,10.5841496726,524288,2909,true,1,0.162537384033 +fp16,4,64,16,64,4096,control_shuffle32_split,1,7,100,249.290962219,1720320,6.90085185875,524288,2909,true,1,0.249290962219 +fp16,4,64,16,64,4096,shuffle32_split,1,8,100,70.9129142761,1720320,24.259615016,524288,2909,true,1,0.0709129142761 +fp16,4,64,16,64,4096,baseline_fused,1,9,100,433.926391602,671744,1.54805979309,524288,2909,true,1,0.433926391602 +fp16,4,64,16,64,4096,control_optimized_fused,1,10,100,371.781692505,671744,1.80682377197,524288,2909,true,1,0.371781692505 +fp16,4,64,16,64,4096,optimized_fused,1,11,100,147.653942108,671744,4.54944846314,524288,2909,true,1,0.147653942108 +fp16,4,64,16,64,4096,control_shuffle32_fused,1,12,100,247.970962524,671744,2.70896234447,524288,2909,true,1,0.247970962524 +fp16,4,64,16,64,4096,shuffle32_fused,1,13,100,68.0484580994,671744,9.87155357759,524288,2909,true,1,0.0680484580994 +fp16,4,64,16,64,4096,baseline_transform,1,14,100,22.8011417389,1048576,45.9878725376,524288,2909,true,1,0.0228011417389 +fp16,4,64,16,64,4096,optimized_transform,2,0,100,24.2114281654,1048576,43.3091345473,524288,2909,true,1,0.0242114281654 +fp16,4,64,16,64,4096,control_shuffle32_transform,2,1,100,11.1028575897,1048576,94.4419931109,524288,2909,true,1,0.0111028575897 +fp16,4,64,16,64,4096,shuffle32_transform,2,2,100,11.412115097,1048576,91.8827045717,524288,2909,true,1,0.011412115097 +fp16,4,64,16,64,4096,baseline_split,2,3,100,437.011871338,1720320,3.93655210037,524288,2909,true,1,0.437011871338 +fp16,4,64,16,64,4096,control_optimized_split,2,4,100,381.82308197,1720320,4.50554217708,524288,2909,true,1,0.38182308197 +fp16,4,64,16,64,4096,optimized_split,2,5,100,162.611198425,1720320,10.5793451906,524288,2909,true,1,0.162611198425 +fp16,4,64,16,64,4096,control_shuffle32_split,2,6,100,248.551769257,1720320,6.9213749922,524288,2909,true,1,0.248551769257 +fp16,4,64,16,64,4096,shuffle32_split,2,7,100,70.584230423,1720320,24.3725827949,524288,2909,true,1,0.070584230423 +fp16,4,64,16,64,4096,baseline_fused,2,8,100,433.973693848,671744,1.54789105774,524288,2909,true,1,0.433973693848 +fp16,4,64,16,64,4096,control_optimized_fused,2,9,100,371.947441101,671744,1.80601860847,524288,2909,true,1,0.371947441101 +fp16,4,64,16,64,4096,optimized_fused,2,10,100,147.472686768,671744,4.55504008724,524288,2909,true,1,0.147472686768 +fp16,4,64,16,64,4096,control_shuffle32_fused,2,11,100,248.29668045,671744,2.70540870213,524288,2909,true,1,0.24829668045 +fp16,4,64,16,64,4096,shuffle32_fused,2,12,100,68.3291435242,671744,9.83100278086,524288,2909,true,1,0.0683291435242 +fp16,4,64,16,64,4096,baseline_transform,2,13,100,23.226287365,1048576,45.1460874277,524288,2909,true,1,0.023226287365 +fp16,4,64,16,64,4096,control_optimized_transform,2,14,100,23.9696002007,1048576,43.7460779997,524288,2909,true,1,0.0239696002007 +fp16,4,64,16,64,4096,control_shuffle32_transform,3,0,100,11.6530287266,1048576,89.9831301032,524288,2909,true,1,0.0116530287266 +fp16,4,64,16,64,4096,shuffle32_transform,3,1,100,11.2379419804,1048576,93.3067639816,524288,2909,true,1,0.0112379419804 +fp16,4,64,16,64,4096,baseline_split,3,2,100,439.051208496,1720320,3.91826731532,524288,2909,true,1,0.439051208496 +fp16,4,64,16,64,4096,control_optimized_split,3,3,100,382.447090149,1720320,4.49819084603,524288,2909,true,1,0.382447090149 +fp16,4,64,16,64,4096,optimized_split,3,4,100,161.949939728,1720320,10.6225417737,524288,2909,true,1,0.161949939728 +fp16,4,64,16,64,4096,control_shuffle32_split,3,5,100,247.200222015,1720320,6.95921705076,524288,2909,true,1,0.247200222015 +fp16,4,64,16,64,4096,shuffle32_split,3,6,100,70.5353116989,1720320,24.3894860399,524288,2909,true,1,0.0705353116989 +fp16,4,64,16,64,4096,baseline_fused,3,7,100,433.846855164,671744,1.54834359638,524288,2909,true,1,0.433846855164 +fp16,4,64,16,64,4096,control_optimized_fused,3,8,100,371.6015625,671744,1.80769961106,524288,2909,true,1,0.3716015625 +fp16,4,64,16,64,4096,optimized_fused,3,9,100,147.93554306,671744,4.54078841436,524288,2909,true,1,0.14793554306 +fp16,4,64,16,64,4096,control_shuffle32_fused,3,10,100,247.294406891,671744,2.71637360685,524288,2909,true,1,0.247294406891 +fp16,4,64,16,64,4096,shuffle32_fused,3,11,100,69.1846847534,671744,9.70943211484,524288,2909,true,1,0.0691846847534 +fp16,4,64,16,64,4096,baseline_transform,3,12,100,23.1890296936,1048576,45.2186233687,524288,2909,true,1,0.0231890296936 +fp16,4,64,16,64,4096,control_optimized_transform,3,13,100,24.1142868996,1048576,43.4835997584,524288,2909,true,1,0.0241142868996 +fp16,4,64,16,64,4096,optimized_transform,3,14,100,24.0676569939,1048576,43.5678471015,524288,2909,true,1,0.0240676569939 +fp16,4,64,16,64,4096,shuffle32_transform,4,0,100,11.369600296,1048576,92.226285243,524288,2909,true,1,0.011369600296 +fp16,4,64,16,64,4096,baseline_split,4,1,100,437.938499451,1720320,3.92822280333,524288,2909,true,1,0.437938499451 +fp16,4,64,16,64,4096,control_optimized_split,4,2,100,381.407775879,1720320,4.51044815758,524288,2909,true,1,0.381407775879 +fp16,4,64,16,64,4096,optimized_split,4,3,100,162.67747879,1720320,10.5750348038,524288,2909,true,1,0.16267747879 +fp16,4,64,16,64,4096,control_shuffle32_split,4,4,100,247.637939453,1720320,6.94691614621,524288,2909,true,1,0.247637939453 +fp16,4,64,16,64,4096,shuffle32_split,4,5,100,71.7942857742,1720320,23.9617955865,524288,2909,true,1,0.0717942857742 +fp16,4,64,16,64,4096,baseline_fused,4,6,100,433.982887268,671744,1.54785826747,524288,2909,true,1,0.433982887268 +fp16,4,64,16,64,4096,control_optimized_fused,4,7,100,372.141952515,671744,1.80507463741,524288,2909,true,1,0.372141952515 +fp16,4,64,16,64,4096,optimized_fused,4,8,100,147.847995758,671744,4.54347721493,524288,2909,true,1,0.147847995758 +fp16,4,64,16,64,4096,control_shuffle32_fused,4,9,100,248.671321869,671744,2.7013328073,524288,2909,true,1,0.248671321869 +fp16,4,64,16,64,4096,shuffle32_fused,4,10,100,68.3337116241,671744,9.83034557957,524288,2909,true,1,0.0683337116241 +fp16,4,64,16,64,4096,baseline_transform,4,11,100,22.7876567841,1048576,46.0150865855,524288,2909,true,1,0.0227876567841 +fp16,4,64,16,64,4096,control_optimized_transform,4,12,100,23.9755415916,1048576,43.7352372622,524288,2909,true,1,0.0239755415916 +fp16,4,64,16,64,4096,optimized_transform,4,13,100,24.0226268768,1048576,43.6495144922,524288,2909,true,1,0.0240226268768 +fp16,4,64,16,64,4096,control_shuffle32_transform,4,14,100,11.1471998692,1048576,94.0663137208,524288,2909,true,1,0.0111471998692 +fp16,16,64,16,64,16384,baseline_transform,0,0,100,68.298740387,4194304,61.4111472076,2097152,2909,true,1,0.068298740387 +fp16,16,64,16,64,16384,control_optimized_transform,0,1,100,73.9970302582,4194304,56.6820585281,2097152,2909,true,1,0.0739970302582 +fp16,16,64,16,64,16384,optimized_transform,0,2,100,74.0484571457,4194304,56.6426926593,2097152,2909,true,1,0.0740484571457 +fp16,16,64,16,64,16384,control_shuffle32_transform,0,3,100,21.2681150436,4194304,197.210894872,2097152,2909,true,1,0.0212681150436 +fp16,16,64,16,64,16384,shuffle32_transform,0,4,100,21.2706279755,4194304,197.187596193,2097152,2909,true,1,0.0212706279755 +fp16,16,64,16,64,16384,baseline_split,0,5,100,1548.35784912,6881280,4.44424394781,2097152,2909,true,1,1.54835784912 +fp16,16,64,16,64,16384,control_optimized_split,0,6,100,1361.84585571,6881280,5.05290666424,2097152,2909,true,1,1.36184585571 +fp16,16,64,16,64,16384,optimized_split,0,7,100,576.817855835,6881280,11.9297277822,2097152,2909,true,1,0.576817855835 +fp16,16,64,16,64,16384,control_shuffle32_split,0,8,100,851.698760986,6881280,8.07947635386,2097152,2909,true,1,0.851698760986 +fp16,16,64,16,64,16384,shuffle32_split,0,9,100,228.516788483,6881280,30.1127984762,2097152,2909,true,1,0.228516788483 +fp16,16,64,16,64,16384,baseline_fused,0,10,100,1526.21673584,2686976,1.7605468063,2097152,2909,true,1,1.52621673584 +fp16,16,64,16,64,16384,control_optimized_fused,0,11,100,1333.61877441,2686976,2.01480066984,2097152,2909,true,1,1.33361877441 +fp16,16,64,16,64,16384,optimized_fused,0,12,100,544.386062622,2686976,4.93579131519,2097152,2909,true,1,0.544386062622 +fp16,16,64,16,64,16384,control_shuffle32_fused,0,13,100,866.503982544,2686976,3.10093900793,2097152,2909,true,1,0.866503982544 +fp16,16,64,16,64,16384,shuffle32_fused,0,14,100,211.431083679,2686976,12.7085192643,2097152,2909,true,1,0.211431083679 +fp16,16,64,16,64,16384,control_optimized_transform,1,0,100,73.9876556396,4194304,56.689240438,2097152,2909,true,1,0.0739876556396 +fp16,16,64,16,64,16384,optimized_transform,1,1,100,74.0518856049,4194304,56.6400702121,2097152,2909,true,1,0.0740518856049 +fp16,16,64,16,64,16384,control_shuffle32_transform,1,2,100,21.4009141922,4194304,195.987141593,2097152,2909,true,1,0.0214009141922 +fp16,16,64,16,64,16384,shuffle32_transform,1,3,100,21.3147425652,4194304,196.779481956,2097152,2909,true,1,0.0213147425652 +fp16,16,64,16,64,16384,baseline_split,1,4,100,1548.5760498,6881280,4.4436177357,2097152,2909,true,1,1.5485760498 +fp16,16,64,16,64,16384,control_optimized_split,1,5,100,1361.74468994,6881280,5.0532820512,2097152,2909,true,1,1.36174468994 +fp16,16,64,16,64,16384,optimized_split,1,6,100,576.692771912,6881280,11.9323153248,2097152,2909,true,1,0.576692771912 +fp16,16,64,16,64,16384,control_shuffle32_split,1,7,100,851.731033325,6881280,8.07917022013,2097152,2909,true,1,0.851731033325 +fp16,16,64,16,64,16384,shuffle32_split,1,8,100,228.429489136,6881280,30.1243067436,2097152,2909,true,1,0.228429489136 +fp16,16,64,16,64,16384,baseline_fused,1,9,100,1526.08840942,2686976,1.76069484796,2097152,2909,true,1,1.52608840942 +fp16,16,64,16,64,16384,control_optimized_fused,1,10,100,1333.28323364,2686976,2.01530772472,2097152,2909,true,1,1.33328323364 +fp16,16,64,16,64,16384,optimized_fused,1,11,100,544.491195679,2686976,4.93483828816,2097152,2909,true,1,0.544491195679 +fp16,16,64,16,64,16384,control_shuffle32_fused,1,12,100,866.502609253,2686976,3.10094392251,2097152,2909,true,1,0.866502609253 +fp16,16,64,16,64,16384,shuffle32_fused,1,13,100,211.367301941,2686976,12.7123541594,2097152,2909,true,1,0.211367301941 +fp16,16,64,16,64,16384,baseline_transform,1,14,100,68.2815980911,4194304,61.4265646566,2097152,2909,true,1,0.0682815980911 +fp16,16,64,16,64,16384,optimized_transform,2,0,100,74.0447998047,4194304,56.6454904472,2097152,2909,true,1,0.0740447998047 +fp16,16,64,16,64,16384,control_shuffle32_transform,2,1,100,21.3840007782,4194304,196.142155226,2097152,2909,true,1,0.0213840007782 +fp16,16,64,16,64,16384,shuffle32_transform,2,2,100,21.2962293625,4194304,196.950545968,2097152,2909,true,1,0.0212962293625 +fp16,16,64,16,64,16384,baseline_split,2,3,100,1548.55514526,6881280,4.44367772181,2097152,2909,true,1,1.54855514526 +fp16,16,64,16,64,16384,control_optimized_split,2,4,100,1361.72683716,6881280,5.05334830175,2097152,2909,true,1,1.36172683716 +fp16,16,64,16,64,16384,optimized_split,2,5,100,576.781692505,6881280,11.9304757578,2097152,2909,true,1,0.576781692505 +fp16,16,64,16,64,16384,control_shuffle32_split,2,6,100,851.739196777,6881280,8.07909278572,2097152,2909,true,1,0.851739196777 +fp16,16,64,16,64,16384,shuffle32_split,2,7,100,228.459434509,6881280,30.120358193,2097152,2909,true,1,0.228459434509 +fp16,16,64,16,64,16384,baseline_fused,2,8,100,1526.10900879,2686976,1.76067108216,2097152,2909,true,1,1.52610900879 +fp16,16,64,16,64,16384,control_optimized_fused,2,9,100,1333.48098755,2686976,2.01500885659,2097152,2909,true,1,1.33348098755 +fp16,16,64,16,64,16384,optimized_fused,2,10,100,544.504890442,2686976,4.93471417276,2097152,2909,true,1,0.544504890442 +fp16,16,64,16,64,16384,control_shuffle32_fused,2,11,100,866.417388916,2686976,3.10124892964,2097152,2909,true,1,0.866417388916 +fp16,16,64,16,64,16384,shuffle32_fused,2,12,100,211.414394379,2686976,12.7095224897,2097152,2909,true,1,0.211414394379 +fp16,16,64,16,64,16384,baseline_transform,2,13,100,68.2781744003,4194304,61.4296447853,2097152,2909,true,1,0.0682781744003 +fp16,16,64,16,64,16384,control_optimized_transform,2,14,100,73.9888000488,4194304,56.6883636068,2097152,2909,true,1,0.0739888000488 +fp16,16,64,16,64,16384,control_shuffle32_transform,3,0,100,21.3819432259,4194304,196.161029692,2097152,2909,true,1,0.0213819432259 +fp16,16,64,16,64,16384,shuffle32_transform,3,1,100,21.2137150764,4194304,197.716617994,2097152,2909,true,1,0.0212137150764 +fp16,16,64,16,64,16384,baseline_split,3,2,100,1548.44955444,6881280,4.44398074206,2097152,2909,true,1,1.54844955444 +fp16,16,64,16,64,16384,control_optimized_split,3,3,100,1361.58157349,6881280,5.05388743062,2097152,2909,true,1,1.36158157349 +fp16,16,64,16,64,16384,optimized_split,3,4,100,576.782188416,6881280,11.9304655002,2097152,2909,true,1,0.576782188416 +fp16,16,64,16,64,16384,control_shuffle32_split,3,5,100,851.698074341,6881280,8.07948286759,2097152,2909,true,1,0.851698074341 +fp16,16,64,16,64,16384,shuffle32_split,3,6,100,228.497142792,6881280,30.1153875096,2097152,2909,true,1,0.228497142792 +fp16,16,64,16,64,16384,baseline_fused,3,7,100,1526.14471436,2686976,1.76062988963,2097152,2909,true,1,1.52614471436 +fp16,16,64,16,64,16384,control_optimized_fused,3,8,100,1333.41598511,2686976,2.01510708587,2097152,2909,true,1,1.33341598511 +fp16,16,64,16,64,16384,optimized_fused,3,9,100,544.401359558,2686976,4.93565262618,2097152,2909,true,1,0.544401359558 +fp16,16,64,16,64,16384,control_shuffle32_fused,3,10,100,866.502380371,2686976,3.10094474161,2097152,2909,true,1,0.866502380371 +fp16,16,64,16,64,16384,shuffle32_fused,3,11,100,211.828804016,2686976,12.684658314,2097152,2909,true,1,0.211828804016 +fp16,16,64,16,64,16384,baseline_transform,3,12,100,68.2882261276,4194304,61.4206026111,2097152,2909,true,1,0.0682882261276 +fp16,16,64,16,64,16384,control_optimized_transform,3,13,100,74.0025138855,4194304,56.6778583561,2097152,2909,true,1,0.0740025138855 +fp16,16,64,16,64,16384,optimized_transform,3,14,100,74.0537118912,4194304,56.6386733749,2097152,2909,true,1,0.0740537118912 +fp16,16,64,16,64,16384,shuffle32_transform,4,0,100,21.280913353,4194304,197.092292536,2097152,2909,true,1,0.021280913353 +fp16,16,64,16,64,16384,baseline_split,4,1,100,1548.50036621,6881280,4.44383491935,2097152,2909,true,1,1.54850036621 +fp16,16,64,16,64,16384,control_optimized_split,4,2,100,1361.55212402,6881280,5.05399674282,2097152,2909,true,1,1.36155212402 +fp16,16,64,16,64,16384,optimized_split,4,3,100,576.856918335,6881280,11.9289199475,2097152,2909,true,1,0.576856918335 +fp16,16,64,16,64,16384,control_shuffle32_split,4,4,100,851.781311035,6881280,8.0786933346,2097152,2909,true,1,0.851781311035 +fp16,16,64,16,64,16384,shuffle32_split,4,5,100,228.443641663,6881280,30.1224404843,2097152,2909,true,1,0.228443641663 +fp16,16,64,16,64,16384,baseline_fused,4,6,100,1526.21643066,2686976,1.76054715833,2097152,2909,true,1,1.52621643066 +fp16,16,64,16,64,16384,control_optimized_fused,4,7,100,1333.43383789,2686976,2.01508010645,2097152,2909,true,1,1.33343383789 +fp16,16,64,16,64,16384,optimized_fused,4,8,100,544.41783905,2686976,4.9355032243,2097152,2909,true,1,0.54441783905 +fp16,16,64,16,64,16384,control_shuffle32_fused,4,9,100,866.450271606,2686976,3.10113123402,2097152,2909,true,1,0.866450271606 +fp16,16,64,16,64,16384,shuffle32_fused,4,10,100,211.380786896,2686976,12.7115431798,2097152,2909,true,1,0.211380786896 +fp16,16,64,16,64,16384,baseline_transform,4,11,100,68.2182836533,4194304,61.4835755956,2097152,2909,true,1,0.0682182836533 +fp16,16,64,16,64,16384,control_optimized_transform,4,12,100,73.9405727386,4194304,56.7253382635,2097152,2909,true,1,0.0739405727386 +fp16,16,64,16,64,16384,optimized_transform,4,13,100,74.0226268768,4194304,56.6624581829,2097152,2909,true,1,0.0740226268768 +fp16,16,64,16,64,16384,control_shuffle32_transform,4,14,100,21.1977148056,4194304,197.865856696,2097152,2909,true,1,0.0211977148056 +fp16,1,1,1,128,1,baseline_transform,0,0,100,9.41760003567,512,0.0543662926925,256,2909,true,1,0.00941760003567 +fp16,1,1,1,128,1,control_optimized_transform,0,1,100,9.55862879753,512,0.0535641681297,256,2909,true,1,0.00955862879753 +fp16,1,1,1,128,1,optimized_transform,0,2,100,9.73942875862,512,0.0525698182808,256,2909,true,1,0.00973942875862 +fp16,1,1,1,128,1,control_shuffle32_transform,0,3,100,9.14308607578,512,0.0559985978209,256,2909,true,1,0.00914308607578 +fp16,1,1,1,128,1,shuffle32_transform,0,4,100,9.2102855444,512,0.0555900246015,256,2909,true,1,0.0092102855444 +fp16,1,1,1,128,1,baseline_split,0,5,100,80.2006816864,836,0.0104238515487,256,2909,true,1,0.0802006816864 +fp16,1,1,1,128,1,control_optimized_split,0,6,100,64.7165727615,836,0.0129178657696,256,2909,true,1,0.0647165727615 +fp16,1,1,1,128,1,optimized_split,0,7,100,34.1003441811,836,0.0245158815864,256,2909,true,1,0.0341003441811 +fp16,1,1,1,128,1,control_shuffle32_split,0,8,100,164.734859467,836,0.00507482145981,256,2909,true,1,0.164734859467 +fp16,1,1,1,128,1,shuffle32_split,0,9,100,46.0717725754,836,0.0181456009454,256,2909,true,1,0.0460717725754 +fp16,1,1,1,128,1,baseline_fused,0,10,100,75.7705163956,324,0.0042760695771,256,2909,true,1,0.0757705163956 +fp16,1,1,1,128,1,control_optimized_fused,0,11,100,59.2063999176,324,0.00547238137179,256,2909,true,1,0.0592063999176 +fp16,1,1,1,128,1,optimized_fused,0,12,100,28.9645719528,324,0.011186079343,256,2909,true,1,0.0289645719528 +fp16,1,1,1,128,1,control_shuffle32_fused,0,13,100,159.349031448,324,0.00203327247775,256,2909,true,1,0.159349031448 +fp16,1,1,1,128,1,shuffle32_fused,0,14,100,40.3734874725,324,0.00802506843682,256,2909,true,1,0.0403734874725 +fp16,1,1,1,128,1,control_optimized_transform,1,0,100,9.56799983978,512,0.0535117065817,256,2909,true,1,0.00956799983978 +fp16,1,1,1,128,1,optimized_transform,1,1,100,9.69439983368,512,0.052813996615,256,2909,true,1,0.00969439983368 +fp16,1,1,1,128,1,control_shuffle32_transform,1,2,100,9.1503995657,512,0.0559538407393,256,2909,true,1,0.0091503995657 +fp16,1,1,1,128,1,shuffle32_transform,1,3,100,9.21097099781,512,0.0555858877551,256,2909,true,1,0.00921097099781 +fp16,1,1,1,128,1,baseline_split,1,4,100,80.1926803589,836,0.0104248916018,256,2909,true,1,0.0801926803589 +fp16,1,1,1,128,1,control_optimized_split,1,5,100,64.7261714935,836,0.0129159500819,256,2909,true,1,0.0647261714935 +fp16,1,1,1,128,1,optimized_split,1,6,100,34.1284584999,836,0.0244956859098,256,2909,true,1,0.0341284584999 +fp16,1,1,1,128,1,control_shuffle32_split,1,7,100,165.381717682,836,0.00505497228907,256,2909,true,1,0.165381717682 +fp16,1,1,1,128,1,shuffle32_split,1,8,100,46.2086868286,836,0.0180918363489,256,2909,true,1,0.0462086868286 +fp16,1,1,1,128,1,baseline_fused,1,9,100,75.7741689682,324,0.004275863456,256,2909,true,1,0.0757741689682 +fp16,1,1,1,128,1,control_optimized_fused,1,10,100,59.2665147781,324,0.0054668306583,256,2909,true,1,0.0592665147781 +fp16,1,1,1,128,1,optimized_fused,1,11,100,28.9465141296,324,0.011193057601,256,2909,true,1,0.0289465141296 +fp16,1,1,1,128,1,control_shuffle32_fused,1,12,100,159.293489456,324,0.0020339814333,256,2909,true,1,0.159293489456 +fp16,1,1,1,128,1,shuffle32_fused,1,13,100,40.3673171997,324,0.00802629509405,256,2909,true,1,0.0403673171997 +fp16,1,1,1,128,1,baseline_transform,1,14,100,9.40800011158,512,0.054421768062,256,2909,true,1,0.00940800011158 +fp16,1,1,1,128,1,optimized_transform,2,0,100,9.61531400681,512,0.0532483910185,256,2909,true,1,0.00961531400681 +fp16,1,1,1,128,1,control_shuffle32_transform,2,1,100,9.26217138767,512,0.0552786143303,256,2909,true,1,0.00926217138767 +fp16,1,1,1,128,1,shuffle32_transform,2,2,100,9.23977136612,512,0.0554126265372,256,2909,true,1,0.00923977136612 +fp16,1,1,1,128,1,baseline_split,2,3,100,80.1999950409,836,0.0104239407942,256,2909,true,1,0.0801999950409 +fp16,1,1,1,128,1,control_optimized_split,2,4,100,64.6946287155,836,0.0129222474354,256,2909,true,1,0.0646946287155 +fp16,1,1,1,128,1,optimized_split,2,5,100,34.0939426422,836,0.0245204847316,256,2909,true,1,0.0340939426422 +fp16,1,1,1,128,1,control_shuffle32_split,2,6,100,165.457839966,836,0.00505264664505,256,2909,true,1,0.165457839966 +fp16,1,1,1,128,1,shuffle32_split,2,7,100,46.1860561371,836,0.0181007011622,256,2909,true,1,0.0461860561371 +fp16,1,1,1,128,1,baseline_fused,2,8,100,75.7711982727,324,0.00427603109606,256,2909,true,1,0.0757711982727 +fp16,1,1,1,128,1,control_optimized_fused,2,9,100,59.2562294006,324,0.00546777956136,256,2909,true,1,0.0592562294006 +fp16,1,1,1,128,1,optimized_fused,2,10,100,28.951086998,324,0.0111912896404,256,2909,true,1,0.028951086998 +fp16,1,1,1,128,1,control_shuffle32_fused,2,11,100,159.349479675,324,0.00203326675845,256,2909,true,1,0.159349479675 +fp16,1,1,1,128,1,shuffle32_fused,2,12,100,40.3721141815,324,0.00802534141619,256,2909,true,1,0.0403721141815 +fp16,1,1,1,128,1,baseline_transform,2,13,100,9.41485702991,512,0.0543821322377,256,2909,true,1,0.00941485702991 +fp16,1,1,1,128,1,control_optimized_transform,2,14,100,9.58377122879,512,0.0534236458464,256,2909,true,1,0.00958377122879 +fp16,1,1,1,128,1,control_shuffle32_transform,3,0,100,9.10971403122,512,0.0562037401224,256,2909,true,1,0.00910971403122 +fp16,1,1,1,128,1,shuffle32_transform,3,1,100,9.20000016689,512,0.0556521729035,256,2909,true,1,0.00920000016689 +fp16,1,1,1,128,1,baseline_split,3,2,100,80.2175998688,836,0.0104216531206,256,2909,true,1,0.0802175998688 +fp16,1,1,1,128,1,control_optimized_split,3,3,100,64.7924566269,836,0.0129027365765,256,2909,true,1,0.0647924566269 +fp16,1,1,1,128,1,optimized_split,3,4,100,34.1636562347,836,0.0244704487791,256,2909,true,1,0.0341636562347 +fp16,1,1,1,128,1,control_shuffle32_split,3,5,100,164.733715057,836,0.00507485671472,256,2909,true,1,0.164733715057 +fp16,1,1,1,128,1,shuffle32_split,3,6,100,46.1490249634,836,0.018115225634,256,2909,true,1,0.0461490249634 +fp16,1,1,1,128,1,baseline_fused,3,7,100,75.7209157944,324,0.00427887059475,256,2909,true,1,0.0757209157944 +fp16,1,1,1,128,1,control_optimized_fused,3,8,100,59.2683458328,324,0.00546666176434,256,2909,true,1,0.0592683458328 +fp16,1,1,1,128,1,optimized_fused,3,9,100,28.953602314,324,0.011190317408,256,2909,true,1,0.028953602314 +fp16,1,1,1,128,1,control_shuffle32_fused,3,10,100,159.306507111,324,0.00203381522749,256,2909,true,1,0.159306507111 +fp16,1,1,1,128,1,shuffle32_fused,3,11,100,40.4253673553,324,0.0080147694677,256,2909,true,1,0.0404253673553 +fp16,1,1,1,128,1,baseline_transform,3,12,100,9.43611443043,512,0.0542596217728,256,2909,true,1,0.00943611443043 +fp16,1,1,1,128,1,control_optimized_transform,3,13,100,9.63817119598,512,0.0531221109886,256,2909,true,1,0.00963817119598 +fp16,1,1,1,128,1,optimized_transform,3,14,100,9.71337139606,512,0.0527108435499,256,2909,true,1,0.00971337139606 +fp16,1,1,1,128,1,shuffle32_transform,4,0,100,9.25919950008,512,0.0552963568822,256,2909,true,1,0.00925919950008 +fp16,1,1,1,128,1,baseline_split,4,1,100,80.2112007141,836,0.0104224845478,256,2909,true,1,0.0802112007141 +fp16,1,1,1,128,1,control_optimized_split,4,2,100,64.7369098663,836,0.0129138076211,256,2909,true,1,0.0647369098663 +fp16,1,1,1,128,1,optimized_split,4,3,100,35.1250290871,836,0.0238006920344,256,2909,true,1,0.0351250290871 +fp16,1,1,1,128,1,control_shuffle32_split,4,4,100,164.832668304,836,0.00507181014904,256,2909,true,1,0.164832668304 +fp16,1,1,1,128,1,shuffle32_split,4,5,100,46.1600017548,836,0.0181109178557,256,2909,true,1,0.0461600017548 +fp16,1,1,1,128,1,baseline_fused,4,6,100,75.728225708,324,0.00427845756283,256,2909,true,1,0.075728225708 +fp16,1,1,1,128,1,control_optimized_fused,4,7,100,59.260802269,324,0.00546735763936,256,2909,true,1,0.059260802269 +fp16,1,1,1,128,1,optimized_fused,4,8,100,28.9172554016,324,0.011204382833,256,2909,true,1,0.0289172554016 +fp16,1,1,1,128,1,control_shuffle32_fused,4,9,100,159.348564148,324,0.00203327844046,256,2909,true,1,0.159348564148 +fp16,1,1,1,128,1,shuffle32_fused,4,10,100,40.4217147827,324,0.00801549369545,256,2909,true,1,0.0404217147827 +fp16,1,1,1,128,1,baseline_transform,4,11,100,9.33554291725,512,0.0548441589887,256,2909,true,1,0.00933554291725 +fp16,1,1,1,128,1,control_optimized_transform,4,12,100,9.64845716953,512,0.053065478864,256,2909,true,1,0.00964845716953 +fp16,1,1,1,128,1,optimized_transform,4,13,100,9.69828546047,512,0.0527928366397,256,2909,true,1,0.00969828546047 +fp16,1,1,1,128,1,control_shuffle32_transform,4,14,100,9.18902873993,512,0.0557186199424,256,2909,true,1,0.00918902873993 +fp16,1,17,1,128,17,baseline_transform,0,0,100,9.58880007267,8704,0.907725673081,4352,2909,true,1,0.00958880007267 +fp16,1,17,1,128,17,control_optimized_transform,0,1,100,9.68982875347,8704,0.898261488562,4352,2909,true,1,0.00968982875347 +fp16,1,17,1,128,17,optimized_transform,0,2,100,9.7931432724,8704,0.888785118107,4352,2909,true,1,0.0097931432724 +fp16,1,17,1,128,17,control_shuffle32_transform,0,3,100,9.28982853889,8704,0.936938713514,4352,2909,true,1,0.00928982853889 +fp16,1,17,1,128,17,shuffle32_transform,0,4,100,9.26125764847,8704,0.939829160399,4352,2909,true,1,0.00926125764847 +fp16,1,17,1,128,17,baseline_split,0,5,100,80.3488063812,14212,0.176878794348,4352,2909,true,1,0.0803488063812 +fp16,1,17,1,128,17,control_optimized_split,0,6,100,64.4400024414,14212,0.220546236213,4352,2909,true,1,0.0644400024414 +fp16,1,17,1,128,17,optimized_split,0,7,100,34.7197699547,14212,0.409334509375,4352,2909,true,1,0.0347197699547 +fp16,1,17,1,128,17,control_shuffle32_split,0,8,100,163.313827515,14212,0.0870226374354,4352,2909,true,1,0.163313827515 +fp16,1,17,1,128,17,shuffle32_split,0,9,100,45.6198883057,14212,0.311530793429,4352,2909,true,1,0.0456198883057 +fp16,1,17,1,128,17,baseline_fused,0,10,100,75.7264041901,5508,0.0727355281016,4352,2909,true,1,0.0757264041901 +fp16,1,17,1,128,17,control_optimized_fused,0,11,100,59.3339395523,5508,0.0928305122087,4352,2909,true,1,0.0593339395523 +fp16,1,17,1,128,17,optimized_fused,0,12,100,29.3835449219,5508,0.187451854929,4352,2909,true,1,0.0293835449219 +fp16,1,17,1,128,17,control_shuffle32_fused,0,13,100,157.950630188,5508,0.0348716557411,4352,2909,true,1,0.157950630188 +fp16,1,17,1,128,17,shuffle32_fused,0,14,100,39.93942976,5508,0.137908829272,4352,2909,true,1,0.03993942976 +fp16,1,17,1,128,17,control_optimized_transform,1,0,100,9.75337147713,8704,0.892409360231,4352,2909,true,1,0.00975337147713 +fp16,1,17,1,128,17,optimized_transform,1,1,100,9.85897123814,8704,0.882850734601,4352,2909,true,1,0.00985897123814 +fp16,1,17,1,128,17,control_shuffle32_transform,1,2,100,9.21074271202,8704,0.94498351242,4352,2909,true,1,0.00921074271202 +fp16,1,17,1,128,17,shuffle32_transform,1,3,100,9.29508566856,8704,0.936408798193,4352,2909,true,1,0.00929508566856 +fp16,1,17,1,128,17,baseline_split,1,4,100,80.3199958801,14212,0.176942240152,4352,2909,true,1,0.0803199958801 +fp16,1,17,1,128,17,control_optimized_split,1,5,100,64.4992017746,14212,0.220343812155,4352,2909,true,1,0.0644992017746 +fp16,1,17,1,128,17,optimized_split,1,6,100,34.8736000061,14212,0.407528904315,4352,2909,true,1,0.0348736000061 +fp16,1,17,1,128,17,control_shuffle32_split,1,7,100,163.448925018,14212,0.0869507095162,4352,2909,true,1,0.163448925018 +fp16,1,17,1,128,17,shuffle32_split,1,8,100,45.6989717484,14212,0.310991680037,4352,2909,true,1,0.0456989717484 +fp16,1,17,1,128,17,baseline_fused,1,9,100,75.738286972,5508,0.0727241164305,4352,2909,true,1,0.075738286972 +fp16,1,17,1,128,17,control_optimized_fused,1,10,100,59.3398857117,5508,0.0928212101176,4352,2909,true,1,0.0593398857117 +fp16,1,17,1,128,17,optimized_fused,1,11,100,29.3901729584,5508,0.187409581012,4352,2909,true,1,0.0293901729584 +fp16,1,17,1,128,17,control_shuffle32_fused,1,12,100,157.991085052,5508,0.0348627265783,4352,2909,true,1,0.157991085052 +fp16,1,17,1,128,17,shuffle32_fused,1,13,100,40.0390863419,5508,0.137565576621,4352,2909,true,1,0.0400390863419 +fp16,1,17,1,128,17,baseline_transform,1,14,100,9.69599962234,8704,0.897689803942,4352,2909,true,1,0.00969599962234 +fp16,1,17,1,128,17,optimized_transform,2,0,100,9.87542927265,8704,0.881379407385,4352,2909,true,1,0.00987542927265 +fp16,1,17,1,128,17,control_shuffle32_transform,2,1,100,9.23291444778,8704,0.942714247947,4352,2909,true,1,0.00923291444778 +fp16,1,17,1,128,17,shuffle32_transform,2,2,100,9.33485686779,8704,0.932419224341,4352,2909,true,1,0.00933485686779 +fp16,1,17,1,128,17,baseline_split,2,3,100,80.3266239166,14212,0.176927640016,4352,2909,true,1,0.0803266239166 +fp16,1,17,1,128,17,control_optimized_split,2,4,100,64.5247983932,14212,0.220256403025,4352,2909,true,1,0.0645247983932 +fp16,1,17,1,128,17,optimized_split,2,5,100,34.7209143639,14212,0.409321017617,4352,2909,true,1,0.0347209143639 +fp16,1,17,1,128,17,control_shuffle32_split,2,6,100,163.359775543,14212,0.0869981606717,4352,2909,true,1,0.163359775543 +fp16,1,17,1,128,17,shuffle32_split,2,7,100,45.6779432297,14212,0.31113484967,4352,2909,true,1,0.0456779432297 +fp16,1,17,1,128,17,baseline_fused,2,8,100,75.7229709625,5508,0.0727388258805,4352,2909,true,1,0.0757229709625 +fp16,1,17,1,128,17,control_optimized_fused,2,9,100,59.3373680115,5508,0.0928251485461,4352,2909,true,1,0.0593373680115 +fp16,1,17,1,128,17,optimized_fused,2,10,100,29.4406843185,5508,0.187088042533,4352,2909,true,1,0.0294406843185 +fp16,1,17,1,128,17,control_shuffle32_fused,2,11,100,157.990627289,5508,0.03486282759,4352,2909,true,1,0.157990627289 +fp16,1,17,1,128,17,shuffle32_fused,2,12,100,39.901714325,5508,0.138039181854,4352,2909,true,1,0.039901714325 +fp16,1,17,1,128,17,baseline_transform,2,13,100,9.70011413097,8704,0.897309029819,4352,2909,true,1,0.00970011413097 +fp16,1,17,1,128,17,control_optimized_transform,2,14,100,9.74605739117,8704,0.893079083229,4352,2909,true,1,0.00974605739117 +fp16,1,17,1,128,17,control_shuffle32_transform,3,0,100,9.1867429018,8704,0.947452224694,4352,2909,true,1,0.0091867429018 +fp16,1,17,1,128,17,shuffle32_transform,3,1,100,9.32365715504,8704,0.933539259892,4352,2909,true,1,0.00932365715504 +fp16,1,17,1,128,17,baseline_split,3,2,100,80.3501701355,14212,0.176875792248,4352,2909,true,1,0.0803501701355 +fp16,1,17,1,128,17,control_optimized_split,3,3,100,64.4813728333,14212,0.220404736679,4352,2909,true,1,0.0644813728333 +fp16,1,17,1,128,17,optimized_split,3,4,100,34.750854969,14212,0.408968355244,4352,2909,true,1,0.034750854969 +fp16,1,17,1,128,17,control_shuffle32_split,3,5,100,163.379898071,14212,0.0869874456269,4352,2909,true,1,0.163379898071 +fp16,1,17,1,128,17,shuffle32_split,3,6,100,45.625371933,14212,0.311493351131,4352,2909,true,1,0.045625371933 +fp16,1,17,1,128,17,baseline_fused,3,7,100,75.677485466,5508,0.0727825451134,4352,2909,true,1,0.075677485466 +fp16,1,17,1,128,17,control_optimized_fused,3,8,100,59.3440008163,5508,0.0928147735952,4352,2909,true,1,0.0593440008163 +fp16,1,17,1,128,17,optimized_fused,3,9,100,29.4045710564,5508,0.187317814956,4352,2909,true,1,0.0294045710564 +fp16,1,17,1,128,17,control_shuffle32_fused,3,10,100,157.995891571,5508,0.0348616659916,4352,2909,true,1,0.157995891571 +fp16,1,17,1,128,17,shuffle32_fused,3,11,100,39.8898267746,5508,0.138080318852,4352,2909,true,1,0.0398898267746 +fp16,1,17,1,128,17,baseline_transform,3,12,100,9.7529143095,8704,0.892451191899,4352,2909,true,1,0.0097529143095 +fp16,1,17,1,128,17,control_optimized_transform,3,13,100,9.80594336987,8704,0.887624950675,4352,2909,true,1,0.00980594336987 +fp16,1,17,1,128,17,optimized_transform,3,14,100,9.7492569685,8704,0.892785986473,4352,2909,true,1,0.0097492569685 +fp16,1,17,1,128,17,shuffle32_transform,4,0,100,9.40571427345,8704,0.925394897926,4352,2909,true,1,0.00940571427345 +fp16,1,17,1,128,17,baseline_split,4,1,100,80.4317760468,14212,0.17669633444,4352,2909,true,1,0.0804317760468 +fp16,1,17,1,128,17,control_optimized_split,4,2,100,64.5298290253,14212,0.220239232223,4352,2909,true,1,0.0645298290253 +fp16,1,17,1,128,17,optimized_split,4,3,100,34.6943998337,14212,0.40963383336,4352,2909,true,1,0.0346943998337 +fp16,1,17,1,128,17,control_shuffle32_split,4,4,100,163.405704498,14212,0.0869737078252,4352,2909,true,1,0.163405704498 +fp16,1,17,1,128,17,shuffle32_split,4,5,100,45.7593154907,14212,0.310581568968,4352,2909,true,1,0.0457593154907 +fp16,1,17,1,128,17,baseline_fused,4,6,100,75.7373714447,5508,0.072724995533,4352,2909,true,1,0.0757373714447 +fp16,1,17,1,128,17,control_optimized_fused,4,7,100,59.3574857712,5508,0.0927936877454,4352,2909,true,1,0.0593574857712 +fp16,1,17,1,128,17,optimized_fused,4,8,100,29.345600605,5508,0.187694233086,4352,2909,true,1,0.029345600605 +fp16,1,17,1,128,17,control_shuffle32_fused,4,9,100,157.987661362,5508,0.0348634820753,4352,2909,true,1,0.157987661362 +fp16,1,17,1,128,17,shuffle32_fused,4,10,100,39.9398851395,5508,0.137907256888,4352,2909,true,1,0.0399398851395 +fp16,1,17,1,128,17,baseline_transform,4,11,100,9.5995426178,8704,0.906709865933,4352,2909,true,1,0.0095995426178 +fp16,1,17,1,128,17,control_optimized_transform,4,12,100,9.81120049953,8704,0.887149335131,4352,2909,true,1,0.00981120049953 +fp16,1,17,1,128,17,optimized_transform,4,13,100,9.82994318008,8704,0.885457814002,4352,2909,true,1,0.00982994318008 +fp16,1,17,1,128,17,control_shuffle32_transform,4,14,100,9.29645717144,8704,0.93627065015,4352,2909,true,1,0.00929645717144 +fp16,1,257,1,128,257,baseline_transform,0,0,100,9.89462852478,131584,13.2985285572,65792,2909,true,1,0.00989462852478 +fp16,1,257,1,128,257,control_optimized_transform,0,1,100,10.0258278847,131584,13.1245021871,65792,2909,true,1,0.0100258278847 +fp16,1,257,1,128,257,optimized_transform,0,2,100,9.95977222919,131584,13.2115471089,65792,2909,true,1,0.00995977222919 +fp16,1,257,1,128,257,control_shuffle32_transform,0,3,100,10.5316579342,131584,12.4941391775,65792,2909,true,1,0.0105316579342 +fp16,1,257,1,128,257,shuffle32_transform,0,4,100,10.6084573269,131584,12.403688486,65792,2909,true,1,0.0106084573269 +fp16,1,257,1,128,257,baseline_split,0,5,100,81.7897129059,214852,2.62688292166,65792,2909,true,1,0.0817897129059 +fp16,1,257,1,128,257,control_optimized_split,0,6,100,65.7449150085,214852,3.26796376529,65792,2909,true,1,0.0657449150085 +fp16,1,257,1,128,257,optimized_split,0,7,100,35.321829319,214852,6.08269741806,65792,2909,true,1,0.035321829319 +fp16,1,257,1,128,257,control_shuffle32_split,0,8,100,177.890758514,214852,1.20777493892,65792,2909,true,1,0.177890758514 +fp16,1,257,1,128,257,shuffle32_split,0,9,100,51.5312004089,214852,4.16935755998,65792,2909,true,1,0.0515312004089 +fp16,1,257,1,128,257,baseline_fused,0,10,100,76.6868543625,83268,1.08581843254,65792,2909,true,1,0.0766868543625 +fp16,1,257,1,128,257,control_optimized_fused,0,11,100,60.2642297745,83268,1.38171516191,65792,2909,true,1,0.0602642297745 +fp16,1,257,1,128,257,optimized_fused,0,12,100,29.6898269653,83268,2.80459701221,65792,2909,true,1,0.0296898269653 +fp16,1,257,1,128,257,control_shuffle32_fused,0,13,100,170.371417999,83268,0.488743951174,65792,2909,true,1,0.170371417999 +fp16,1,257,1,128,257,shuffle32_fused,0,14,100,44.9081134796,83268,1.85418610465,65792,2909,true,1,0.0449081134796 +fp16,1,257,1,128,257,control_optimized_transform,1,0,100,10.0340569019,131584,13.1137386688,65792,2909,true,1,0.0100340569019 +fp16,1,257,1,128,257,optimized_transform,1,1,100,10.1126861572,131584,13.0117753042,65792,2909,true,1,0.0101126861572 +fp16,1,257,1,128,257,control_shuffle32_transform,1,2,100,10.4873132706,131584,12.5469695245,65792,2909,true,1,0.0104873132706 +fp16,1,257,1,128,257,shuffle32_transform,1,3,100,10.6221711636,131584,12.3876746076,65792,2909,true,1,0.0106221711636 +fp16,1,257,1,128,257,baseline_split,1,4,100,81.7378330231,214852,2.62855023254,65792,2909,true,1,0.0817378330231 +fp16,1,257,1,128,257,control_optimized_split,1,5,100,65.747885704,214852,3.26781610845,65792,2909,true,1,0.065747885704 +fp16,1,257,1,128,257,optimized_split,1,6,100,35.234515667,214852,6.09777077769,65792,2909,true,1,0.035234515667 +fp16,1,257,1,128,257,control_shuffle32_split,1,7,100,177.394962311,214852,1.21115051522,65792,2909,true,1,0.177394962311 +fp16,1,257,1,128,257,shuffle32_split,1,8,100,51.4898252487,214852,4.17270788864,65792,2909,true,1,0.0514898252487 +fp16,1,257,1,128,257,baseline_fused,1,9,100,76.675658226,83268,1.08597698313,65792,2909,true,1,0.076675658226 +fp16,1,257,1,128,257,control_optimized_fused,1,10,100,60.2834320068,83268,1.38127504072,65792,2909,true,1,0.0602834320068 +fp16,1,257,1,128,257,optimized_fused,1,11,100,29.7124576569,83268,2.80246087219,65792,2909,true,1,0.0297124576569 +fp16,1,257,1,128,257,control_shuffle32_fused,1,12,100,170.328445435,83268,0.488867257536,65792,2909,true,1,0.170328445435 +fp16,1,257,1,128,257,shuffle32_fused,1,13,100,44.9094867706,83268,1.85412940534,65792,2909,true,1,0.0449094867706 +fp16,1,257,1,128,257,baseline_transform,1,14,100,10.0975990295,131584,13.0312165907,65792,2909,true,1,0.0100975990295 +fp16,1,257,1,128,257,optimized_transform,2,0,100,10.050971508,131584,13.0916697848,65792,2909,true,1,0.010050971508 +fp16,1,257,1,128,257,control_shuffle32_transform,2,1,100,10.5812573433,131584,12.4355731773,65792,2909,true,1,0.0105812573433 +fp16,1,257,1,128,257,shuffle32_transform,2,2,100,10.5010282993,131584,12.5305823629,65792,2909,true,1,0.0105010282993 +fp16,1,257,1,128,257,baseline_split,2,3,100,81.6795444489,214852,2.630426032,65792,2909,true,1,0.0816795444489 +fp16,1,257,1,128,257,control_optimized_split,2,4,100,65.7741737366,214852,3.26651005698,65792,2909,true,1,0.0657741737366 +fp16,1,257,1,128,257,optimized_split,2,5,100,35.2146291733,214852,6.10121432609,65792,2909,true,1,0.0352146291733 +fp16,1,257,1,128,257,control_shuffle32_split,2,6,100,177.295074463,214852,1.21183287607,65792,2909,true,1,0.177295074463 +fp16,1,257,1,128,257,shuffle32_split,2,7,100,51.4969158173,214852,4.17213335188,65792,2909,true,1,0.0514969158173 +fp16,1,257,1,128,257,baseline_fused,2,8,100,76.6304016113,83268,1.08661834271,65792,2909,true,1,0.0766304016113 +fp16,1,257,1,128,257,control_optimized_fused,2,9,100,60.2367973328,83268,1.38234440885,65792,2909,true,1,0.0602367973328 +fp16,1,257,1,128,257,optimized_fused,2,10,100,29.6799993515,83268,2.80552566777,65792,2909,true,1,0.0296799993515 +fp16,1,257,1,128,257,control_shuffle32_fused,2,11,100,170.358409882,83268,0.488781270369,65792,2909,true,1,0.170358409882 +fp16,1,257,1,128,257,shuffle32_fused,2,12,100,44.9099445343,83268,1.85411050634,65792,2909,true,1,0.0449099445343 +fp16,1,257,1,128,257,baseline_transform,2,13,100,9.99657094479,131584,13.1629136358,65792,2909,true,1,0.00999657094479 +fp16,1,257,1,128,257,control_optimized_transform,2,14,100,10.0438857079,131584,13.1009057478,65792,2909,true,1,0.0100438857079 +fp16,1,257,1,128,257,control_shuffle32_transform,3,0,100,10.6063997746,131584,12.4060946973,65792,2909,true,1,0.0106063997746 +fp16,1,257,1,128,257,shuffle32_transform,3,1,100,10.6395435333,131584,12.3674478692,65792,2909,true,1,0.0106395435333 +fp16,1,257,1,128,257,baseline_split,3,2,100,81.7812633514,214852,2.62715432845,65792,2909,true,1,0.0817812633514 +fp16,1,257,1,128,257,control_optimized_split,3,3,100,65.7341718674,214852,3.26849785882,65792,2909,true,1,0.0657341718674 +fp16,1,257,1,128,257,optimized_split,3,4,100,35.1792001724,214852,6.10735886396,65792,2909,true,1,0.0351792001724 +fp16,1,257,1,128,257,control_shuffle32_split,3,5,100,177.274742126,214852,1.21197186595,65792,2909,true,1,0.177274742126 +fp16,1,257,1,128,257,shuffle32_split,3,6,100,51.5602302551,214852,4.16701009551,65792,2909,true,1,0.0515602302551 +fp16,1,257,1,128,257,baseline_fused,3,7,100,76.6820573807,83268,1.08588635783,65792,2909,true,1,0.0766820573807 +fp16,1,257,1,128,257,control_optimized_fused,3,8,100,60.2619457245,83268,1.38176753171,65792,2909,true,1,0.0602619457245 +fp16,1,257,1,128,257,optimized_fused,3,9,100,29.6765708923,83268,2.80584978305,65792,2909,true,1,0.0296765708923 +fp16,1,257,1,128,257,control_shuffle32_fused,3,10,100,170.365028381,83268,0.488762281738,65792,2909,true,1,0.170365028381 +fp16,1,257,1,128,257,shuffle32_fused,3,11,100,44.9110841751,83268,1.85406345737,65792,2909,true,1,0.0449110841751 +fp16,1,257,1,128,257,baseline_transform,3,12,100,9.96959984303,131584,13.1985237193,65792,2909,true,1,0.00996959984303 +fp16,1,257,1,128,257,control_optimized_transform,3,13,100,10.0244569778,131584,13.1262970444,65792,2909,true,1,0.0100244569778 +fp16,1,257,1,128,257,optimized_transform,3,14,100,10.0052571297,131584,13.1514860932,65792,2909,true,1,0.0100052571297 +fp16,1,257,1,128,257,shuffle32_transform,4,0,100,10.5510866642,131584,12.4711325182,65792,2909,true,1,0.0105510866642 +fp16,1,257,1,128,257,baseline_split,4,1,100,81.7088031769,214852,2.62948411489,65792,2909,true,1,0.0817088031769 +fp16,1,257,1,128,257,control_optimized_split,4,2,100,65.703086853,214852,3.27004422913,65792,2909,true,1,0.065703086853 +fp16,1,257,1,128,257,optimized_split,4,3,100,35.2761149406,214852,6.09057999617,65792,2909,true,1,0.0352761149406 +fp16,1,257,1,128,257,control_shuffle32_split,4,4,100,177.282276154,214852,1.21192036035,65792,2909,true,1,0.177282276154 +fp16,1,257,1,128,257,shuffle32_split,4,5,100,51.5268564224,214852,4.16970905888,65792,2909,true,1,0.0515268564224 +fp16,1,257,1,128,257,baseline_fused,4,6,100,76.6832017899,83268,1.08587015222,65792,2909,true,1,0.0766832017899 +fp16,1,257,1,128,257,control_optimized_fused,4,7,100,60.2617168427,83268,1.38177277985,65792,2909,true,1,0.0602617168427 +fp16,1,257,1,128,257,optimized_fused,4,8,100,29.6715426445,83268,2.80632527259,65792,2909,true,1,0.0296715426445 +fp16,1,257,1,128,257,control_shuffle32_fused,4,9,100,170.361824036,83268,0.488771474897,65792,2909,true,1,0.170361824036 +fp16,1,257,1,128,257,shuffle32_fused,4,10,100,44.9156570435,83268,1.85387469495,65792,2909,true,1,0.0449156570435 +fp16,1,257,1,128,257,baseline_transform,4,11,100,9.93417084217,131584,13.245594634,65792,2909,true,1,0.00993417084217 +fp16,1,257,1,128,257,control_optimized_transform,4,12,100,9.95222806931,131584,13.2215619541,65792,2909,true,1,0.00995222806931 +fp16,1,257,1,128,257,optimized_transform,4,13,100,9.96868610382,131584,13.1997335085,65792,2909,true,1,0.00996868610382 +fp16,1,257,1,128,257,control_shuffle32_transform,4,14,100,10.5104005337,131584,12.5194087112,65792,2909,true,1,0.0105104005337 +fp16,4,64,16,128,4096,baseline_transform,0,0,100,25.8450269699,2097152,81.1433473233,1048576,2909,true,1,0.0258450269699 +fp16,4,64,16,128,4096,control_optimized_transform,0,1,100,27.756114006,2097152,75.5563981162,1048576,2909,true,1,0.027756114006 +fp16,4,64,16,128,4096,optimized_transform,0,2,100,27.8436589241,2097152,75.3188367131,1048576,2909,true,1,0.0278436589241 +fp16,4,64,16,128,4096,control_shuffle32_transform,0,3,100,15.6724572182,2097152,133.811308004,1048576,2909,true,1,0.0156724572182 +fp16,4,64,16,128,4096,shuffle32_transform,0,4,100,15.9638857841,2097152,131.368516936,1048576,2909,true,1,0.0159638857841 +fp16,4,64,16,128,4096,baseline_split,0,5,100,526.315193176,3424256,6.50609377118,1048576,2909,true,1,0.526315193176 +fp16,4,64,16,128,4096,control_optimized_split,0,6,100,403.751754761,3424256,8.48109255161,1048576,2909,true,1,0.403751754761 +fp16,4,64,16,128,4096,optimized_split,0,7,100,168.871765137,3424256,20.2772559239,1048576,2909,true,1,0.168871765137 +fp16,4,64,16,128,4096,control_shuffle32_split,0,8,100,455.617828369,3424256,7.51563215219,1048576,2909,true,1,0.455617828369 +fp16,4,64,16,128,4096,shuffle32_split,0,9,100,105.843887329,3424256,32.3519485764,1048576,2909,true,1,0.105843887329 +fp16,4,64,16,128,4096,baseline_fused,0,10,100,529.039993286,1327104,2.50851356578,1048576,2909,true,1,0.529039993286 +fp16,4,64,16,128,4096,control_optimized_fused,0,11,100,392.894630432,1327104,3.37776059332,1048576,2909,true,1,0.392894630432 +fp16,4,64,16,128,4096,optimized_fused,0,12,100,153.549947739,1327104,8.64281635744,1048576,2909,true,1,0.153549947739 +fp16,4,64,16,128,4096,control_shuffle32_fused,0,13,100,442.181282043,1327104,3.00126679688,1048576,2909,true,1,0.442181282043 +fp16,4,64,16,128,4096,shuffle32_fused,0,14,100,92.9135990143,1327104,14.2832051936,1048576,2909,true,1,0.0929135990143 +fp16,4,64,16,128,4096,control_optimized_transform,1,0,100,27.7862858772,2097152,75.4743548406,1048576,2909,true,1,0.0277862858772 +fp16,4,64,16,128,4096,optimized_transform,1,1,100,27.8672003746,2097152,75.255209415,1048576,2909,true,1,0.0278672003746 +fp16,4,64,16,128,4096,control_shuffle32_transform,1,2,100,15.8386278152,2097152,132.407429764,1048576,2909,true,1,0.0158386278152 +fp16,4,64,16,128,4096,shuffle32_transform,1,3,100,15.7469713688,2097152,133.178117295,1048576,2909,true,1,0.0157469713688 +fp16,4,64,16,128,4096,baseline_split,1,4,100,525.186271667,3424256,6.5200790362,1048576,2909,true,1,0.525186271667 +fp16,4,64,16,128,4096,control_optimized_split,1,5,100,403.101463318,3424256,8.49477442184,1048576,2909,true,1,0.403101463318 +fp16,4,64,16,128,4096,optimized_split,1,6,100,168.776016235,3424256,20.2887594836,1048576,2909,true,1,0.168776016235 +fp16,4,64,16,128,4096,control_shuffle32_split,1,7,100,453.146057129,3424256,7.5566275953,1048576,2909,true,1,0.453146057129 +fp16,4,64,16,128,4096,shuffle32_split,1,8,100,105.780563354,3424256,32.3713155934,1048576,2909,true,1,0.105780563354 +fp16,4,64,16,128,4096,baseline_fused,1,9,100,528.998184204,1327104,2.50871182478,1048576,2909,true,1,0.528998184204 +fp16,4,64,16,128,4096,control_optimized_fused,1,10,100,394.618530273,1327104,3.36300477091,1048576,2909,true,1,0.394618530273 +fp16,4,64,16,128,4096,optimized_fused,1,11,100,153.887996674,1327104,8.62383050456,1048576,2909,true,1,0.153887996674 +fp16,4,64,16,128,4096,control_shuffle32_fused,1,12,100,442.458724976,1327104,2.99938485804,1048576,2909,true,1,0.442458724976 +fp16,4,64,16,128,4096,shuffle32_fused,1,13,100,92.8921127319,1327104,14.2865089508,1048576,2909,true,1,0.0928921127319 +fp16,4,64,16,128,4096,baseline_transform,1,14,100,25.9055995941,2097152,80.9536174749,1048576,2909,true,1,0.0259055995941 +fp16,4,64,16,128,4096,optimized_transform,2,0,100,27.8306293488,2097152,75.3540990295,1048576,2909,true,1,0.0278306293488 +fp16,4,64,16,128,4096,control_shuffle32_transform,2,1,100,15.7714283466,2097152,132.971596098,1048576,2909,true,1,0.0157714283466 +fp16,4,64,16,128,4096,shuffle32_transform,2,2,100,15.728456974,2097152,133.334884882,1048576,2909,true,1,0.015728456974 +fp16,4,64,16,128,4096,baseline_split,2,3,100,525.949707031,3424256,6.51061490143,1048576,2909,true,1,0.525949707031 +fp16,4,64,16,128,4096,control_optimized_split,2,4,100,402.319984436,3424256,8.51127493654,1048576,2909,true,1,0.402319984436 +fp16,4,64,16,128,4096,optimized_split,2,5,100,168.645019531,3424256,20.3045189803,1048576,2909,true,1,0.168645019531 +fp16,4,64,16,128,4096,control_shuffle32_split,2,6,100,454.620361328,3424256,7.53212194455,1048576,2909,true,1,0.454620361328 +fp16,4,64,16,128,4096,shuffle32_split,2,7,100,105.927085876,3424256,32.3265383133,1048576,2909,true,1,0.105927085876 +fp16,4,64,16,128,4096,baseline_fused,2,8,100,528.942642212,1327104,2.50897525382,1048576,2909,true,1,0.528942642212 +fp16,4,64,16,128,4096,control_optimized_fused,2,9,100,392.323226929,1327104,3.3826801701,1048576,2909,true,1,0.392323226929 +fp16,4,64,16,128,4096,optimized_fused,2,10,100,153.514976501,1327104,8.6447852206,1048576,2909,true,1,0.153514976501 +fp16,4,64,16,128,4096,control_shuffle32_fused,2,11,100,439.834060669,1327104,3.01728337724,1048576,2909,true,1,0.439834060669 +fp16,4,64,16,128,4096,shuffle32_fused,2,12,100,92.9145145416,1327104,14.283064455,1048576,2909,true,1,0.0929145145416 +fp16,4,64,16,128,4096,baseline_transform,2,13,100,25.8173727989,2097152,81.2302636807,1048576,2909,true,1,0.0258173727989 +fp16,4,64,16,128,4096,control_optimized_transform,2,14,100,27.7606868744,2097152,75.5439521179,1048576,2909,true,1,0.0277606868744 +fp16,4,64,16,128,4096,control_shuffle32_transform,3,0,100,15.7583999634,2097152,133.081531429,1048576,2909,true,1,0.0157583999634 +fp16,4,64,16,128,4096,shuffle32_transform,3,1,100,15.7099425793,2097152,133.492021974,1048576,2909,true,1,0.0157099425793 +fp16,4,64,16,128,4096,baseline_split,3,2,100,526.333694458,3424256,6.50586507392,1048576,2909,true,1,0.526333694458 +fp16,4,64,16,128,4096,control_optimized_split,3,3,100,404.876098633,3424256,8.45754049588,1048576,2909,true,1,0.404876098633 +fp16,4,64,16,128,4096,optimized_split,3,4,100,169.062862396,3424256,20.2543358811,1048576,2909,true,1,0.169062862396 +fp16,4,64,16,128,4096,control_shuffle32_split,3,5,100,455.95703125,3424256,7.51004100236,1048576,2909,true,1,0.45595703125 +fp16,4,64,16,128,4096,shuffle32_split,3,6,100,105.362062454,3424256,32.4998953156,1048576,2909,true,1,0.105362062454 +fp16,4,64,16,128,4096,baseline_fused,3,7,100,528.957023621,1327104,2.50890703921,1048576,2909,true,1,0.528957023621 +fp16,4,64,16,128,4096,control_optimized_fused,3,8,100,392.365264893,1327104,3.38231775018,1048576,2909,true,1,0.392365264893 +fp16,4,64,16,128,4096,optimized_fused,3,9,100,152.939424515,1327104,8.67731786105,1048576,2909,true,1,0.152939424515 +fp16,4,64,16,128,4096,control_shuffle32_fused,3,10,100,442.117271423,1327104,3.00170132627,1048576,2909,true,1,0.442117271423 +fp16,4,64,16,128,4096,shuffle32_fused,3,11,100,92.9135990143,1327104,14.2832051936,1048576,2909,true,1,0.0929135990143 +fp16,4,64,16,128,4096,baseline_transform,3,12,100,25.8777141571,2097152,81.04085188,1048576,2909,true,1,0.0258777141571 +fp16,4,64,16,128,4096,control_optimized_transform,3,13,100,27.7021718025,2097152,75.7035229927,1048576,2909,true,1,0.0277021718025 +fp16,4,64,16,128,4096,optimized_transform,3,14,100,27.8041148186,2097152,75.4259581247,1048576,2909,true,1,0.0278041148186 +fp16,4,64,16,128,4096,shuffle32_transform,4,0,100,15.8395433426,2097152,132.3997766,1048576,2909,true,1,0.0158395433426 +fp16,4,64,16,128,4096,baseline_split,4,1,100,524.728012085,3424256,6.52577320276,1048576,2909,true,1,0.524728012085 +fp16,4,64,16,128,4096,control_optimized_split,4,2,100,402.603187561,3424256,8.50528785116,1048576,2909,true,1,0.402603187561 +fp16,4,64,16,128,4096,optimized_split,4,3,100,169.15725708,3424256,20.2430333709,1048576,2909,true,1,0.16915725708 +fp16,4,64,16,128,4096,control_shuffle32_split,4,4,100,454.522285461,3424256,7.53374721005,1048576,2909,true,1,0.454522285461 +fp16,4,64,16,128,4096,shuffle32_split,4,5,100,105.810747147,3424256,32.3620812851,1048576,2909,true,1,0.105810747147 +fp16,4,64,16,128,4096,baseline_fused,4,6,100,528.908576965,1327104,2.50913684859,1048576,2909,true,1,0.528908576965 +fp16,4,64,16,128,4096,control_optimized_fused,4,7,100,394.11151886,1327104,3.36733116515,1048576,2909,true,1,0.39411151886 +fp16,4,64,16,128,4096,optimized_fused,4,8,100,154.181251526,1327104,8.60742786082,1048576,2909,true,1,0.154181251526 +fp16,4,64,16,128,4096,control_shuffle32_fused,4,9,100,441.340370178,1327104,3.00698528771,1048576,2909,true,1,0.441340370178 +fp16,4,64,16,128,4096,shuffle32_fused,4,10,100,92.944688797,1327104,14.2784274946,1048576,2909,true,1,0.092944688797 +fp16,4,64,16,128,4096,baseline_transform,4,11,100,25.8157706261,2097152,81.235304976,1048576,2909,true,1,0.0258157706261 +fp16,4,64,16,128,4096,control_optimized_transform,4,12,100,27.7222847939,2097152,75.6485987932,1048576,2909,true,1,0.0277222847939 +fp16,4,64,16,128,4096,optimized_transform,4,13,100,27.7707433701,2097152,75.5165957229,1048576,2909,true,1,0.0277707433701 +fp16,4,64,16,128,4096,control_shuffle32_transform,4,14,100,15.6607997417,2097152,133.910913528,1048576,2909,true,1,0.0156607997417 +fp16,16,64,16,128,16384,baseline_transform,0,0,100,78.8420534134,8388608,106.397634724,4194304,2909,true,1,0.0788420534134 +fp16,16,64,16,128,16384,control_optimized_transform,0,1,100,87.0630931854,8388608,96.3509070616,4194304,2909,true,1,0.0870630931854 +fp16,16,64,16,128,16384,optimized_transform,0,2,100,87.1940612793,8388608,96.2061851108,4194304,2909,true,1,0.0871940612793 +fp16,16,64,16,128,16384,control_shuffle32_transform,0,3,100,26.7355418205,8388608,313.762408718,4194304,2909,true,1,0.0267355418205 +fp16,16,64,16,128,16384,shuffle32_transform,0,4,100,26.716799736,8388608,313.982515978,4194304,2909,true,1,0.026716799736 +fp16,16,64,16,128,16384,baseline_split,0,5,100,1884.15924072,13697024,7.26956814688,4194304,2909,true,1,1.88415924072 +fp16,16,64,16,128,16384,control_optimized_split,0,6,100,1453.91494751,13697024,9.42078766262,4194304,2909,true,1,1.45391494751 +fp16,16,64,16,128,16384,optimized_split,0,7,100,601.828079224,13697024,22.7590311467,4194304,2909,true,1,0.601828079224 +fp16,16,64,16,128,16384,control_shuffle32_split,0,8,100,1601.22421265,13697024,8.55409498047,4194304,2909,true,1,1.60122421265 +fp16,16,64,16,128,16384,shuffle32_split,0,9,100,288.731651306,13697024,47.4385954503,4194304,2909,true,1,0.288731651306 +fp16,16,64,16,128,16384,baseline_fused,0,10,100,1865.20965576,5308416,2.84601571925,4194304,2909,true,1,1.86520965576 +fp16,16,64,16,128,16384,control_optimized_fused,0,11,100,1421.51611328,5308416,3.7343340328,4194304,2909,true,1,1.42151611328 +fp16,16,64,16,128,16384,optimized_fused,0,12,100,566.117515564,5308416,9.3768799835,4194304,2909,true,1,0.566117515564 +fp16,16,64,16,128,16384,control_shuffle32_fused,0,13,100,1574.12582397,5308416,3.37229458989,4194304,2909,true,1,1.57412582397 +fp16,16,64,16,128,16384,shuffle32_fused,0,14,100,265.220565796,5308416,20.0150994478,4194304,2909,true,1,0.265220565796 +fp16,16,64,16,128,16384,control_optimized_transform,1,0,100,87.0571517944,8388608,96.3574827236,4194304,2909,true,1,0.0870571517944 +fp16,16,64,16,128,16384,optimized_transform,1,1,100,87.1263980865,8388608,96.2808997529,4194304,2909,true,1,0.0871263980865 +fp16,16,64,16,128,16384,control_shuffle32_transform,1,2,100,26.7405724525,8388608,313.70338144,4194304,2909,true,1,0.0267405724525 +fp16,16,64,16,128,16384,shuffle32_transform,1,3,100,26.7330288887,8388608,313.791902703,4194304,2909,true,1,0.0267330288887 +fp16,16,64,16,128,16384,baseline_split,1,4,100,1884.38018799,13697024,7.26871577578,4194304,2909,true,1,1.88438018799 +fp16,16,64,16,128,16384,control_optimized_split,1,5,100,1453.88244629,13697024,9.42099826225,4194304,2909,true,1,1.45388244629 +fp16,16,64,16,128,16384,optimized_split,1,6,100,601.815986633,13697024,22.7594884553,4194304,2909,true,1,0.601815986633 +fp16,16,64,16,128,16384,control_shuffle32_split,1,7,100,1601.42074585,13697024,8.55304518534,4194304,2909,true,1,1.60142074585 +fp16,16,64,16,128,16384,shuffle32_split,1,8,100,288.733711243,13697024,47.4382570052,4194304,2909,true,1,0.288733711243 +fp16,16,64,16,128,16384,baseline_fused,1,9,100,1865.20492554,5308416,2.84602293685,4194304,2909,true,1,1.86520492554 +fp16,16,64,16,128,16384,control_optimized_fused,1,10,100,1421.34918213,5308416,3.73477261376,4194304,2909,true,1,1.42134918213 +fp16,16,64,16,128,16384,optimized_fused,1,11,100,566.189460754,5308416,9.37568847171,4194304,2909,true,1,0.566189460754 +fp16,16,64,16,128,16384,control_shuffle32_fused,1,12,100,1574.36035156,5308416,3.37179222961,4194304,2909,true,1,1.57436035156 +fp16,16,64,16,128,16384,shuffle32_fused,1,13,100,265.238170624,5308416,20.0137709724,4194304,2909,true,1,0.265238170624 +fp16,16,64,16,128,16384,baseline_transform,1,14,100,78.8550806046,8388608,106.380057387,4194304,2909,true,1,0.0788550806046 +fp16,16,64,16,128,16384,optimized_transform,2,0,100,87.1940612793,8388608,96.2061851108,4194304,2909,true,1,0.0871940612793 +fp16,16,64,16,128,16384,control_shuffle32_transform,2,1,100,26.7026305199,8388608,314.149124513,4194304,2909,true,1,0.0267026305199 +fp16,16,64,16,128,16384,shuffle32_transform,2,2,100,26.6662859917,8388608,314.577290689,4194304,2909,true,1,0.0266662859917 +fp16,16,64,16,128,16384,baseline_split,2,3,100,1884.2678833,13697024,7.2691490002,4194304,2909,true,1,1.8842678833 +fp16,16,64,16,128,16384,control_optimized_split,2,4,100,1453.53530884,13697024,9.42324821194,4194304,2909,true,1,1.45353530884 +fp16,16,64,16,128,16384,optimized_split,2,5,100,601.761360168,13697024,22.7615545075,4194304,2909,true,1,0.601761360168 +fp16,16,64,16,128,16384,control_shuffle32_split,2,6,100,1601.63864136,13697024,8.55188158322,4194304,2909,true,1,1.60163864136 +fp16,16,64,16,128,16384,shuffle32_split,2,7,100,288.859882355,13697024,47.4175364483,4194304,2909,true,1,0.288859882355 +fp16,16,64,16,128,16384,baseline_fused,2,8,100,1865.07659912,5308416,2.84621875718,4194304,2909,true,1,1.86507659912 +fp16,16,64,16,128,16384,control_optimized_fused,2,9,100,1421.15905762,5308416,3.73527225651,4194304,2909,true,1,1.42115905762 +fp16,16,64,16,128,16384,optimized_fused,2,10,100,566.239089966,5308416,9.37486671985,4194304,2909,true,1,0.566239089966 +fp16,16,64,16,128,16384,control_shuffle32_fused,2,11,100,1574.43099976,5308416,3.37164092985,4194304,2909,true,1,1.57443099976 +fp16,16,64,16,128,16384,shuffle32_fused,2,12,100,265.22195816,5308416,20.0149943723,4194304,2909,true,1,0.26522195816 +fp16,16,64,16,128,16384,baseline_transform,2,13,100,78.8464021683,8388608,106.391766388,4194304,2909,true,1,0.0788464021683 +fp16,16,64,16,128,16384,control_optimized_transform,2,14,100,87.0847988129,8388608,96.3268918842,4194304,2909,true,1,0.0870847988129 +fp16,16,64,16,128,16384,control_shuffle32_transform,3,0,100,26.7460560799,8388608,313.639064203,4194304,2909,true,1,0.0267460560799 +fp16,16,64,16,128,16384,shuffle32_transform,3,1,100,26.7261719704,8388608,313.87240976,4194304,2909,true,1,0.0267261719704 +fp16,16,64,16,128,16384,baseline_split,3,2,100,1884.30221558,13697024,7.26901655519,4194304,2909,true,1,1.88430221558 +fp16,16,64,16,128,16384,control_optimized_split,3,3,100,1453.66989136,13697024,9.42237579621,4194304,2909,true,1,1.45366989136 +fp16,16,64,16,128,16384,optimized_split,3,4,100,601.850318909,13697024,22.7581901507,4194304,2909,true,1,0.601850318909 +fp16,16,64,16,128,16384,control_shuffle32_split,3,5,100,1602.64068604,13697024,8.54653455347,4194304,2909,true,1,1.60264068604 +fp16,16,64,16,128,16384,shuffle32_split,3,6,100,288.709945679,13697024,47.4421619519,4194304,2909,true,1,0.288709945679 +fp16,16,64,16,128,16384,baseline_fused,3,7,100,1865.23025513,5308416,2.84598428822,4194304,2909,true,1,1.86523025513 +fp16,16,64,16,128,16384,control_optimized_fused,3,8,100,1421.64230347,5308416,3.73400255961,4194304,2909,true,1,1.42164230347 +fp16,16,64,16,128,16384,optimized_fused,3,9,100,566.145133972,5308416,9.37642254868,4194304,2909,true,1,0.566145133972 +fp16,16,64,16,128,16384,control_shuffle32_fused,3,10,100,1574.00436401,5308416,3.37255481711,4194304,2909,true,1,1.57400436401 +fp16,16,64,16,128,16384,shuffle32_fused,3,11,100,265.229034424,5308416,20.0144603758,4194304,2909,true,1,0.265229034424 +fp16,16,64,16,128,16384,baseline_transform,3,12,100,78.8665103912,8388608,106.36464018,4194304,2909,true,1,0.0788665103912 +fp16,16,64,16,128,16384,control_optimized_transform,3,13,100,87.1311950684,8388608,96.2755990368,4194304,2909,true,1,0.0871311950684 +fp16,16,64,16,128,16384,optimized_transform,3,14,100,87.2233104706,8388608,96.1739236305,4194304,2909,true,1,0.0872233104706 +fp16,16,64,16,128,16384,shuffle32_transform,4,0,100,26.6923427582,8388608,314.270203856,4194304,2909,true,1,0.0266923427582 +fp16,16,64,16,128,16384,baseline_split,4,1,100,1884.2616272,13697024,7.26917313514,4194304,2909,true,1,1.8842616272 +fp16,16,64,16,128,16384,control_optimized_split,4,2,100,1453.92044067,13697024,9.42075206925,4194304,2909,true,1,1.45392044067 +fp16,16,64,16,128,16384,optimized_split,4,3,100,601.75037384,13697024,22.7619700717,4194304,2909,true,1,0.60175037384 +fp16,16,64,16,128,16384,control_shuffle32_split,4,4,100,1601.26754761,13697024,8.55386348176,4194304,2909,true,1,1.60126754761 +fp16,16,64,16,128,16384,shuffle32_split,4,5,100,288.635196686,13697024,47.4544482353,4194304,2909,true,1,0.288635196686 +fp16,16,64,16,128,16384,baseline_fused,4,6,100,1865.20187378,5308416,2.84602759338,4194304,2909,true,1,1.86520187378 +fp16,16,64,16,128,16384,control_optimized_fused,4,7,100,1421.59301758,5308416,3.73413201554,4194304,2909,true,1,1.42159301758 +fp16,16,64,16,128,16384,optimized_fused,4,8,100,566.07749939,5308416,9.37754283773,4194304,2909,true,1,0.56607749939 +fp16,16,64,16,128,16384,control_shuffle32_fused,4,9,100,1574.28634644,5308416,3.37195073312,4194304,2909,true,1,1.57428634644 +fp16,16,64,16,128,16384,shuffle32_fused,4,10,100,265.228347778,5308416,20.014512191,4194304,2909,true,1,0.265228347778 +fp16,16,64,16,128,16384,baseline_transform,4,11,100,78.8029766083,8388608,106.450395163,4194304,2909,true,1,0.0788029766083 +fp16,16,64,16,128,16384,control_optimized_transform,4,12,100,87.1195411682,8388608,96.2884777343,4194304,2909,true,1,0.0871195411682 +fp16,16,64,16,128,16384,optimized_transform,4,13,100,87.1664047241,8388608,96.2367098488,4194304,2909,true,1,0.0871664047241 +fp16,16,64,16,128,16384,control_shuffle32_transform,4,14,100,26.7485713959,8388608,313.609570988,4194304,2909,true,1,0.0267485713959 +fp16,1,1,1,256,1,baseline_transform,0,0,100,9.8859423399,1024,0.103581425502,512,2909,true,1,0.0098859423399 +fp16,1,1,1,256,1,control_optimized_transform,0,1,100,10.5714285374,1024,0.096864865177,512,2909,true,1,0.0105714285374 +fp16,1,1,1,256,1,optimized_transform,0,2,100,10.6315433979,1024,0.0963171537448,512,2909,true,1,0.0106315433979 +fp16,1,1,1,256,1,control_shuffle32_transform,0,3,100,11.4877724648,1024,0.0891382557534,512,2909,true,1,0.0114877724648 +fp16,1,1,1,256,1,shuffle32_transform,0,4,100,11.5364563465,1024,0.0887620920361,512,2909,true,1,0.0115364563465 +fp16,1,1,1,256,1,baseline_split,0,5,100,100.666742325,1668,0.0165695239707,512,2909,true,1,0.100666742325 +fp16,1,1,1,256,1,control_optimized_split,0,6,100,66.9250249863,1668,0.0249234124357,512,2909,true,1,0.0669250249863 +fp16,1,1,1,256,1,optimized_split,0,7,100,35.4873132706,1668,0.0470027129775,512,2909,true,1,0.0354873132706 +fp16,1,1,1,256,1,control_shuffle32_split,0,8,100,303.295783997,1668,0.00549958188677,512,2909,true,1,0.303295783997 +fp16,1,1,1,256,1,shuffle32_split,0,9,100,67.7830886841,1668,0.0246079078481,512,2909,true,1,0.0677830886841 +fp16,1,1,1,256,1,baseline_fused,0,10,100,96.1403465271,644,0.00669854044908,512,2909,true,1,0.0961403465271 +fp16,1,1,1,256,1,control_optimized_fused,0,11,100,61.299200058,644,0.0105058467222,512,2909,true,1,0.061299200058 +fp16,1,1,1,256,1,optimized_fused,0,12,100,29.7444558144,644,0.0216510937036,512,2909,true,1,0.0297444558144 +fp16,1,1,1,256,1,control_shuffle32_fused,0,13,100,299.446392059,644,0.00215063536271,512,2909,true,1,0.299446392059 +fp16,1,1,1,256,1,shuffle32_fused,0,14,100,61.6290283203,644,0.0104496211859,512,2909,true,1,0.0616290283203 +fp16,1,1,1,256,1,control_optimized_transform,1,0,100,10.6178283691,1024,0.0964415664295,512,2909,true,1,0.0106178283691 +fp16,1,1,1,256,1,optimized_transform,1,1,100,10.6822860241,1024,0.0958596313271,512,2909,true,1,0.0106822860241 +fp16,1,1,1,256,1,control_shuffle32_transform,1,2,100,11.5101718903,1024,0.088964787821,512,2909,true,1,0.0115101718903 +fp16,1,1,1,256,1,shuffle32_transform,1,3,100,11.6057145596,1024,0.0882323957517,512,2909,true,1,0.0116057145596 +fp16,1,1,1,256,1,baseline_split,1,4,100,100.654859543,1668,0.0165714800813,512,2909,true,1,0.100654859543 +fp16,1,1,1,256,1,control_optimized_split,1,5,100,66.9280004501,1668,0.0249223043985,512,2909,true,1,0.0669280004501 +fp16,1,1,1,256,1,optimized_split,1,6,100,35.4843425751,1668,0.0470066479736,512,2909,true,1,0.0354843425751 +fp16,1,1,1,256,1,control_shuffle32_split,1,7,100,302.654628754,1668,0.00551123241323,512,2909,true,1,0.302654628754 +fp16,1,1,1,256,1,shuffle32_split,1,8,100,67.7424001694,1668,0.02462268824,512,2909,true,1,0.0677424001694 +fp16,1,1,1,256,1,baseline_fused,1,9,100,96.1412620544,644,0.00669847666068,512,2909,true,1,0.0961412620544 +fp16,1,1,1,256,1,control_optimized_fused,1,10,100,61.2425136566,644,0.0105155709906,512,2909,true,1,0.0612425136566 +fp16,1,1,1,256,1,optimized_fused,1,11,100,29.677028656,644,0.021700285681,512,2909,true,1,0.029677028656 +fp16,1,1,1,256,1,control_shuffle32_fused,1,12,100,299.443664551,644,0.00215065495196,512,2909,true,1,0.299443664551 +fp16,1,1,1,256,1,shuffle32_fused,1,13,100,61.6292572021,644,0.0104495823775,512,2909,true,1,0.0616292572021 +fp16,1,1,1,256,1,baseline_transform,1,14,100,9.78148519993,1024,0.104687578529,512,2909,true,1,0.00978148519993 +fp16,1,1,1,256,1,optimized_transform,2,0,100,10.6249141693,1024,0.0963772491412,512,2909,true,1,0.0106249141693 +fp16,1,1,1,256,1,control_shuffle32_transform,2,1,100,11.4756572247,1024,0.0892323620298,512,2909,true,1,0.0114756572247 +fp16,1,1,1,256,1,shuffle32_transform,2,2,100,11.6038858891,1024,0.0882463004024,512,2909,true,1,0.0116038858891 +fp16,1,1,1,256,1,baseline_split,2,3,100,100.578975677,1668,0.0165839827734,512,2909,true,1,0.100578975677 +fp16,1,1,1,256,1,control_optimized_split,2,4,100,66.9517707825,1668,0.024913456067,512,2909,true,1,0.0669517707825 +fp16,1,1,1,256,1,optimized_split,2,5,100,35.5284571648,1668,0.0469482812683,512,2909,true,1,0.0355284571648 +fp16,1,1,1,256,1,control_shuffle32_split,2,6,100,302.691879272,1668,0.00551055417809,512,2909,true,1,0.302691879272 +fp16,1,1,1,256,1,shuffle32_split,2,7,100,67.7332544327,1668,0.0246260129381,512,2909,true,1,0.0677332544327 +fp16,1,1,1,256,1,baseline_fused,2,8,100,96.0962295532,644,0.00670161569288,512,2909,true,1,0.0960962295532 +fp16,1,1,1,256,1,control_optimized_fused,2,9,100,61.2895965576,644,0.010507492889,512,2909,true,1,0.0612895965576 +fp16,1,1,1,256,1,optimized_fused,2,10,100,29.7250294685,644,0.0216652434502,512,2909,true,1,0.0297250294685 +fp16,1,1,1,256,1,control_shuffle32_fused,2,11,100,299.450969696,644,0.00215060248646,512,2909,true,1,0.299450969696 +fp16,1,1,1,256,1,shuffle32_fused,2,12,100,61.6329145432,644,0.0104489622919,512,2909,true,1,0.0616329145432 +fp16,1,1,1,256,1,baseline_transform,2,13,100,9.87611472607,1024,0.103684498247,512,2909,true,1,0.00987611472607 +fp16,1,1,1,256,1,control_optimized_transform,2,14,100,10.5538284779,1024,0.0970264015706,512,2909,true,1,0.0105538284779 +fp16,1,1,1,256,1,control_shuffle32_transform,3,0,100,11.4706289768,1024,0.0892714777951,512,2909,true,1,0.0114706289768 +fp16,1,1,1,256,1,shuffle32_transform,3,1,100,11.6114282608,1024,0.0881889787372,512,2909,true,1,0.0116114282608 +fp16,1,1,1,256,1,baseline_split,3,2,100,100.593137741,1668,0.0165816479877,512,2909,true,1,0.100593137741 +fp16,1,1,1,256,1,control_optimized_split,3,3,100,67.0393133163,1668,0.0248809231104,512,2909,true,1,0.0670393133163 +fp16,1,1,1,256,1,optimized_split,3,4,100,35.5771446228,1668,0.0468840323664,512,2909,true,1,0.0355771446228 +fp16,1,1,1,256,1,control_shuffle32_split,3,5,100,303.307437897,1668,0.00549937057781,512,2909,true,1,0.303307437897 +fp16,1,1,1,256,1,shuffle32_split,3,6,100,67.7609157562,1668,0.0246159601207,512,2909,true,1,0.0677609157562 +fp16,1,1,1,256,1,baseline_fused,3,7,100,96.1408042908,644,0.00669850855473,512,2909,true,1,0.0961408042908 +fp16,1,1,1,256,1,control_optimized_fused,3,8,100,61.2946271896,644,0.0105066305079,512,2909,true,1,0.0612946271896 +fp16,1,1,1,256,1,optimized_fused,3,9,100,29.7222876549,644,0.0216672420198,512,2909,true,1,0.0297222876549 +fp16,1,1,1,256,1,control_shuffle32_fused,3,10,100,299.439086914,644,0.00215068782983,512,2909,true,1,0.299439086914 +fp16,1,1,1,256,1,shuffle32_fused,3,11,100,61.624455452,644,0.0104503966043,512,2909,true,1,0.061624455452 +fp16,1,1,1,256,1,baseline_transform,3,12,100,9.81874287128,1024,0.104290336698,512,2909,true,1,0.00981874287128 +fp16,1,1,1,256,1,control_optimized_transform,3,13,100,10.7021713257,1024,0.0956815181553,512,2909,true,1,0.0107021713257 +fp16,1,1,1,256,1,optimized_transform,3,14,100,10.6299424171,1024,0.0963316601178,512,2909,true,1,0.0106299424171 +fp16,1,1,1,256,1,shuffle32_transform,4,0,100,11.6779434681,1024,0.0876866721266,512,2909,true,1,0.0116779434681 +fp16,1,1,1,256,1,baseline_split,4,1,100,100.672912598,1668,0.0165685084196,512,2909,true,1,0.100672912598 +fp16,1,1,1,256,1,control_optimized_split,4,2,100,66.9462823868,1668,0.0249154985241,512,2909,true,1,0.0669462823868 +fp16,1,1,1,256,1,optimized_split,4,3,100,35.5131435394,1668,0.0469685258402,512,2909,true,1,0.0355131435394 +fp16,1,1,1,256,1,control_shuffle32_split,4,4,100,302.694187164,1668,0.00551051216287,512,2909,true,1,0.302694187164 +fp16,1,1,1,256,1,shuffle32_split,4,5,100,67.7588558197,1668,0.0246167084704,512,2909,true,1,0.0677588558197 +fp16,1,1,1,256,1,baseline_fused,4,6,100,96.1606884003,644,0.00669712343696,512,2909,true,1,0.0961606884003 +fp16,1,1,1,256,1,control_optimized_fused,4,7,100,61.2943983078,644,0.0105066697411,512,2909,true,1,0.0612943983078 +fp16,1,1,1,256,1,optimized_fused,4,8,100,29.7238874435,644,0.0216660758531,512,2909,true,1,0.0297238874435 +fp16,1,1,1,256,1,control_shuffle32_fused,4,9,100,299.446868896,644,0.00215063193806,512,2909,true,1,0.299446868896 +fp16,1,1,1,256,1,shuffle32_fused,4,10,100,61.6667413712,644,0.0104432305921,512,2909,true,1,0.0616667413712 +fp16,1,1,1,256,1,baseline_transform,4,11,100,9.76251423359,1024,0.104891012243,512,2909,true,1,0.00976251423359 +fp16,1,1,1,256,1,control_optimized_transform,4,12,100,10.4671990871,1024,0.0978294184982,512,2909,true,1,0.0104671990871 +fp16,1,1,1,256,1,optimized_transform,4,13,100,10.6873142719,1024,0.0958145305682,512,2909,true,1,0.0106873142719 +fp16,1,1,1,256,1,control_shuffle32_transform,4,14,100,11.4994287491,1024,0.0890479016257,512,2909,true,1,0.0114994287491 +fp16,1,17,1,256,17,baseline_transform,0,0,100,10.0653707981,17408,1.72949415865,8704,2909,true,1,0.0100653707981 +fp16,1,17,1,256,17,control_optimized_transform,0,1,100,10.6591999531,17408,1.63314320743,8704,2909,true,1,0.0106591999531 +fp16,1,17,1,256,17,optimized_transform,0,2,100,10.6379425526,17408,1.63640665608,8704,2909,true,1,0.0106379425526 +fp16,1,17,1,256,17,control_shuffle32_transform,0,3,100,11.6045713425,17408,1.50009849449,8704,2909,true,1,0.0116045713425 +fp16,1,17,1,256,17,shuffle32_transform,0,4,100,11.7028570175,17408,1.48750001593,8704,2909,true,1,0.0117028570175 +fp16,1,17,1,256,17,baseline_split,0,5,100,101.21805191,28356,0.280147656123,8704,2909,true,1,0.10121805191 +fp16,1,17,1,256,17,control_optimized_split,0,6,100,67.281370163,28356,0.421453961644,8704,2909,true,1,0.067281370163 +fp16,1,17,1,256,17,optimized_split,0,7,100,36.8509721756,28356,0.769477664385,8704,2909,true,1,0.0368509721756 +fp16,1,17,1,256,17,control_shuffle32_split,0,8,100,303.241825104,28356,0.0935095282133,8704,2909,true,1,0.303241825104 +fp16,1,17,1,256,17,shuffle32_split,0,9,100,68.61348629,28356,0.413271523329,8704,2909,true,1,0.06861348629 +fp16,1,17,1,256,17,baseline_fused,0,10,100,96.5883350372,10948,0.113347020588,8704,2909,true,1,0.0965883350372 +fp16,1,17,1,256,17,control_optimized_fused,0,11,100,61.6457128525,10948,0.177595480584,8704,2909,true,1,0.0616457128525 +fp16,1,17,1,256,17,optimized_fused,0,12,100,30.4779434204,10948,0.359210588752,8704,2909,true,1,0.0304779434204 +fp16,1,17,1,256,17,control_shuffle32_fused,0,13,100,299.919319153,10948,0.0365031503503,8704,2909,true,1,0.299919319153 +fp16,1,17,1,256,17,shuffle32_fused,0,14,100,61.6050291061,10948,0.17771276402,8704,2909,true,1,0.0616050291061 +fp16,1,17,1,256,17,control_optimized_transform,1,0,100,10.9001147747,17408,1.59704740361,8704,2909,true,1,0.0109001147747 +fp16,1,17,1,256,17,optimized_transform,1,1,100,10.7284581661,17408,1.6226003523,8704,2909,true,1,0.0107284581661 +fp16,1,17,1,256,17,control_shuffle32_transform,1,2,100,11.7021715641,17408,1.48758714608,8704,2909,true,1,0.0117021715641 +fp16,1,17,1,256,17,shuffle32_transform,1,3,100,11.8059432507,17408,1.47451157696,8704,2909,true,1,0.0118059432507 +fp16,1,17,1,256,17,baseline_split,1,4,100,101.218748093,28356,0.280145729268,8704,2909,true,1,0.101218748093 +fp16,1,17,1,256,17,control_optimized_split,1,5,100,67.2816038132,28356,0.421452498052,8704,2909,true,1,0.0672816038132 +fp16,1,17,1,256,17,optimized_split,1,6,100,36.3954305649,28356,0.779108793601,8704,2909,true,1,0.0363954305649 +fp16,1,17,1,256,17,control_shuffle32_split,1,7,100,303.840694427,28356,0.093325221144,8704,2909,true,1,0.303840694427 +fp16,1,17,1,256,17,shuffle32_split,1,8,100,68.1394290924,28356,0.416146721182,8704,2909,true,1,0.0681394290924 +fp16,1,17,1,256,17,baseline_fused,1,9,100,96.6102790833,10948,0.11332127496,8704,2909,true,1,0.0966102790833 +fp16,1,17,1,256,17,control_optimized_fused,1,10,100,61.6969108582,10948,0.177448106359,8704,2909,true,1,0.0616969108582 +fp16,1,17,1,256,17,optimized_fused,1,11,100,30.4973697662,10948,0.358981777246,8704,2909,true,1,0.0304973697662 +fp16,1,17,1,256,17,control_shuffle32_fused,1,12,100,299.936904907,10948,0.0365010101154,8704,2909,true,1,0.299936904907 +fp16,1,17,1,256,17,shuffle32_fused,1,13,100,61.6034317017,10948,0.177717372192,8704,2909,true,1,0.0616034317017 +fp16,1,17,1,256,17,baseline_transform,1,14,100,10.0948572159,17408,1.72444241932,8704,2909,true,1,0.0100948572159 +fp16,1,17,1,256,17,optimized_transform,2,0,100,10.9611427784,17408,1.58815557392,8704,2909,true,1,0.0109611427784 +fp16,1,17,1,256,17,control_shuffle32_transform,2,1,100,11.6278862953,17408,1.49709066273,8704,2909,true,1,0.0116278862953 +fp16,1,17,1,256,17,shuffle32_transform,2,2,100,11.7906284332,17408,1.47642681631,8704,2909,true,1,0.0117906284332 +fp16,1,17,1,256,17,baseline_split,2,3,100,101.276807785,28356,0.279985128088,8704,2909,true,1,0.101276807785 +fp16,1,17,1,256,17,control_optimized_split,2,4,100,67.2715425491,28356,0.421515531315,8704,2909,true,1,0.0672715425491 +fp16,1,17,1,256,17,optimized_split,2,5,100,36.3328003883,28356,0.780451814804,8704,2909,true,1,0.0363328003883 +fp16,1,17,1,256,17,control_shuffle32_split,2,6,100,303.860797882,28356,0.09331904674,8704,2909,true,1,0.303860797882 +fp16,1,17,1,256,17,shuffle32_split,2,7,100,68.1040000916,28356,0.41636320865,8704,2909,true,1,0.0681040000916 +fp16,1,17,1,256,17,baseline_fused,2,8,100,96.595659256,10948,0.113338426223,8704,2909,true,1,0.096595659256 +fp16,1,17,1,256,17,control_optimized_fused,2,9,100,61.642742157,10948,0.17760403929,8704,2909,true,1,0.061642742157 +fp16,1,17,1,256,17,optimized_fused,2,10,100,30.5008006096,10948,0.358941397642,8704,2909,true,1,0.0305008006096 +fp16,1,17,1,256,17,control_shuffle32_fused,2,11,100,299.890060425,10948,0.0365067117746,8704,2909,true,1,0.299890060425 +fp16,1,17,1,256,17,shuffle32_fused,2,12,100,61.6011428833,10948,0.177723975361,8704,2909,true,1,0.0616011428833 +fp16,1,17,1,256,17,baseline_transform,2,13,100,10.089828968,17408,1.72530179204,8704,2909,true,1,0.010089828968 +fp16,1,17,1,256,17,control_optimized_transform,2,14,100,10.7259428501,17408,1.62298086455,8704,2909,true,1,0.0107259428501 +fp16,1,17,1,256,17,control_shuffle32_transform,3,0,100,11.737600565,17408,1.48309698423,8704,2909,true,1,0.011737600565 +fp16,1,17,1,256,17,shuffle32_transform,3,1,100,11.9474279881,17408,1.45705000419,8704,2909,true,1,0.0119474279881 +fp16,1,17,1,256,17,baseline_split,3,2,100,101.230859756,28356,0.280112211515,8704,2909,true,1,0.101230859756 +fp16,1,17,1,256,17,control_optimized_split,3,3,100,67.287774086,28356,0.421413850958,8704,2909,true,1,0.067287774086 +fp16,1,17,1,256,17,optimized_split,3,4,100,36.2710857391,28356,0.781779740588,8704,2909,true,1,0.0362710857391 +fp16,1,17,1,256,17,control_shuffle32_split,3,5,100,303.247089386,28356,0.0935079049148,8704,2909,true,1,0.303247089386 +fp16,1,17,1,256,17,shuffle32_split,3,6,100,68.0889129639,28356,0.416455466326,8704,2909,true,1,0.0680889129639 +fp16,1,17,1,256,17,baseline_fused,3,7,100,96.5622806549,10948,0.11337760382,8704,2909,true,1,0.0965622806549 +fp16,1,17,1,256,17,control_optimized_fused,3,8,100,61.7124557495,10948,0.177403408551,8704,2909,true,1,0.0617124557495 +fp16,1,17,1,256,17,optimized_fused,3,9,100,30.5508565903,10948,0.358353290935,8704,2909,true,1,0.0305508565903 +fp16,1,17,1,256,17,control_shuffle32_fused,3,10,100,299.930973053,10948,0.0365017320104,8704,2909,true,1,0.299930973053 +fp16,1,17,1,256,17,shuffle32_fused,3,11,100,61.5999984741,10948,0.17772727713,8704,2909,true,1,0.0615999984741 +fp16,1,17,1,256,17,baseline_transform,3,12,100,10.2073144913,17408,1.70544368109,8704,2909,true,1,0.0102073144913 +fp16,1,17,1,256,17,control_optimized_transform,3,13,100,10.8210289478,17408,1.60871947427,8704,2909,true,1,0.0108210289478 +fp16,1,17,1,256,17,optimized_transform,3,14,100,10.8150863647,17408,1.60960341997,8704,2909,true,1,0.0108150863647 +fp16,1,17,1,256,17,shuffle32_transform,4,0,100,11.9677710533,17408,1.45457328039,8704,2909,true,1,0.0119677710533 +fp16,1,17,1,256,17,baseline_split,4,1,100,101.188344955,28356,0.280229902095,8704,2909,true,1,0.101188344955 +fp16,1,17,1,256,17,control_optimized_split,4,2,100,67.2507429123,28356,0.421645899689,8704,2909,true,1,0.0672507429123 +fp16,1,17,1,256,17,optimized_split,4,3,100,36.2708592415,28356,0.781784622504,8704,2909,true,1,0.0362708592415 +fp16,1,17,1,256,17,control_shuffle32_split,4,4,100,303.314743042,28356,0.0934870481916,8704,2909,true,1,0.303314743042 +fp16,1,17,1,256,17,shuffle32_split,4,5,100,68.0326843262,28356,0.416799664468,8704,2909,true,1,0.0680326843262 +fp16,1,17,1,256,17,baseline_fused,4,6,100,96.5926933289,10948,0.113341906336,8704,2909,true,1,0.0965926933289 +fp16,1,17,1,256,17,control_optimized_fused,4,7,100,61.6841173172,10948,0.17748490983,8704,2909,true,1,0.0616841173172 +fp16,1,17,1,256,17,optimized_fused,4,8,100,30.4756569862,10948,0.359237538503,8704,2909,true,1,0.0304756569862 +fp16,1,17,1,256,17,control_shuffle32_fused,4,9,100,299.925479889,10948,0.0365024005431,8704,2909,true,1,0.299925479889 +fp16,1,17,1,256,17,shuffle32_fused,4,10,100,61.6011428833,10948,0.177723975361,8704,2909,true,1,0.0616011428833 +fp16,1,17,1,256,17,baseline_transform,4,11,100,9.95839953423,17408,1.7480720612,8704,2909,true,1,0.00995839953423 +fp16,1,17,1,256,17,control_optimized_transform,4,12,100,10.6672000885,17408,1.63191839054,8704,2909,true,1,0.0106672000885 +fp16,1,17,1,256,17,optimized_transform,4,13,100,10.7645714283,17408,1.61715681074,8704,2909,true,1,0.0107645714283 +fp16,1,17,1,256,17,control_shuffle32_transform,4,14,100,11.6873145103,17408,1.48947818462,8704,2909,true,1,0.0116873145103 +fp16,1,257,1,256,257,baseline_transform,0,0,100,10.4482281208,263168,25.1878114602,131584,2909,true,1,0.0104482281208 +fp16,1,257,1,256,257,control_optimized_transform,0,1,100,11.1136007309,263168,23.679814164,131584,2909,true,1,0.0111136007309 +fp16,1,257,1,256,257,optimized_transform,0,2,100,11.0838854313,263168,23.7432984698,131584,2909,true,1,0.0110838854313 +fp16,1,257,1,256,257,control_shuffle32_transform,0,3,100,13.6669719219,263168,19.25576503,131584,2909,true,1,0.0136669719219 +fp16,1,257,1,256,257,shuffle32_transform,0,4,100,13.7675428391,263168,19.1151030417,131584,2909,true,1,0.0137675428391 +fp16,1,257,1,256,257,baseline_split,0,5,100,103.927087784,428676,4.12477640951,131584,2909,true,1,0.103927087784 +fp16,1,257,1,256,257,control_optimized_split,0,6,100,69.8052597046,428676,6.14102722079,131584,2909,true,1,0.0698052597046 +fp16,1,257,1,256,257,optimized_split,0,7,100,37.0018267632,428676,11.5852658504,131584,2909,true,1,0.0370018267632 +fp16,1,257,1,256,257,control_shuffle32_split,0,8,100,325.687522888,428676,1.31621867549,131584,2909,true,1,0.325687522888 +fp16,1,257,1,256,257,shuffle32_split,0,9,100,76.7782831192,428676,5.58329754957,131584,2909,true,1,0.0767782831192 +fp16,1,257,1,256,257,baseline_fused,0,10,100,98.9590835571,165508,1.67248921525,131584,2909,true,1,0.0989590835571 +fp16,1,257,1,256,257,control_optimized_fused,0,11,100,63.993601799,165508,2.58632105941,131584,2909,true,1,0.063993601799 +fp16,1,257,1,256,257,optimized_fused,0,12,100,30.8473157883,165508,5.36539390124,131584,2909,true,1,0.0308473157883 +fp16,1,257,1,256,257,control_shuffle32_fused,0,13,100,320.781021118,165508,0.515953217628,131584,2909,true,1,0.320781021118 +fp16,1,257,1,256,257,shuffle32_fused,0,14,100,68.0425167084,165508,2.4324203161,131584,2909,true,1,0.0680425167084 +fp16,1,257,1,256,257,control_optimized_transform,1,0,100,11.0772562027,263168,23.7575077424,131584,2909,true,1,0.0110772562027 +fp16,1,257,1,256,257,optimized_transform,1,1,100,11.1684572697,263168,23.563505115,131584,2909,true,1,0.0111684572697 +fp16,1,257,1,256,257,control_shuffle32_transform,1,2,100,13.8253712654,263168,19.0351488541,131584,2909,true,1,0.0138253712654 +fp16,1,257,1,256,257,shuffle32_transform,1,3,100,13.9321136475,263168,18.8893090208,131584,2909,true,1,0.0139321136475 +fp16,1,257,1,256,257,baseline_split,1,4,100,103.885259628,428676,4.12643720133,131584,2909,true,1,0.103885259628 +fp16,1,257,1,256,257,control_optimized_split,1,5,100,69.8194265366,428676,6.139781165,131584,2909,true,1,0.0698194265366 +fp16,1,257,1,256,257,optimized_split,1,6,100,37.0038866997,428676,11.5846209205,131584,2909,true,1,0.0370038866997 +fp16,1,257,1,256,257,control_shuffle32_split,1,7,100,325.739860535,428676,1.3160071945,131584,2909,true,1,0.325739860535 +fp16,1,257,1,256,257,shuffle32_split,1,8,100,76.7145109177,428676,5.58793890324,131584,2909,true,1,0.0767145109177 +fp16,1,257,1,256,257,baseline_fused,1,9,100,98.9497184753,165508,1.67264750775,131584,2909,true,1,0.0989497184753 +fp16,1,257,1,256,257,control_optimized_fused,1,10,100,63.987660408,165508,2.58656120484,131584,2909,true,1,0.063987660408 +fp16,1,257,1,256,257,optimized_fused,1,11,100,30.8797717094,165508,5.35975464966,131584,2909,true,1,0.0308797717094 +fp16,1,257,1,256,257,control_shuffle32_fused,1,12,100,320.831108093,165508,0.515872668906,131584,2909,true,1,0.320831108093 +fp16,1,257,1,256,257,shuffle32_fused,1,13,100,68.0480003357,165508,2.43222430025,131584,2909,true,1,0.0680480003357 +fp16,1,257,1,256,257,baseline_transform,1,14,100,10.6128001213,263168,24.7972257078,131584,2909,true,1,0.0106128001213 +fp16,1,257,1,256,257,optimized_transform,2,0,100,11.1588573456,263168,23.5837767121,131584,2909,true,1,0.0111588573456 +fp16,1,257,1,256,257,control_shuffle32_transform,2,1,100,13.6950862408,263168,19.2162353251,131584,2909,true,1,0.0136950862408 +fp16,1,257,1,256,257,shuffle32_transform,2,2,100,13.8473141193,263168,19.0049852074,131584,2909,true,1,0.0138473141193 +fp16,1,257,1,256,257,baseline_split,2,3,100,103.917264938,428676,4.12516630662,131584,2909,true,1,0.103917264938 +fp16,1,257,1,256,257,control_optimized_split,2,4,100,69.8237705231,428676,6.13939918725,131584,2909,true,1,0.0698237705231 +fp16,1,257,1,256,257,optimized_split,2,5,100,36.969370842,428676,11.5954367152,131584,2909,true,1,0.036969370842 +fp16,1,257,1,256,257,control_shuffle32_split,2,6,100,326.285018921,428676,1.31380840413,131584,2909,true,1,0.326285018921 +fp16,1,257,1,256,257,shuffle32_split,2,7,100,76.7604589462,428676,5.58459401995,131584,2909,true,1,0.0767604589462 +fp16,1,257,1,256,257,baseline_fused,2,8,100,98.9622879028,165508,1.67243506094,131584,2909,true,1,0.0989622879028 +fp16,1,257,1,256,257,control_optimized_fused,2,9,100,63.9981746674,165508,2.58613625873,131584,2909,true,1,0.0639981746674 +fp16,1,257,1,256,257,optimized_fused,2,10,100,30.8957719803,165508,5.35697894539,131584,2909,true,1,0.0308957719803 +fp16,1,257,1,256,257,control_shuffle32_fused,2,11,100,320.829277039,165508,0.515875613123,131584,2909,true,1,0.320829277039 +fp16,1,257,1,256,257,shuffle32_fused,2,12,100,68.0528020859,165508,2.43205268449,131584,2909,true,1,0.0680528020859 +fp16,1,257,1,256,257,baseline_transform,2,13,100,10.5663990974,263168,24.906119632,131584,2909,true,1,0.0105663990974 +fp16,1,257,1,256,257,control_optimized_transform,2,14,100,11.1764574051,263168,23.5466383006,131584,2909,true,1,0.0111764574051 +fp16,1,257,1,256,257,control_shuffle32_transform,3,0,100,13.745828867,263168,19.1452987337,131584,2909,true,1,0.013745828867 +fp16,1,257,1,256,257,shuffle32_transform,3,1,100,13.9168000221,263168,18.9100942445,131584,2909,true,1,0.0139168000221 +fp16,1,257,1,256,257,baseline_split,3,2,100,103.899202347,428676,4.12588345548,131584,2909,true,1,0.103899202347 +fp16,1,257,1,256,257,control_optimized_split,3,3,100,69.6982860565,428676,6.15045253268,131584,2909,true,1,0.0696982860565 +fp16,1,257,1,256,257,optimized_split,3,4,100,36.9915437698,428676,11.5884863489,131584,2909,true,1,0.0369915437698 +fp16,1,257,1,256,257,control_shuffle32_split,3,5,100,325.752906799,428676,1.31595448898,131584,2909,true,1,0.325752906799 +fp16,1,257,1,256,257,shuffle32_split,3,6,100,76.7076587677,428676,5.58843806325,131584,2909,true,1,0.0767076587677 +fp16,1,257,1,256,257,baseline_fused,3,7,100,98.9577198029,165508,1.67251226412,131584,2909,true,1,0.0989577198029 +fp16,1,257,1,256,257,control_optimized_fused,3,8,100,63.9883422852,165508,2.58653364174,131584,2909,true,1,0.0639883422852 +fp16,1,257,1,256,257,optimized_fused,3,9,100,30.8473157883,165508,5.36539390124,131584,2909,true,1,0.0308473157883 +fp16,1,257,1,256,257,control_shuffle32_fused,3,10,100,320.824699402,165508,0.515882973813,131584,2909,true,1,0.320824699402 +fp16,1,257,1,256,257,shuffle32_fused,3,11,100,68.0338287354,165508,2.43273093807,131584,2909,true,1,0.0680338287354 +fp16,1,257,1,256,257,baseline_transform,3,12,100,10.6043422222,263168,24.8170036845,131584,2909,true,1,0.0106043422222 +fp16,1,257,1,256,257,control_optimized_transform,3,13,100,11.2150859833,263168,23.4655356537,131584,2909,true,1,0.0112150859833 +fp16,1,257,1,256,257,optimized_transform,3,14,100,11.0580575466,263168,23.7987547895,131584,2909,true,1,0.0110580575466 +fp16,1,257,1,256,257,shuffle32_transform,4,0,100,13.8845717907,263168,18.9539874882,131584,2909,true,1,0.0138845717907 +fp16,1,257,1,256,257,baseline_split,4,1,100,103.909263611,428676,4.1254839569,131584,2909,true,1,0.103909263611 +fp16,1,257,1,256,257,control_optimized_split,4,2,100,69.6943998337,428676,6.15079548749,131584,2909,true,1,0.0696943998337 +fp16,1,257,1,256,257,optimized_split,4,3,100,36.8886852264,428676,11.6207990978,131584,2909,true,1,0.0368886852264 +fp16,1,257,1,256,257,control_shuffle32_split,4,4,100,325.736923218,428676,1.31601906153,131584,2909,true,1,0.325736923218 +fp16,1,257,1,256,257,shuffle32_split,4,5,100,76.6665172577,428676,5.59143698362,131584,2909,true,1,0.0766665172577 +fp16,1,257,1,256,257,baseline_fused,4,6,100,98.9581775665,165508,1.67250452737,131584,2909,true,1,0.0989581775665 +fp16,1,257,1,256,257,control_optimized_fused,4,7,100,63.9929151535,165508,2.58634881069,131584,2909,true,1,0.0639929151535 +fp16,1,257,1,256,257,optimized_fused,4,8,100,30.8774852753,165508,5.36015153192,131584,2909,true,1,0.0308774852753 +fp16,1,257,1,256,257,control_shuffle32_fused,4,9,100,320.828819275,165508,0.515876349182,131584,2909,true,1,0.320828819275 +fp16,1,257,1,256,257,shuffle32_fused,4,10,100,68.0381727219,165508,2.43257561717,131584,2909,true,1,0.0680381727219 +fp16,1,257,1,256,257,baseline_transform,4,11,100,10.6822860241,263168,24.6359252511,131584,2909,true,1,0.0106822860241 +fp16,1,257,1,256,257,control_optimized_transform,4,12,100,11.3222849369,263168,23.2433648744,131584,2909,true,1,0.0113222849369 +fp16,1,257,1,256,257,optimized_transform,4,13,100,11.1056005955,263168,23.6968723787,131584,2909,true,1,0.0111056005955 +fp16,1,257,1,256,257,control_shuffle32_transform,4,14,100,13.7778294086,263168,19.1008316473,131584,2909,true,1,0.0137778294086 +fp16,4,64,16,256,4096,baseline_transform,0,0,100,49.2054891586,4194304,85.2405711582,2097152,2909,true,1,0.0492054891586 +fp16,4,64,16,256,4096,control_optimized_transform,0,1,100,37.3846840858,4194304,112.193110697,2097152,2909,true,1,0.0373846840858 +fp16,4,64,16,256,4096,optimized_transform,0,2,100,37.4758839607,4194304,111.920081842,2097152,2909,true,1,0.0374758839607 +fp16,4,64,16,256,4096,control_shuffle32_transform,0,3,100,22.706284523,4194304,184.719961372,2097152,2909,true,1,0.022706284523 +fp16,4,64,16,256,4096,shuffle32_transform,0,4,100,22.7117729187,4194304,184.675323015,2097152,2909,true,1,0.0227117729187 +fp16,4,64,16,256,4096,baseline_split,0,5,100,1139.77966309,6832128,5.9942532941,2097152,2909,true,1,1.13977966309 +fp16,4,64,16,256,4096,control_optimized_split,0,6,100,501.357498169,6832128,13.6272580443,2097152,2909,true,1,0.501357498169 +fp16,4,64,16,256,4096,optimized_split,0,7,100,188.882274628,6832128,36.1713560125,2097152,2909,true,1,0.188882274628 +fp16,4,64,16,256,4096,control_shuffle32_split,0,8,100,841.649398804,6832128,8.11754634378,2097152,2909,true,1,0.841649398804 +fp16,4,64,16,256,4096,shuffle32_split,0,9,100,149.530973434,6832128,45.690386701,2097152,2909,true,1,0.149530973434 +fp16,4,64,16,256,4096,baseline_fused,0,10,100,1129.94812012,2637824,2.33446470067,2097152,2909,true,1,1.12994812012 +fp16,4,64,16,256,4096,control_optimized_fused,0,11,100,496.430625916,2637824,5.31358031172,2097152,2909,true,1,0.496430625916 +fp16,4,64,16,256,4096,optimized_fused,0,12,100,171.281604767,2637824,15.4005096087,2097152,2909,true,1,0.171281604767 +fp16,4,64,16,256,4096,control_shuffle32_fused,0,13,100,981.268386841,2637824,2.68817790869,2097152,2909,true,1,0.981268386841 +fp16,4,64,16,256,4096,shuffle32_fused,0,14,100,131.983547211,2637824,19.9860062542,2097152,2909,true,1,0.131983547211 +fp16,4,64,16,256,4096,control_optimized_transform,1,0,100,37.362742424,4194304,112.258997276,2097152,2909,true,1,0.037362742424 +fp16,4,64,16,256,4096,optimized_transform,1,1,100,37.5330281258,4194304,111.749683131,2097152,2909,true,1,0.0375330281258 +fp16,4,64,16,256,4096,control_shuffle32_transform,1,2,100,22.319085598,4194304,187.924544739,2097152,2909,true,1,0.022319085598 +fp16,4,64,16,256,4096,shuffle32_transform,1,3,100,23.1732559204,4194304,180.997612697,2097152,2909,true,1,0.0231732559204 +fp16,4,64,16,256,4096,baseline_split,1,4,100,1142.40463257,6832128,5.98047995012,2097152,2909,true,1,1.14240463257 +fp16,4,64,16,256,4096,control_optimized_split,1,5,100,503.130531311,6832128,13.5792355558,2097152,2909,true,1,0.503130531311 +fp16,4,64,16,256,4096,optimized_split,1,6,100,188.80777359,6832128,36.1856287487,2097152,2909,true,1,0.18880777359 +fp16,4,64,16,256,4096,control_shuffle32_split,1,7,100,841.608200073,6832128,8.11794371705,2097152,2909,true,1,0.841608200073 +fp16,4,64,16,256,4096,shuffle32_split,1,8,100,149.390630722,6832128,45.7333098266,2097152,2909,true,1,0.149390630722 +fp16,4,64,16,256,4096,baseline_fused,1,9,100,1132.81852722,2637824,2.32854948662,2097152,2909,true,1,1.13281852722 +fp16,4,64,16,256,4096,control_optimized_fused,1,10,100,496.620330811,2637824,5.3115505676,2097152,2909,true,1,0.496620330811 +fp16,4,64,16,256,4096,optimized_fused,1,11,100,171.133480072,2637824,15.4138395298,2097152,2909,true,1,0.171133480072 +fp16,4,64,16,256,4096,control_shuffle32_fused,1,12,100,991.271743774,2637824,2.6610503291,2097152,2909,true,1,0.991271743774 +fp16,4,64,16,256,4096,shuffle32_fused,1,13,100,131.96480751,2637824,19.9888443727,2097152,2909,true,1,0.13196480751 +fp16,4,64,16,256,4096,baseline_transform,1,14,100,49.2118835449,4194304,85.2294953549,2097152,2909,true,1,0.0492118835449 +fp16,4,64,16,256,4096,optimized_transform,2,0,100,37.5229716301,4194304,111.779633056,2097152,2909,true,1,0.0375229716301 +fp16,4,64,16,256,4096,control_shuffle32_transform,2,1,100,23.1037712097,4194304,181.541963947,2097152,2909,true,1,0.0231037712097 +fp16,4,64,16,256,4096,shuffle32_transform,2,2,100,22.7362298965,4194304,184.476670894,2097152,2909,true,1,0.0227362298965 +fp16,4,64,16,256,4096,baseline_split,2,3,100,1141.93458557,6832128,5.98294165561,2097152,2909,true,1,1.14193458557 +fp16,4,64,16,256,4096,control_optimized_split,2,4,100,503.74786377,6832128,13.562594487,2097152,2909,true,1,0.50374786377 +fp16,4,64,16,256,4096,optimized_split,2,5,100,188.736457825,6832128,36.1993018135,2097152,2909,true,1,0.188736457825 +fp16,4,64,16,256,4096,control_shuffle32_split,2,6,100,840.345840454,6832128,8.13013841576,2097152,2909,true,1,0.840345840454 +fp16,4,64,16,256,4096,shuffle32_split,2,7,100,149.488229752,6832128,45.7034511102,2097152,2909,true,1,0.149488229752 +fp16,4,64,16,256,4096,baseline_fused,2,8,100,1131.45011902,2637824,2.33136570111,2097152,2909,true,1,1.13145011902 +fp16,4,64,16,256,4096,control_optimized_fused,2,9,100,496.117477417,2637824,5.31693423448,2097152,2909,true,1,0.496117477417 +fp16,4,64,16,256,4096,optimized_fused,2,10,100,171.221942902,2637824,15.4058758784,2097152,2909,true,1,0.171221942902 +fp16,4,64,16,256,4096,control_shuffle32_fused,2,11,100,952.227630615,2637824,2.7701611623,2097152,2909,true,1,0.952227630615 +fp16,4,64,16,256,4096,shuffle32_fused,2,12,100,131.495990753,2637824,20.0601097029,2097152,2909,true,1,0.131495990753 +fp16,4,64,16,256,4096,baseline_transform,2,13,100,49.1588544846,4194304,85.3214348458,2097152,2909,true,1,0.0491588544846 +fp16,4,64,16,256,4096,control_optimized_transform,2,14,100,37.4086856842,4194304,112.12112704,2097152,2909,true,1,0.0374086856842 +fp16,4,64,16,256,4096,control_shuffle32_transform,3,0,100,22.5039982796,4194304,186.380391071,2097152,2909,true,1,0.0225039982796 +fp16,4,64,16,256,4096,shuffle32_transform,3,1,100,22.8557705879,4194304,183.511817458,2097152,2909,true,1,0.0228557705879 +fp16,4,64,16,256,4096,baseline_split,3,2,100,1142.45735168,6832128,5.98020397867,2097152,2909,true,1,1.14245735168 +fp16,4,64,16,256,4096,control_optimized_split,3,3,100,503.601341248,6832128,13.5665405161,2097152,2909,true,1,0.503601341248 +fp16,4,64,16,256,4096,optimized_split,3,4,100,188.843421936,6832128,36.1787979161,2097152,2909,true,1,0.188843421936 +fp16,4,64,16,256,4096,control_shuffle32_split,3,5,100,841.457366943,6832128,8.11939887676,2097152,2909,true,1,0.841457366943 +fp16,4,64,16,256,4096,shuffle32_split,3,6,100,149.297370911,6832128,45.7618775088,2097152,2909,true,1,0.149297370911 +fp16,4,64,16,256,4096,baseline_fused,3,7,100,1131.05239868,2637824,2.33218549651,2097152,2909,true,1,1.13105239868 +fp16,4,64,16,256,4096,control_optimized_fused,3,8,100,495.001144409,2637824,5.32892505359,2097152,2909,true,1,0.495001144409 +fp16,4,64,16,256,4096,optimized_fused,3,9,100,171.21301651,2637824,15.4066790818,2097152,2909,true,1,0.17121301651 +fp16,4,64,16,256,4096,control_shuffle32_fused,3,10,100,988.690261841,2637824,2.66799836289,2097152,2909,true,1,0.988690261841 +fp16,4,64,16,256,4096,shuffle32_fused,3,11,100,131.867198944,2637824,20.0036401859,2097152,2909,true,1,0.131867198944 +fp16,4,64,16,256,4096,baseline_transform,3,12,100,49.1929149628,4194304,85.2623594917,2097152,2909,true,1,0.0491929149628 +fp16,4,64,16,256,4096,control_optimized_transform,3,13,100,37.3908567429,4194304,112.174589335,2097152,2909,true,1,0.0373908567429 +fp16,4,64,16,256,4096,optimized_transform,3,14,100,37.4285721779,4194304,112.061555008,2097152,2909,true,1,0.0374285721779 +fp16,4,64,16,256,4096,shuffle32_transform,4,0,100,22.8502845764,4194304,183.555875901,2097152,2909,true,1,0.0228502845764 +fp16,4,64,16,256,4096,baseline_split,4,1,100,1141.74034119,6832128,5.983959534,2097152,2909,true,1,1.14174034119 +fp16,4,64,16,256,4096,control_optimized_split,4,2,100,502.050056458,6832128,13.6084597783,2097152,2909,true,1,0.502050056458 +fp16,4,64,16,256,4096,optimized_split,4,3,100,188.882961273,6832128,36.1712245189,2097152,2909,true,1,0.188882961273 +fp16,4,64,16,256,4096,control_shuffle32_split,4,4,100,845.250701904,6832128,8.0829604573,2097152,2909,true,1,0.845250701904 +fp16,4,64,16,256,4096,shuffle32_split,4,5,100,149.340343475,6832128,45.7487095651,2097152,2909,true,1,0.149340343475 +fp16,4,64,16,256,4096,baseline_fused,4,6,100,1130.50079346,2637824,2.33332343972,2097152,2909,true,1,1.13050079346 +fp16,4,64,16,256,4096,control_optimized_fused,4,7,100,494.701919556,2637824,5.33214830128,2097152,2909,true,1,0.494701919556 +fp16,4,64,16,256,4096,optimized_fused,4,8,100,171.445484161,2637824,15.3857887416,2097152,2909,true,1,0.171445484161 +fp16,4,64,16,256,4096,control_shuffle32_fused,4,9,100,971.508560181,2637824,2.71518348692,2097152,2909,true,1,0.971508560181 +fp16,4,64,16,256,4096,shuffle32_fused,4,10,100,131.876573563,2637824,20.002218201,2097152,2909,true,1,0.131876573563 +fp16,4,64,16,256,4096,baseline_transform,4,11,100,49.1949748993,4194304,85.258789309,2097152,2909,true,1,0.0491949748993 +fp16,4,64,16,256,4096,control_optimized_transform,4,12,100,37.3497128487,4194304,112.298159212,2097152,2909,true,1,0.0373497128487 +fp16,4,64,16,256,4096,optimized_transform,4,13,100,37.4774861336,4194304,111.915297228,2097152,2909,true,1,0.0374774861336 +fp16,4,64,16,256,4096,control_shuffle32_transform,4,14,100,22.7216005325,4194304,184.595446698,2097152,2909,true,1,0.0227216005325 +fp16,16,64,16,256,16384,baseline_transform,0,0,100,170.537834167,16777216,98.3782635795,8388608,2909,true,1,0.170537834167 +fp16,16,64,16,256,16384,control_optimized_transform,0,1,100,120.761833191,16777216,138.928132811,8388608,2909,true,1,0.120761833191 +fp16,16,64,16,256,16384,optimized_transform,0,2,100,120.843429565,16777216,138.834325212,8388608,2909,true,1,0.120843429565 +fp16,16,64,16,256,16384,control_shuffle32_transform,0,3,100,43.2841157913,16777216,387.606762741,8388608,2909,true,1,0.0432841157913 +fp16,16,64,16,256,16384,shuffle32_transform,0,4,100,43.5023975372,16777216,385.661870375,8388608,2909,true,1,0.0435023975372 +fp16,16,64,16,256,16384,baseline_split,0,5,100,4263.64868164,27328512,6.40965380606,8388608,2909,true,1,4.26364868164 +fp16,16,64,16,256,16384,control_optimized_split,0,6,100,1870.68618774,27328512,14.6088169031,8388608,2909,true,1,1.87068618774 +fp16,16,64,16,256,16384,optimized_split,0,7,100,678.003005981,27328512,40.3073611163,8388608,2909,true,1,0.678003005981 +fp16,16,64,16,256,16384,control_shuffle32_split,0,8,100,3159.89990234,27328512,8.6485372463,8388608,2909,true,1,3.15989990234 +fp16,16,64,16,256,16384,shuffle32_split,0,9,100,436.298294067,27328512,62.6372194703,8388608,2909,true,1,0.436298294067 +fp16,16,64,16,256,16384,baseline_fused,0,10,100,4263.90106201,10551296,2.47456398414,8388608,2909,true,1,4.26390106201 +fp16,16,64,16,256,16384,control_optimized_fused,0,11,100,1824.09118652,10551296,5.7844125765,8388608,2909,true,1,1.82409118652 +fp16,16,64,16,256,16384,optimized_fused,0,12,100,631.356544495,10551296,16.7121036315,8388608,2909,true,1,0.631356544495 +fp16,16,64,16,256,16384,control_shuffle32_fused,0,13,100,2898.07403564,10551296,3.64079587693,8388608,2909,true,1,2.89807403564 +fp16,16,64,16,256,16384,shuffle32_fused,0,14,100,391.717262268,10551296,26.9359995495,8388608,2909,true,1,0.391717262268 +fp16,16,64,16,256,16384,control_optimized_transform,1,0,100,120.797939301,16777216,138.886607645,8388608,2909,true,1,0.120797939301 +fp16,16,64,16,256,16384,optimized_transform,1,1,100,120.86856842,16777216,138.805449748,8388608,2909,true,1,0.12086856842 +fp16,16,64,16,256,16384,control_shuffle32_transform,1,2,100,43.4155416489,16777216,386.43341446,8388608,2909,true,1,0.0434155416489 +fp16,16,64,16,256,16384,shuffle32_transform,1,3,100,43.5309696198,16777216,385.408736505,8388608,2909,true,1,0.0435309696198 +fp16,16,64,16,256,16384,baseline_split,1,4,100,4267.02514648,27328512,6.40458189531,8388608,2909,true,1,4.26702514648 +fp16,16,64,16,256,16384,control_optimized_split,1,5,100,1870.74951172,27328512,14.6083224017,8388608,2909,true,1,1.87074951172 +fp16,16,64,16,256,16384,optimized_split,1,6,100,677.895507812,27328512,40.3137529089,8388608,2909,true,1,0.677895507812 +fp16,16,64,16,256,16384,control_shuffle32_split,1,7,100,3182.75512695,27328512,8.58643248064,8388608,2909,true,1,3.18275512695 +fp16,16,64,16,256,16384,shuffle32_split,1,8,100,436.311340332,27328512,62.6353465376,8388608,2909,true,1,0.436311340332 +fp16,16,64,16,256,16384,baseline_fused,1,9,100,4262.66174316,10551296,2.47528343456,8388608,2909,true,1,4.26266174316 +fp16,16,64,16,256,16384,control_optimized_fused,1,10,100,1824.01473999,10551296,5.78465500781,8388608,2909,true,1,1.82401473999 +fp16,16,64,16,256,16384,optimized_fused,1,11,100,631.384048462,10551296,16.711375629,8388608,2909,true,1,0.631384048462 +fp16,16,64,16,256,16384,control_shuffle32_fused,1,12,100,2898.10699463,10551296,3.64075447164,8388608,2909,true,1,2.89810699463 +fp16,16,64,16,256,16384,shuffle32_fused,1,13,100,391.880912781,10551296,26.9247510044,8388608,2909,true,1,0.391880912781 +fp16,16,64,16,256,16384,baseline_transform,1,14,100,170.525951385,16777216,98.3851188848,8388608,2909,true,1,0.170525951385 +fp16,16,64,16,256,16384,optimized_transform,2,0,100,120.853481293,16777216,138.822777967,8388608,2909,true,1,0.120853481293 +fp16,16,64,16,256,16384,control_shuffle32_transform,2,1,100,43.3856010437,16777216,386.700093957,8388608,2909,true,1,0.0433856010437 +fp16,16,64,16,256,16384,shuffle32_transform,2,2,100,43.5236549377,16777216,385.473509153,8388608,2909,true,1,0.0435236549377 +fp16,16,64,16,256,16384,baseline_split,2,3,100,4266.79077148,27328512,6.40493369927,8388608,2909,true,1,4.26679077148 +fp16,16,64,16,256,16384,control_optimized_split,2,4,100,1870.47241211,27328512,14.6104865397,8388608,2909,true,1,1.87047241211 +fp16,16,64,16,256,16384,optimized_split,2,5,100,677.996139526,27328512,40.307769332,8388608,2909,true,1,0.677996139526 +fp16,16,64,16,256,16384,control_shuffle32_split,2,6,100,3175.79711914,27328512,8.60524491168,8388608,2909,true,1,3.17579711914 +fp16,16,64,16,256,16384,shuffle32_split,2,7,100,436.465148926,27328512,62.613274089,8388608,2909,true,1,0.436465148926 +fp16,16,64,16,256,16384,baseline_fused,2,8,100,4263.2244873,10551296,2.47495669802,8388608,2909,true,1,4.2632244873 +fp16,16,64,16,256,16384,control_optimized_fused,2,9,100,1824.19189453,10551296,5.78409323692,8388608,2909,true,1,1.82419189453 +fp16,16,64,16,256,16384,optimized_fused,2,10,100,631.37298584,10551296,16.7116684379,8388608,2909,true,1,0.63137298584 +fp16,16,64,16,256,16384,control_shuffle32_fused,2,11,100,2902.50732422,10551296,3.6352349267,8388608,2909,true,1,2.90250732422 +fp16,16,64,16,256,16384,shuffle32_fused,2,12,100,391.829719543,10551296,26.9282687702,8388608,2909,true,1,0.391829719543 +fp16,16,64,16,256,16384,baseline_transform,2,13,100,170.509719849,16777216,98.3944845777,8388608,2909,true,1,0.170509719849 +fp16,16,64,16,256,16384,control_optimized_transform,2,14,100,120.711774826,16777216,138.985745377,8388608,2909,true,1,0.120711774826 +fp16,16,64,16,256,16384,control_shuffle32_transform,3,0,100,43.4137153625,16777216,386.449670568,8388608,2909,true,1,0.0434137153625 +fp16,16,64,16,256,16384,shuffle32_transform,3,1,100,43.4662866592,16777216,385.982270156,8388608,2909,true,1,0.0434662866592 +fp16,16,64,16,256,16384,baseline_split,3,2,100,4265.00427246,27328512,6.4076165589,8388608,2909,true,1,4.26500427246 +fp16,16,64,16,256,16384,control_optimized_split,3,3,100,1870.51208496,27328512,14.6101766568,8388608,2909,true,1,1.87051208496 +fp16,16,64,16,256,16384,optimized_split,3,4,100,678.007278442,27328512,40.3071071195,8388608,2909,true,1,0.678007278442 +fp16,16,64,16,256,16384,control_shuffle32_split,3,5,100,3179.82055664,27328512,8.59435666674,8388608,2909,true,1,3.17982055664 +fp16,16,64,16,256,16384,shuffle32_split,3,6,100,436.737136841,27328512,62.5742802586,8388608,2909,true,1,0.436737136841 +fp16,16,64,16,256,16384,baseline_fused,3,7,100,4265.18951416,10551296,2.4738164541,8388608,2909,true,1,4.26518951416 +fp16,16,64,16,256,16384,control_optimized_fused,3,8,100,1823.91235352,10551296,5.78497973308,8388608,2909,true,1,1.82391235352 +fp16,16,64,16,256,16384,optimized_fused,3,9,100,631.290092468,10551296,16.7138628119,8388608,2909,true,1,0.631290092468 +fp16,16,64,16,256,16384,control_shuffle32_fused,3,10,100,2898.04107666,10551296,3.64083728315,8388608,2909,true,1,2.89804107666 +fp16,16,64,16,256,16384,shuffle32_fused,3,11,100,392.140769958,10551296,26.9069089682,8388608,2909,true,1,0.392140769958 +fp16,16,64,16,256,16384,baseline_transform,3,12,100,170.562744141,16777216,98.3638958468,8388608,2909,true,1,0.170562744141 +fp16,16,64,16,256,16384,control_optimized_transform,3,13,100,120.773935318,16777216,138.914211546,8388608,2909,true,1,0.120773935318 +fp16,16,64,16,256,16384,optimized_transform,3,14,100,120.864458084,16777216,138.810170218,8388608,2909,true,1,0.120864458084 +fp16,16,64,16,256,16384,shuffle32_transform,4,0,100,43.5129117966,16777216,385.568680819,8388608,2909,true,1,0.0435129117966 +fp16,16,64,16,256,16384,baseline_split,4,1,100,4266.33972168,27328512,6.40561084743,8388608,2909,true,1,4.26633972168 +fp16,16,64,16,256,16384,control_optimized_split,4,2,100,1870.70281982,27328512,14.6086870188,8388608,2909,true,1,1.87070281982 +fp16,16,64,16,256,16384,optimized_split,4,3,100,677.928237915,27328512,40.3118065771,8388608,2909,true,1,0.677928237915 +fp16,16,64,16,256,16384,control_shuffle32_split,4,4,100,3192.99835205,27328512,8.55888697294,8388608,2909,true,1,3.19299835205 +fp16,16,64,16,256,16384,shuffle32_split,4,5,100,436.317939758,27328512,62.6343991612,8388608,2909,true,1,0.436317939758 +fp16,16,64,16,256,16384,baseline_fused,4,6,100,4263.60351562,10551296,2.47473667787,8388608,2909,true,1,4.26360351563 +fp16,16,64,16,256,16384,control_optimized_fused,4,7,100,1824.02267456,10551296,5.78462984433,8388608,2909,true,1,1.82402267456 +fp16,16,64,16,256,16384,optimized_fused,4,8,100,631.271743774,10551296,16.7143486209,8388608,2909,true,1,0.631271743774 +fp16,16,64,16,256,16384,control_shuffle32_fused,4,9,100,2896.42089844,10551296,3.64287386743,8388608,2909,true,1,2.89642089844 +fp16,16,64,16,256,16384,shuffle32_fused,4,10,100,391.827201843,10551296,26.9284417987,8388608,2909,true,1,0.391827201843 +fp16,16,64,16,256,16384,baseline_transform,4,11,100,170.523204803,16777216,98.3867035536,8388608,2909,true,1,0.170523204803 +fp16,16,64,16,256,16384,control_optimized_transform,4,12,100,120.778970718,16777216,138.908420069,8388608,2909,true,1,0.120778970718 +fp16,16,64,16,256,16384,optimized_transform,4,13,100,120.852575302,16777216,138.823818674,8388608,2909,true,1,0.120852575302 +fp16,16,64,16,256,16384,control_shuffle32_transform,4,14,100,43.4278821945,16777216,386.323604841,8388608,2909,true,1,0.0434278821945 +bf16,1,1,1,64,1,baseline_transform,0,0,100,9.00639951229,256,0.0284242331967,128,2909,true,1,0.00900639951229 +bf16,1,1,1,64,1,control_optimized_transform,0,1,100,9.3561142683,256,0.0273617863847,128,2909,true,1,0.0093561142683 +bf16,1,1,1,64,1,optimized_transform,0,2,100,9.22628581524,256,0.0277468100519,128,2909,true,1,0.00922628581524 +bf16,1,1,1,64,1,control_shuffle32_transform,0,3,100,7.89600014687,256,0.0324214786269,128,2909,true,1,0.00789600014687 +bf16,1,1,1,64,1,shuffle32_transform,0,4,100,8.09462845325,256,0.0316259111185,128,2909,true,1,0.00809462845325 +bf16,1,1,1,64,1,baseline_split,0,5,100,69.6370267868,420,0.00603127415658,128,2909,true,1,0.0696370267868 +bf16,1,1,1,64,1,control_optimized_split,0,6,100,64.1460561752,420,0.0065475576371,128,2909,true,1,0.0641460561752 +bf16,1,1,1,64,1,optimized_split,0,7,100,33.7522292137,420,0.0124436225335,128,2909,true,1,0.0337522292137 +bf16,1,1,1,64,1,control_shuffle32_split,0,8,100,93.8669681549,420,0.0044744174469,128,2909,true,1,0.0938669681549 +bf16,1,1,1,64,1,shuffle32_split,0,9,100,34.7588562965,420,0.0120832514286,128,2909,true,1,0.0347588562965 +bf16,1,1,1,64,1,baseline_fused,0,10,100,65.1305103302,164,0.00251802111128,128,2909,true,1,0.0651305103302 +bf16,1,1,1,64,1,control_optimized_fused,0,11,100,58.5170269012,164,0.00280260308298,128,2909,true,1,0.0585170269012 +bf16,1,1,1,64,1,optimized_fused,0,12,100,28.7252569199,164,0.00570926138128,128,2909,true,1,0.0287252569199 +bf16,1,1,1,64,1,control_shuffle32_fused,0,13,100,89.3714237213,164,0.00183503846276,128,2909,true,1,0.0893714237213 +bf16,1,1,1,64,1,shuffle32_fused,0,14,100,29.6745157242,164,0.00552662768027,128,2909,true,1,0.0296745157242 +bf16,1,1,1,64,1,control_optimized_transform,1,0,100,9.36845719814,256,0.0273257372677,128,2909,true,1,0.00936845719814 +bf16,1,1,1,64,1,optimized_transform,1,1,100,9.27382826805,256,0.027604565515,128,2909,true,1,0.00927382826805 +bf16,1,1,1,64,1,control_shuffle32_transform,1,2,100,7.97417104244,256,0.0321036504782,128,2909,true,1,0.00797417104244 +bf16,1,1,1,64,1,shuffle32_transform,1,3,100,7.98308610916,256,0.0320677989063,128,2909,true,1,0.00798308610916 +bf16,1,1,1,64,1,baseline_split,1,4,100,69.6317720413,420,0.00603172930528,128,2909,true,1,0.0696317720413 +bf16,1,1,1,64,1,control_optimized_split,1,5,100,64.1318893433,420,0.00654900400255,128,2909,true,1,0.0641318893433 +bf16,1,1,1,64,1,optimized_split,1,6,100,33.7188577652,420,0.012455937948,128,2909,true,1,0.0337188577652 +bf16,1,1,1,64,1,control_shuffle32_split,1,7,100,94.5234298706,420,0.00444334278364,128,2909,true,1,0.0945234298706 +bf16,1,1,1,64,1,shuffle32_split,1,8,100,34.6914291382,420,0.0121067367483,128,2909,true,1,0.0346914291382 +bf16,1,1,1,64,1,baseline_fused,1,9,100,65.1378297806,164,0.00251773816463,128,2909,true,1,0.0651378297806 +bf16,1,1,1,64,1,control_optimized_fused,1,10,100,58.5188579559,164,0.00280251538954,128,2909,true,1,0.0585188579559 +bf16,1,1,1,64,1,optimized_fused,1,11,100,28.8066291809,164,0.00569313399947,128,2909,true,1,0.0288066291809 +bf16,1,1,1,64,1,control_shuffle32_fused,1,12,100,89.3714237213,164,0.00183503846276,128,2909,true,1,0.0893714237213 +bf16,1,1,1,64,1,shuffle32_fused,1,13,100,29.6338272095,164,0.00553421597692,128,2909,true,1,0.0296338272095 +bf16,1,1,1,64,1,baseline_transform,1,14,100,9.14811432362,256,0.0279839091362,128,2909,true,1,0.00914811432362 +bf16,1,1,1,64,1,optimized_transform,2,0,100,9.23405706882,256,0.0277234587237,128,2909,true,1,0.00923405706882 +bf16,1,1,1,64,1,control_shuffle32_transform,2,1,100,7.94994294643,256,0.0322014889572,128,2909,true,1,0.00794994294643 +bf16,1,1,1,64,1,shuffle32_transform,2,2,100,7.98834323883,256,0.0320466950839,128,2909,true,1,0.00798834323883 +bf16,1,1,1,64,1,baseline_split,2,3,100,69.5837688446,420,0.00603589036601,128,2909,true,1,0.0695837688446 +bf16,1,1,1,64,1,control_optimized_split,2,4,100,64.0834283829,420,0.00655395646891,128,2909,true,1,0.0640834283829 +bf16,1,1,1,64,1,optimized_split,2,5,100,34.0258288383,420,0.0123435641199,128,2909,true,1,0.0340258288383 +bf16,1,1,1,64,1,control_shuffle32_split,2,6,100,93.871088028,420,0.00447422107087,128,2909,true,1,0.093871088028 +bf16,1,1,1,64,1,shuffle32_split,2,7,100,34.8925709724,420,0.0120369462122,128,2909,true,1,0.0348925709724 +bf16,1,1,1,64,1,baseline_fused,2,8,100,65.1325702667,164,0.00251794147426,128,2909,true,1,0.0651325702667 +bf16,1,1,1,64,1,control_optimized_fused,2,9,100,58.5924577713,164,0.00279899506247,128,2909,true,1,0.0585924577713 +bf16,1,1,1,64,1,optimized_fused,2,10,100,28.7634301186,164,0.00570168437227,128,2909,true,1,0.0287634301186 +bf16,1,1,1,64,1,control_shuffle32_fused,2,11,100,89.3238830566,164,0.00183601512147,128,2909,true,1,0.0893238830566 +bf16,1,1,1,64,1,shuffle32_fused,2,12,100,29.6009135246,164,0.00554036955189,128,2909,true,1,0.0296009135246 +bf16,1,1,1,64,1,baseline_transform,2,13,100,9.0660572052,256,0.0282371922221,128,2909,true,1,0.0090660572052 +bf16,1,1,1,64,1,control_optimized_transform,2,14,100,9.47634279728,256,0.0270146411413,128,2909,true,1,0.00947634279728 +bf16,1,1,1,64,1,control_shuffle32_transform,3,0,100,8.01188588142,256,0.0319525270066,128,2909,true,1,0.00801188588142 +bf16,1,1,1,64,1,shuffle32_transform,3,1,100,7.99337148666,256,0.0320265360402,128,2909,true,1,0.00799337148666 +bf16,1,1,1,64,1,baseline_split,3,2,100,69.6409130096,420,0.00603093758897,128,2909,true,1,0.0696409130096 +bf16,1,1,1,64,1,control_optimized_split,3,3,100,64.1161155701,420,0.006550615181,128,2909,true,1,0.0641161155701 +bf16,1,1,1,64,1,optimized_split,3,4,100,33.789255619,420,0.0124299867607,128,2909,true,1,0.033789255619 +bf16,1,1,1,64,1,control_shuffle32_split,3,5,100,93.9037704468,420,0.00447266385579,128,2909,true,1,0.0939037704468 +bf16,1,1,1,64,1,shuffle32_split,3,6,100,34.7369146347,420,0.0120908838455,128,2909,true,1,0.0347369146347 +bf16,1,1,1,64,1,baseline_fused,3,7,100,65.1376008987,164,0.00251774701151,128,2909,true,1,0.0651376008987 +bf16,1,1,1,64,1,control_optimized_fused,3,8,100,58.5707426071,164,0.00280003279282,128,2909,true,1,0.0585707426071 +bf16,1,1,1,64,1,optimized_fused,3,9,100,28.8018274307,164,0.00569408314089,128,2909,true,1,0.0288018274307 +bf16,1,1,1,64,1,control_shuffle32_fused,3,10,100,89.3737125397,164,0.00183499146829,128,2909,true,1,0.0893737125397 +bf16,1,1,1,64,1,shuffle32_fused,3,11,100,29.6948575974,164,0.00552284177361,128,2909,true,1,0.0296948575974 +bf16,1,1,1,64,1,baseline_transform,3,12,100,9.03611421585,256,0.0283307618612,128,2909,true,1,0.00903611421585 +bf16,1,1,1,64,1,control_optimized_transform,3,13,100,9.32457149029,256,0.0274543447135,128,2909,true,1,0.00932457149029 +bf16,1,1,1,64,1,optimized_transform,3,14,100,9.23702836037,256,0.0277145408688,128,2909,true,1,0.00923702836037 +bf16,1,1,1,64,1,shuffle32_transform,4,0,100,7.96800017357,256,0.0321285133564,128,2909,true,1,0.00796800017357 +bf16,1,1,1,64,1,baseline_split,4,1,100,69.6086835861,420,0.00603372996532,128,2909,true,1,0.0696086835861 +bf16,1,1,1,64,1,control_optimized_split,4,2,100,64.1661691666,420,0.00654550529438,128,2909,true,1,0.0641661691666 +bf16,1,1,1,64,1,optimized_split,4,3,100,33.7835431099,420,0.0124320885655,128,2909,true,1,0.0337835431099 +bf16,1,1,1,64,1,control_shuffle32_split,4,4,100,93.9058303833,420,0.00447256574257,128,2909,true,1,0.0939058303833 +bf16,1,1,1,64,1,shuffle32_split,4,5,100,35.1469707489,420,0.0119498207399,128,2909,true,1,0.0351469707489 +bf16,1,1,1,64,1,baseline_fused,4,6,100,65.1316595078,164,0.00251797668353,128,2909,true,1,0.0651316595078 +bf16,1,1,1,64,1,control_optimized_fused,4,7,100,58.5595417023,164,0.00280056836568,128,2909,true,1,0.0585595417023 +bf16,1,1,1,64,1,optimized_fused,4,8,100,28.7597727776,164,0.00570240944768,128,2909,true,1,0.0287597727776 +bf16,1,1,1,64,1,control_shuffle32_fused,4,9,100,89.3776035309,164,0.00183491158323,128,2909,true,1,0.0893776035309 +bf16,1,1,1,64,1,shuffle32_fused,4,10,100,29.6038866043,164,0.00553981313981,128,2909,true,1,0.0296038866043 +bf16,1,1,1,64,1,baseline_transform,4,11,100,9.09348547459,256,0.028152021655,128,2909,true,1,0.00909348547459 +bf16,1,1,1,64,1,control_optimized_transform,4,12,100,9.44251418114,256,0.0271114234079,128,2909,true,1,0.00944251418114 +bf16,1,1,1,64,1,optimized_transform,4,13,100,9.26034271717,256,0.0276447651905,128,2909,true,1,0.00926034271717 +bf16,1,1,1,64,1,control_shuffle32_transform,4,14,100,7.96937167645,256,0.032122984144,128,2909,true,1,0.00796937167645 +bf16,1,17,1,64,17,baseline_transform,0,0,100,9.30377185345,4352,0.467767274236,2176,2909,true,1,0.00930377185345 +bf16,1,17,1,64,17,control_optimized_transform,0,1,100,9.43702876568,4352,0.461162099646,2176,2909,true,1,0.00943702876568 +bf16,1,17,1,64,17,optimized_transform,0,2,100,9.45051431656,4352,0.460504037582,2176,2909,true,1,0.00945051431656 +bf16,1,17,1,64,17,control_shuffle32_transform,0,3,100,7.91474282742,4352,0.549859937953,2176,2909,true,1,0.00791474282742 +bf16,1,17,1,64,17,shuffle32_transform,0,4,100,7.94445693493,4352,0.547803334532,2176,2909,true,1,0.00794445693493 +bf16,1,17,1,64,17,baseline_split,0,5,100,70.2034282684,7140,0.101704434899,2176,2909,true,1,0.0702034282684 +bf16,1,17,1,64,17,control_optimized_split,0,6,100,64.4493722916,7140,0.110784632125,2176,2909,true,1,0.0644493722916 +bf16,1,17,1,64,17,optimized_split,0,7,100,34.2221713066,7140,0.208636674045,2176,2909,true,1,0.0342221713066 +bf16,1,17,1,64,17,control_shuffle32_split,0,8,100,94.5042228699,7140,0.0755521793966,2176,2909,true,1,0.0945042228699 +bf16,1,17,1,64,17,shuffle32_split,0,9,100,34.5796585083,7140,0.206479771866,2176,2909,true,1,0.0345796585083 +bf16,1,17,1,64,17,baseline_fused,0,10,100,65.4454851151,2788,0.0426003412626,2176,2909,true,1,0.0654454851151 +bf16,1,17,1,64,17,control_optimized_fused,0,11,100,58.8185167313,2788,0.0474000392213,2176,2909,true,1,0.0588185167313 +bf16,1,17,1,64,17,optimized_fused,0,12,100,28.9307427406,2788,0.0963680754758,2176,2909,true,1,0.0289307427406 +bf16,1,17,1,64,17,control_shuffle32_fused,0,13,100,89.0660572052,2788,0.0313026094057,2176,2909,true,1,0.0890660572052 +bf16,1,17,1,64,17,shuffle32_fused,0,14,100,29.7051429749,2788,0.0938558014132,2176,2909,true,1,0.0297051429749 +bf16,1,17,1,64,17,control_optimized_transform,1,0,100,9.59200024605,4352,0.45371141455,2176,2909,true,1,0.00959200024605 +bf16,1,17,1,64,17,optimized_transform,1,1,100,9.51931416988,4352,0.457175792534,2176,2909,true,1,0.00951931416988 +bf16,1,17,1,64,17,control_shuffle32_transform,1,2,100,8.07657122612,4352,0.538842520936,2176,2909,true,1,0.00807657122612 +bf16,1,17,1,64,17,shuffle32_transform,1,3,100,8.11725735664,4352,0.53614168047,2176,2909,true,1,0.00811725735664 +bf16,1,17,1,64,17,baseline_split,1,4,100,70.1853704453,7140,0.101730602185,2176,2909,true,1,0.0701853704453 +bf16,1,17,1,64,17,control_optimized_split,1,5,100,64.4443416595,7140,0.110793280157,2176,2909,true,1,0.0644443416595 +bf16,1,17,1,64,17,optimized_split,1,6,100,34.3364572525,7140,0.207942244813,2176,2909,true,1,0.0343364572525 +bf16,1,17,1,64,17,control_shuffle32_split,1,7,100,94.5286846161,7140,0.0755326283127,2176,2909,true,1,0.0945286846161 +bf16,1,17,1,64,17,shuffle32_split,1,8,100,34.6249127388,7140,0.206209905967,2176,2909,true,1,0.0346249127388 +bf16,1,17,1,64,17,baseline_fused,1,9,100,65.4489135742,2788,0.0425981096972,2176,2909,true,1,0.0654489135742 +bf16,1,17,1,64,17,control_optimized_fused,1,10,100,58.8237714767,2788,0.0473958049613,2176,2909,true,1,0.0588237714767 +bf16,1,17,1,64,17,optimized_fused,1,11,100,28.9714288712,2788,0.0962327406218,2176,2909,true,1,0.0289714288712 +bf16,1,17,1,64,17,control_shuffle32_fused,1,12,100,89.1108512878,2788,0.0312868742662,2176,2909,true,1,0.0891108512878 +bf16,1,17,1,64,17,shuffle32_fused,1,13,100,29.7035431862,2788,0.0938608563471,2176,2909,true,1,0.0297035431862 +bf16,1,17,1,64,17,baseline_transform,1,14,100,9.34925675392,4352,0.465491548103,2176,2909,true,1,0.00934925675392 +bf16,1,17,1,64,17,optimized_transform,2,0,100,9.60114300251,4352,0.45327936464,2176,2909,true,1,0.00960114300251 +bf16,1,17,1,64,17,control_shuffle32_transform,2,1,100,8.02971422672,4352,0.54198690976,2176,2909,true,1,0.00802971422672 +bf16,1,17,1,64,17,shuffle32_transform,2,2,100,8.08868527412,4352,0.538035521536,2176,2909,true,1,0.00808868527412 +bf16,1,17,1,64,17,baseline_split,2,3,100,70.2052545547,7140,0.101701789208,2176,2909,true,1,0.0702052545547 +bf16,1,17,1,64,17,control_optimized_split,2,4,100,64.4962310791,7140,0.110704143181,2176,2909,true,1,0.0644962310791 +bf16,1,17,1,64,17,optimized_split,2,5,100,34.3821716309,7140,0.207665765754,2176,2909,true,1,0.0343821716309 +bf16,1,17,1,64,17,control_shuffle32_split,2,6,100,94.4703960419,7140,0.0755792322161,2176,2909,true,1,0.0944703960419 +bf16,1,17,1,64,17,shuffle32_split,2,7,100,34.8875427246,7140,0.204657578103,2176,2909,true,1,0.0348875427246 +bf16,1,17,1,64,17,baseline_fused,2,8,100,65.4626274109,2788,0.0425891857731,2176,2909,true,1,0.0654626274109 +bf16,1,17,1,64,17,control_optimized_fused,2,9,100,58.8557720184,2788,0.0473700353319,2176,2909,true,1,0.0588557720184 +bf16,1,17,1,64,17,optimized_fused,2,10,100,28.9428567886,2788,0.0963277405669,2176,2909,true,1,0.0289428567886 +bf16,1,17,1,64,17,control_shuffle32_fused,2,11,100,89.0614795685,2788,0.0313042183165,2176,2909,true,1,0.0890614795685 +bf16,1,17,1,64,17,shuffle32_fused,2,12,100,29.703772068,2788,0.0938601331041,2176,2909,true,1,0.029703772068 +bf16,1,17,1,64,17,baseline_transform,2,13,100,9.32799994946,4352,0.466552318137,2176,2909,true,1,0.00932799994946 +bf16,1,17,1,64,17,control_optimized_transform,2,14,100,9.56342875957,4352,0.455066912653,2176,2909,true,1,0.00956342875957 +bf16,1,17,1,64,17,control_shuffle32_transform,3,0,100,8.00320029259,4352,0.543782467125,2176,2909,true,1,0.00800320029259 +bf16,1,17,1,64,17,shuffle32_transform,3,1,100,8.08251440525,4352,0.538446302944,2176,2909,true,1,0.00808251440525 +bf16,1,17,1,64,17,baseline_split,3,2,100,70.2182865143,7140,0.101682914159,2176,2909,true,1,0.0702182865143 +bf16,1,17,1,64,17,control_optimized_split,3,3,100,64.5739459991,7140,0.110570910443,2176,2909,true,1,0.0645739459991 +bf16,1,17,1,64,17,optimized_split,3,4,100,34.5286870003,7140,0.206784578862,2176,2909,true,1,0.0345286870003 +bf16,1,17,1,64,17,control_shuffle32_split,3,5,100,93.9147472382,7140,0.0760263985154,2176,2909,true,1,0.0939147472382 +bf16,1,17,1,64,17,shuffle32_split,3,6,100,34.7506284714,7140,0.20546390998,2176,2909,true,1,0.0347506284714 +bf16,1,17,1,64,17,baseline_fused,3,7,100,65.4550886154,2788,0.0425940909863,2176,2909,true,1,0.0654550886154 +bf16,1,17,1,64,17,control_optimized_fused,3,8,100,58.8228559494,2788,0.0473965426364,2176,2909,true,1,0.0588228559494 +bf16,1,17,1,64,17,optimized_fused,3,9,100,28.9346265793,2788,0.0963551401764,2176,2909,true,1,0.0289346265793 +bf16,1,17,1,64,17,control_shuffle32_fused,3,10,100,89.0658283234,2788,0.0313026898473,2176,2909,true,1,0.0890658283234 +bf16,1,17,1,64,17,shuffle32_fused,3,11,100,29.6884560585,2788,0.0939085547091,2176,2909,true,1,0.0296884560585 +bf16,1,17,1,64,17,baseline_transform,3,12,100,9.42651391029,4352,0.461676505378,2176,2909,true,1,0.00942651391029 +bf16,1,17,1,64,17,control_optimized_transform,3,13,100,9.51017141342,4352,0.457615305846,2176,2909,true,1,0.00951017141342 +bf16,1,17,1,64,17,optimized_transform,3,14,100,9.51474308968,4352,0.457395429281,2176,2909,true,1,0.00951474308968 +bf16,1,17,1,64,17,shuffle32_transform,4,0,100,8.14674317837,4352,0.534201202212,2176,2909,true,1,0.00814674317837 +bf16,1,17,1,64,17,baseline_split,4,1,100,70.2011442184,7140,0.101707743933,2176,2909,true,1,0.0702011442184 +bf16,1,17,1,64,17,control_optimized_split,4,2,100,64.4548559189,7140,0.110775206898,2176,2909,true,1,0.0644548559189 +bf16,1,17,1,64,17,optimized_split,4,3,100,34.3019413948,7140,0.208151483842,2176,2909,true,1,0.0343019413948 +bf16,1,17,1,64,17,control_shuffle32_split,4,4,100,93.8745117188,7140,0.0760589841617,2176,2909,true,1,0.0938745117187 +bf16,1,17,1,64,17,shuffle32_split,4,5,100,34.5938277245,7140,0.206395200233,2176,2909,true,1,0.0345938277245 +bf16,1,17,1,64,17,baseline_fused,4,6,100,65.4534864426,2788,0.042595133606,2176,2909,true,1,0.0654534864426 +bf16,1,17,1,64,17,control_optimized_fused,4,7,100,58.8191986084,2788,0.0473994897238,2176,2909,true,1,0.0588191986084 +bf16,1,17,1,64,17,optimized_fused,4,8,100,28.9419436455,2788,0.0963307797898,2176,2909,true,1,0.0289419436455 +bf16,1,17,1,64,17,control_shuffle32_fused,4,9,100,89.1168022156,2788,0.0312847850314,2176,2909,true,1,0.0891168022156 +bf16,1,17,1,64,17,shuffle32_fused,4,10,100,29.7133708,2788,0.0938298121329,2176,2909,true,1,0.0297133708 +bf16,1,17,1,64,17,baseline_transform,4,11,100,9.24822866917,4352,0.470576599658,2176,2909,true,1,0.00924822866917 +bf16,1,17,1,64,17,control_optimized_transform,4,12,100,9.42719995975,4352,0.461642907606,2176,2909,true,1,0.00942719995975 +bf16,1,17,1,64,17,optimized_transform,4,13,100,9.51405704021,4352,0.457428411623,2176,2909,true,1,0.00951405704021 +bf16,1,17,1,64,17,control_shuffle32_transform,4,14,100,7.98628568649,4352,0.544934174764,2176,2909,true,1,0.00798628568649 +bf16,1,257,1,64,257,baseline_transform,0,0,100,9.51725721359,65792,6.91291603489,32896,2909,true,1,0.00951725721359 +bf16,1,257,1,64,257,control_optimized_transform,0,1,100,9.61714327335,65792,6.84111675681,32896,2909,true,1,0.00961714327335 +bf16,1,257,1,64,257,optimized_transform,0,2,100,9.6303999424,65792,6.83169965874,32896,2909,true,1,0.0096303999424 +bf16,1,257,1,64,257,control_shuffle32_transform,0,3,100,8.92434239388,65792,7.37219585447,32896,2909,true,1,0.00892434239388 +bf16,1,257,1,64,257,shuffle32_transform,0,4,100,8.9275431633,65792,7.36955271977,32896,2909,true,1,0.0089275431633 +bf16,1,257,1,64,257,baseline_split,0,5,100,70.8717727661,107940,1.52303231297,32896,2909,true,1,0.0708717727661 +bf16,1,257,1,64,257,control_optimized_split,0,6,100,64.5894813538,107940,1.67116994498,32896,2909,true,1,0.0645894813538 +bf16,1,257,1,64,257,optimized_split,0,7,100,34.6644568443,107940,3.11385233828,32896,2909,true,1,0.0346644568443 +bf16,1,257,1,64,257,control_shuffle32_split,0,8,100,103.026514053,107940,1.04769147041,32896,2909,true,1,0.103026514053 +bf16,1,257,1,64,257,shuffle32_split,0,9,100,39.3981719017,107940,2.73972102739,32896,2909,true,1,0.0393981719017 +bf16,1,257,1,64,257,baseline_fused,0,10,100,65.9250259399,42148,0.639332323333,32896,2909,true,1,0.0659250259399 +bf16,1,257,1,64,257,control_optimized_fused,0,11,100,59.1803455353,42148,0.71219590928,32896,2909,true,1,0.0591803455353 +bf16,1,257,1,64,257,optimized_fused,0,12,100,29.1650271416,42148,1.44515552121,32896,2909,true,1,0.0291650271416 +bf16,1,257,1,64,257,control_shuffle32_fused,0,13,100,95.9741687775,42148,0.439159833702,32896,2909,true,1,0.0959741687775 +bf16,1,257,1,64,257,shuffle32_fused,0,14,100,33.3515429497,42148,1.26374962812,32896,2909,true,1,0.0333515429497 +bf16,1,257,1,64,257,control_optimized_transform,1,0,100,9.56914246082,65792,6.87543322397,32896,2909,true,1,0.00956914246082 +bf16,1,257,1,64,257,optimized_transform,1,1,100,9.67702805996,65792,6.79878156727,32896,2909,true,1,0.00967702805996 +bf16,1,257,1,64,257,control_shuffle32_transform,1,2,100,9.01120007038,65792,7.30113630661,32896,2909,true,1,0.00901120007038 +bf16,1,257,1,64,257,shuffle32_transform,1,3,100,8.95017147064,65792,7.3509206182,32896,2909,true,1,0.00895017147064 +bf16,1,257,1,64,257,baseline_split,1,4,100,70.9149694443,107940,1.52210458308,32896,2909,true,1,0.0709149694443 +bf16,1,257,1,64,257,control_optimized_split,1,5,100,64.6258306503,107940,1.67022998256,32896,2909,true,1,0.0646258306503 +bf16,1,257,1,64,257,optimized_split,1,6,100,34.6633148193,107940,3.11395492793,32896,2909,true,1,0.0346633148193 +bf16,1,257,1,64,257,control_shuffle32_split,1,7,100,102.975997925,107940,1.04820542821,32896,2909,true,1,0.102975997925 +bf16,1,257,1,64,257,shuffle32_split,1,8,100,39.3967986107,107940,2.73981652841,32896,2909,true,1,0.0393967986107 +bf16,1,257,1,64,257,baseline_fused,1,9,100,65.9261751175,42148,0.639321178953,32896,2909,true,1,0.0659261751175 +bf16,1,257,1,64,257,control_optimized_fused,1,10,100,59.1871976852,42148,0.712113457781,32896,2909,true,1,0.0591871976852 +bf16,1,257,1,64,257,optimized_fused,1,11,100,29.1823983192,42148,1.44429527481,32896,2909,true,1,0.0291823983192 +bf16,1,257,1,64,257,control_shuffle32_fused,1,12,100,95.9993171692,42148,0.439044789514,32896,2909,true,1,0.0959993171692 +bf16,1,257,1,64,257,shuffle32_fused,1,13,100,33.2848000526,42148,1.26628370708,32896,2909,true,1,0.0332848000526 +bf16,1,257,1,64,257,baseline_transform,1,14,100,9.61394309998,65792,6.84339394521,32896,2909,true,1,0.00961394309998 +bf16,1,257,1,64,257,optimized_transform,2,0,100,9.64571416378,65792,6.82085316679,32896,2909,true,1,0.00964571416378 +bf16,1,257,1,64,257,control_shuffle32_transform,2,1,100,8.9003431797,65792,7.39207451574,32896,2909,true,1,0.0089003431797 +bf16,1,257,1,64,257,shuffle32_transform,2,2,100,8.93485724926,65792,7.36351999417,32896,2909,true,1,0.00893485724926 +bf16,1,257,1,64,257,baseline_split,2,3,100,70.8806848526,107940,1.52284081657,32896,2909,true,1,0.0708806848526 +bf16,1,257,1,64,257,control_optimized_split,2,4,100,64.6642255783,107940,1.66923826946,32896,2909,true,1,0.0646642255783 +bf16,1,257,1,64,257,optimized_split,2,5,100,34.6377134323,107940,3.11625651072,32896,2909,true,1,0.0346377134323 +bf16,1,257,1,64,257,control_shuffle32_split,2,6,100,102.357025146,107940,1.05454412968,32896,2909,true,1,0.102357025146 +bf16,1,257,1,64,257,shuffle32_split,2,7,100,39.5309686661,107940,2.73051745612,32896,2909,true,1,0.0395309686661 +bf16,1,257,1,64,257,baseline_fused,2,8,100,65.9165716171,42148,0.639414322772,32896,2909,true,1,0.0659165716171 +bf16,1,257,1,64,257,control_optimized_fused,2,9,100,59.1865158081,42148,0.712121661911,32896,2909,true,1,0.0591865158081 +bf16,1,257,1,64,257,optimized_fused,2,10,100,29.1364574432,42148,1.44657256573,32896,2909,true,1,0.0291364574432 +bf16,1,257,1,64,257,control_shuffle32_fused,2,11,100,95.9846878052,42148,0.439111705875,32896,2909,true,1,0.0959846878052 +bf16,1,257,1,64,257,shuffle32_fused,2,12,100,33.2973718643,42148,1.26580560687,32896,2909,true,1,0.0332973718643 +bf16,1,257,1,64,257,baseline_transform,2,13,100,9.63931441307,65792,6.82538167972,32896,2909,true,1,0.00963931441307 +bf16,1,257,1,64,257,control_optimized_transform,2,14,100,9.61028575897,65792,6.84599830328,32896,2909,true,1,0.00961028575897 +bf16,1,257,1,64,257,control_shuffle32_transform,3,0,100,9.03565704823,65792,7.28137418772,32896,2909,true,1,0.00903565704823 +bf16,1,257,1,64,257,shuffle32_transform,3,1,100,8.98674309254,65792,7.32100598877,32896,2909,true,1,0.00898674309254 +bf16,1,257,1,64,257,baseline_split,3,2,100,70.8676528931,107940,1.52312085406,32896,2909,true,1,0.0708676528931 +bf16,1,257,1,64,257,control_optimized_split,3,3,100,64.6272039413,107940,1.67019449113,32896,2909,true,1,0.0646272039413 +bf16,1,257,1,64,257,optimized_split,3,4,100,34.6914291382,107940,3.11143134433,32896,2909,true,1,0.0346914291382 +bf16,1,257,1,64,257,control_shuffle32_split,3,5,100,102.98743248,107940,1.04808904738,32896,2909,true,1,0.10298743248 +bf16,1,257,1,64,257,shuffle32_split,3,6,100,39.4706296921,107940,2.73469161354,32896,2909,true,1,0.0394706296921 +bf16,1,257,1,64,257,baseline_fused,3,7,100,65.9236574173,42148,0.639345595363,32896,2909,true,1,0.0659236574173 +bf16,1,257,1,64,257,control_optimized_fused,3,8,100,59.1824007034,42148,0.712171177564,32896,2909,true,1,0.0591824007034 +bf16,1,257,1,64,257,optimized_fused,3,9,100,29.1572570801,42148,1.44554063794,32896,2909,true,1,0.0291572570801 +bf16,1,257,1,64,257,control_shuffle32_fused,3,10,100,95.9906291962,42148,0.439084526823,32896,2909,true,1,0.0959906291962 +bf16,1,257,1,64,257,shuffle32_fused,3,11,100,33.2861709595,42148,1.26623155458,32896,2909,true,1,0.0332861709595 +bf16,1,257,1,64,257,baseline_transform,3,12,100,9.59839999676,65792,6.85447574827,32896,2909,true,1,0.00959839999676 +bf16,1,257,1,64,257,control_optimized_transform,3,13,100,9.61348593235,65792,6.8437193816,32896,2909,true,1,0.00961348593235 +bf16,1,257,1,64,257,optimized_transform,3,14,100,9.61897134781,65792,6.83981661043,32896,2909,true,1,0.00961897134781 +bf16,1,257,1,64,257,shuffle32_transform,4,0,100,9.02971446514,65792,7.28616616328,32896,2909,true,1,0.00902971446514 +bf16,1,257,1,64,257,baseline_split,4,1,100,70.896229744,107940,1.52250691454,32896,2909,true,1,0.070896229744 +bf16,1,257,1,64,257,control_optimized_split,4,2,100,64.5807981491,107940,1.67139464196,32896,2909,true,1,0.0645807981491 +bf16,1,257,1,64,257,optimized_split,4,3,100,34.7156572342,107940,3.10925987291,32896,2909,true,1,0.0347156572342 +bf16,1,257,1,64,257,control_shuffle32_split,4,4,100,102.340803146,107940,1.05471128505,32896,2909,true,1,0.102340803146 +bf16,1,257,1,64,257,shuffle32_split,4,5,100,39.3490314484,107940,2.7431424873,32896,2909,true,1,0.0393490314484 +bf16,1,257,1,64,257,baseline_fused,4,6,100,65.919084549,42148,0.639389947363,32896,2909,true,1,0.065919084549 +bf16,1,257,1,64,257,control_optimized_fused,4,7,100,59.1837692261,42148,0.712154709833,32896,2909,true,1,0.0591837692261 +bf16,1,257,1,64,257,optimized_fused,4,8,100,29.1355419159,42148,1.44661802144,32896,2909,true,1,0.0291355419159 +bf16,1,257,1,64,257,control_shuffle32_fused,4,9,100,95.986738205,42148,0.439102325886,32896,2909,true,1,0.095986738205 +bf16,1,257,1,64,257,shuffle32_fused,4,10,100,33.2354283333,42148,1.26816479022,32896,2909,true,1,0.0332354283333 +bf16,1,257,1,64,257,baseline_transform,4,11,100,9.51405704021,65792,6.9152412816,32896,2909,true,1,0.00951405704021 +bf16,1,257,1,64,257,control_optimized_transform,4,12,100,9.60754275322,65792,6.84795287306,32896,2909,true,1,0.00960754275322 +bf16,1,257,1,64,257,optimized_transform,4,13,100,9.61005687714,65792,6.84616135379,32896,2909,true,1,0.00961005687714 +bf16,1,257,1,64,257,control_shuffle32_transform,4,14,100,9.02377128601,65792,7.29096493192,32896,2909,true,1,0.00902377128601 +bf16,4,64,16,64,4096,baseline_transform,0,0,100,23.1785130501,1048576,45.2391401353,524288,2909,true,1,0.0231785130501 +bf16,4,64,16,64,4096,control_optimized_transform,0,1,100,23.9766860008,1048576,43.7331497757,524288,2909,true,1,0.0239766860008 +bf16,4,64,16,64,4096,optimized_transform,0,2,100,24.0121150017,1048576,43.6686231066,524288,2909,true,1,0.0240121150017 +bf16,4,64,16,64,4096,control_shuffle32_transform,0,3,100,11.2308573723,1048576,93.3656234107,524288,2909,true,1,0.0112308573723 +bf16,4,64,16,64,4096,shuffle32_transform,0,4,100,11.2272000313,1048576,93.3960379327,524288,2909,true,1,0.0112272000313 +bf16,4,64,16,64,4096,baseline_split,0,5,100,437.782859802,1720320,3.9296193569,524288,2909,true,1,0.437782859802 +bf16,4,64,16,64,4096,control_optimized_split,0,6,100,380.768699646,1720320,4.51801842326,524288,2909,true,1,0.380768699646 +bf16,4,64,16,64,4096,optimized_split,0,7,100,162.241611481,1720320,10.6034449751,524288,2909,true,1,0.162241611481 +bf16,4,64,16,64,4096,control_shuffle32_split,0,8,100,248.62537384,1720320,6.91932594581,524288,2909,true,1,0.24862537384 +bf16,4,64,16,64,4096,shuffle32_split,0,9,100,72.8886842728,1720320,23.6020174759,524288,2909,true,1,0.0728886842728 +bf16,4,64,16,64,4096,baseline_fused,0,10,100,430.543327332,671744,1.56022392488,524288,2909,true,1,0.430543327332 +bf16,4,64,16,64,4096,control_optimized_fused,0,11,100,373.686180115,671744,1.79761531399,524288,2909,true,1,0.373686180115 +bf16,4,64,16,64,4096,optimized_fused,0,12,100,148.698291779,671744,4.51749641482,524288,2909,true,1,0.148698291779 +bf16,4,64,16,64,4096,control_shuffle32_fused,0,13,100,247.959766388,671744,2.70908466234,524288,2909,true,1,0.247959766388 +bf16,4,64,16,64,4096,shuffle32_fused,0,14,100,66.7499446869,671744,10.0635888636,524288,2909,true,1,0.0667499446869 +bf16,4,64,16,64,4096,control_optimized_transform,1,0,100,23.941257,1048576,43.797867422,524288,2909,true,1,0.023941257 +bf16,4,64,16,64,4096,optimized_transform,1,1,100,24.0477705002,1048576,43.6038758766,524288,2909,true,1,0.0240477705002 +bf16,4,64,16,64,4096,control_shuffle32_transform,1,2,100,11.393828392,1048576,92.0301731711,524288,2909,true,1,0.011393828392 +bf16,4,64,16,64,4096,shuffle32_transform,1,3,100,11.3929140568,1048576,92.0375590279,524288,2909,true,1,0.0113929140568 +bf16,4,64,16,64,4096,baseline_split,1,4,100,438.460540771,1720320,3.92354576987,524288,2909,true,1,0.438460540771 +bf16,4,64,16,64,4096,control_optimized_split,1,5,100,381.882972717,1720320,4.50483557242,524288,2909,true,1,0.381882972717 +bf16,4,64,16,64,4096,optimized_split,1,6,100,162.025604248,1720320,10.6175811409,524288,2909,true,1,0.162025604248 +bf16,4,64,16,64,4096,control_shuffle32_split,1,7,100,248.804569244,1720320,6.91434247058,524288,2909,true,1,0.248804569244 +bf16,4,64,16,64,4096,shuffle32_split,1,8,100,73.7785148621,1720320,23.317357407,524288,2909,true,1,0.0737785148621 +bf16,4,64,16,64,4096,baseline_fused,1,9,100,429.654388428,671744,1.56345196998,524288,2909,true,1,0.429654388428 +bf16,4,64,16,64,4096,control_optimized_fused,1,10,100,371.614379883,671744,1.80763726154,524288,2909,true,1,0.371614379883 +bf16,4,64,16,64,4096,optimized_fused,1,11,100,148.793144226,671744,4.51461660747,524288,2909,true,1,0.148793144226 +bf16,4,64,16,64,4096,control_shuffle32_fused,1,12,100,245.291423798,671744,2.73855477538,524288,2909,true,1,0.245291423798 +bf16,4,64,16,64,4096,shuffle32_fused,1,13,100,65.9780597687,671744,10.1813239485,524288,2909,true,1,0.0659780597687 +bf16,4,64,16,64,4096,baseline_transform,1,14,100,22.7668571472,1048576,46.0571256375,524288,2909,true,1,0.0227668571472 +bf16,4,64,16,64,4096,optimized_transform,2,0,100,24.028570652,1048576,43.6387172248,524288,2909,true,1,0.024028570652 +bf16,4,64,16,64,4096,control_shuffle32_transform,2,1,100,11.3044571877,1048576,92.7577487883,524288,2909,true,1,0.0113044571877 +bf16,4,64,16,64,4096,shuffle32_transform,2,2,100,11.1810278893,1048576,93.7817176011,524288,2909,true,1,0.0111810278893 +bf16,4,64,16,64,4096,baseline_split,2,3,100,438.608665466,1720320,3.92222072989,524288,2909,true,1,0.438608665466 +bf16,4,64,16,64,4096,control_optimized_split,2,4,100,381.776237488,1720320,4.50609501346,524288,2909,true,1,0.381776237488 +bf16,4,64,16,64,4096,optimized_split,2,5,100,162.238407135,1720320,10.6036544021,524288,2909,true,1,0.162238407135 +bf16,4,64,16,64,4096,control_shuffle32_split,2,6,100,248.956794739,1720320,6.91011467193,524288,2909,true,1,0.248956794739 +bf16,4,64,16,64,4096,shuffle32_split,2,7,100,71.8374872208,1720320,23.9473855024,524288,2909,true,1,0.0718374872208 +bf16,4,64,16,64,4096,baseline_fused,2,8,100,430.838890076,671744,1.55915358496,524288,2909,true,1,0.430838890076 +bf16,4,64,16,64,4096,control_optimized_fused,2,9,100,371.277275085,671744,1.8092785233,524288,2909,true,1,0.371277275085 +bf16,4,64,16,64,4096,optimized_fused,2,10,100,147.893257141,671744,4.54208672515,524288,2909,true,1,0.147893257141 +bf16,4,64,16,64,4096,control_shuffle32_fused,2,11,100,244.293022156,671744,2.74974698038,524288,2909,true,1,0.244293022156 +bf16,4,64,16,64,4096,shuffle32_fused,2,12,100,68.570971489,671744,9.79633196692,524288,2909,true,1,0.068570971489 +bf16,4,64,16,64,4096,baseline_transform,2,13,100,22.7350854874,1048576,46.1214892103,524288,2909,true,1,0.0227350854874 +bf16,4,64,16,64,4096,control_optimized_transform,2,14,100,24.0038847923,1048576,43.6835957626,524288,2909,true,1,0.0240038847923 +bf16,4,64,16,64,4096,control_shuffle32_transform,3,0,100,11.2381720543,1048576,93.3048537551,524288,2909,true,1,0.0112381720543 +bf16,4,64,16,64,4096,shuffle32_transform,3,1,100,11.3250279427,1048576,92.5892638243,524288,2909,true,1,0.0113250279427 +bf16,4,64,16,64,4096,baseline_split,3,2,100,437.733726501,1720320,3.93006043594,524288,2909,true,1,0.437733726501 +bf16,4,64,16,64,4096,control_optimized_split,3,3,100,381.440658569,1720320,4.51005932732,524288,2909,true,1,0.381440658569 +bf16,4,64,16,64,4096,optimized_split,3,4,100,162.18170166,1720320,10.6073618811,524288,2909,true,1,0.16218170166 +bf16,4,64,16,64,4096,control_shuffle32_split,3,5,100,247.650279999,1720320,6.94656997767,524288,2909,true,1,0.247650279999 +bf16,4,64,16,64,4096,shuffle32_split,3,6,100,73.193602562,1720320,23.5036934894,524288,2909,true,1,0.073193602562 +bf16,4,64,16,64,4096,baseline_fused,3,7,100,429.859199524,671744,1.56270704627,524288,2909,true,1,0.429859199524 +bf16,4,64,16,64,4096,control_optimized_fused,3,8,100,373.322753906,671744,1.79936527568,524288,2909,true,1,0.373322753906 +bf16,4,64,16,64,4096,optimized_fused,3,9,100,148.427429199,671744,4.5257403138,524288,2909,true,1,0.148427429199 +bf16,4,64,16,64,4096,control_shuffle32_fused,3,10,100,247.973480225,671744,2.70893484009,524288,2909,true,1,0.247973480225 +bf16,4,64,16,64,4096,shuffle32_fused,3,11,100,65.2397680283,671744,10.2965418226,524288,2909,true,1,0.0652397680283 +bf16,4,64,16,64,4096,baseline_transform,3,12,100,22.7673149109,1048576,46.0561996047,524288,2909,true,1,0.0227673149109 +bf16,4,64,16,64,4096,control_optimized_transform,3,13,100,23.9332580566,1048576,43.81250549,524288,2909,true,1,0.0239332580566 +bf16,4,64,16,64,4096,optimized_transform,3,14,100,24.0313148499,1048576,43.6337340071,524288,2909,true,1,0.0240313148499 +bf16,4,64,16,64,4096,shuffle32_transform,4,0,100,11.093941927,1048576,94.5178915577,524288,2909,true,1,0.011093941927 +bf16,4,64,16,64,4096,baseline_split,4,1,100,438.199081421,1720320,3.92588682391,524288,2909,true,1,0.438199081421 +bf16,4,64,16,64,4096,control_optimized_split,4,2,100,381.525726318,1720320,4.50905373171,524288,2909,true,1,0.381525726318 +bf16,4,64,16,64,4096,optimized_split,4,3,100,162.075195312,1720320,10.6143324195,524288,2909,true,1,0.162075195313 +bf16,4,64,16,64,4096,control_shuffle32_split,4,4,100,249.119777679,1720320,6.90559383131,524288,2909,true,1,0.249119777679 +bf16,4,64,16,64,4096,shuffle32_split,4,5,100,71.8555402756,1720320,23.9413689383,524288,2909,true,1,0.0718555402756 +bf16,4,64,16,64,4096,baseline_fused,4,6,100,429.232673645,671744,1.56498803853,524288,2909,true,1,0.429232673645 +bf16,4,64,16,64,4096,control_optimized_fused,4,7,100,372.549705505,671744,1.80309899612,524288,2909,true,1,0.372549705505 +bf16,4,64,16,64,4096,optimized_fused,4,8,100,148.411426544,671744,4.52622830763,524288,2909,true,1,0.148411426544 +bf16,4,64,16,64,4096,control_shuffle32_fused,4,9,100,247.943305969,671744,2.7092645126,524288,2909,true,1,0.247943305969 +bf16,4,64,16,64,4096,shuffle32_fused,4,10,100,67.6781749725,671744,9.9255631564,524288,2909,true,1,0.0676781749725 +bf16,4,64,16,64,4096,baseline_transform,4,11,100,22.696685791,1048576,46.1995204787,524288,2909,true,1,0.022696685791 +bf16,4,64,16,64,4096,control_optimized_transform,4,12,100,23.9510846138,1048576,43.7798962722,524288,2909,true,1,0.0239510846138 +bf16,4,64,16,64,4096,optimized_transform,4,13,100,24.0372562408,1048576,43.6229488713,524288,2909,true,1,0.0240372562408 +bf16,4,64,16,64,4096,control_shuffle32_transform,4,14,100,11.4719998837,1048576,91.4030692673,524288,2909,true,1,0.0114719998837 +bf16,16,64,16,64,16384,baseline_transform,0,0,100,68.1702852249,4194304,61.5268659382,2097152,2909,true,1,0.0681702852249 +bf16,16,64,16,64,16384,control_optimized_transform,0,1,100,73.8978290558,4194304,56.7581491039,2097152,2909,true,1,0.0738978290558 +bf16,16,64,16,64,16384,optimized_transform,0,2,100,73.9515447617,4194304,56.7169220537,2097152,2909,true,1,0.0739515447617 +bf16,16,64,16,64,16384,control_shuffle32_transform,0,3,100,21.3199996948,4194304,196.730959664,2097152,2909,true,1,0.0213199996948 +bf16,16,64,16,64,16384,shuffle32_transform,0,4,100,21.2459421158,4194304,197.416710313,2097152,2909,true,1,0.0212459421158 +bf16,16,64,16,64,16384,baseline_split,0,5,100,1548.26766968,6881280,4.4445028045,2097152,2909,true,1,1.54826766968 +bf16,16,64,16,64,16384,control_optimized_split,0,6,100,1361.00799561,6881280,5.05601732115,2097152,2909,true,1,1.36100799561 +bf16,16,64,16,64,16384,optimized_split,0,7,100,576.609382629,6881280,11.934040977,2097152,2909,true,1,0.576609382629 +bf16,16,64,16,64,16384,control_shuffle32_split,0,8,100,851.780548096,6881280,8.07870057069,2097152,2909,true,1,0.851780548096 +bf16,16,64,16,64,16384,shuffle32_split,0,9,100,228.865375519,6881280,30.0669333856,2097152,2909,true,1,0.228865375519 +bf16,16,64,16,64,16384,baseline_fused,0,10,100,1525.97183228,2686976,1.76082935685,2097152,2909,true,1,1.52597183228 +bf16,16,64,16,64,16384,control_optimized_fused,0,11,100,1333.6491394,2686976,2.01475479615,2097152,2909,true,1,1.3336491394 +bf16,16,64,16,64,16384,optimized_fused,0,12,100,544.566192627,2686976,4.93415866864,2097152,2909,true,1,0.544566192627 +bf16,16,64,16,64,16384,control_shuffle32_fused,0,13,100,867.039108276,2686976,3.09902514702,2097152,2909,true,1,0.867039108276 +bf16,16,64,16,64,16384,shuffle32_fused,0,14,100,211.920452118,2686976,12.6791726478,2097152,2909,true,1,0.211920452118 +bf16,16,64,16,64,16384,control_optimized_transform,1,0,100,73.8918876648,4194304,56.7627128302,2097152,2909,true,1,0.0738918876648 +bf16,16,64,16,64,16384,optimized_transform,1,1,100,73.9567995071,4194304,56.7128922283,2097152,2909,true,1,0.0739567995071 +bf16,16,64,16,64,16384,control_shuffle32_transform,1,2,100,21.2948584557,4194304,196.963225125,2097152,2909,true,1,0.0212948584557 +bf16,16,64,16,64,16384,shuffle32_transform,1,3,100,21.2815999985,4194304,197.085933403,2097152,2909,true,1,0.0212815999985 +bf16,16,64,16,64,16384,baseline_split,1,4,100,1548.30001831,6881280,4.4444099455,2097152,2909,true,1,1.54830001831 +bf16,16,64,16,64,16384,control_optimized_split,1,5,100,1361.10336304,6881280,5.05566306489,2097152,2909,true,1,1.36110336304 +bf16,16,64,16,64,16384,optimized_split,1,6,100,576.504249573,6881280,11.9362173047,2097152,2909,true,1,0.576504249573 +bf16,16,64,16,64,16384,control_shuffle32_split,1,7,100,851.821746826,6881280,8.07830984081,2097152,2909,true,1,0.851821746826 +bf16,16,64,16,64,16384,shuffle32_split,1,8,100,228.76707077,6881280,30.0798536119,2097152,2909,true,1,0.22876707077 +bf16,16,64,16,64,16384,baseline_fused,1,9,100,1525.88088989,2686976,1.76093430215,2097152,2909,true,1,1.52588088989 +bf16,16,64,16,64,16384,control_optimized_fused,1,10,100,1333.53607178,2686976,2.01492562284,2097152,2909,true,1,1.33353607178 +bf16,16,64,16,64,16384,optimized_fused,1,11,100,544.697608948,2686976,4.93296823019,2097152,2909,true,1,0.544697608948 +bf16,16,64,16,64,16384,control_shuffle32_fused,1,12,100,867.093048096,2686976,3.09883236396,2097152,2909,true,1,0.867093048096 +bf16,16,64,16,64,16384,shuffle32_fused,1,13,100,211.983547211,2686976,12.6753988003,2097152,2909,true,1,0.211983547211 +bf16,16,64,16,64,16384,baseline_transform,1,14,100,68.1419420242,4194304,61.5524576407,2097152,2909,true,1,0.0681419420242 +bf16,16,64,16,64,16384,optimized_transform,2,0,100,73.9602279663,4194304,56.7102632771,2097152,2909,true,1,0.0739602279663 +bf16,16,64,16,64,16384,control_shuffle32_transform,2,1,100,21.3508558273,4194304,196.446645227,2097152,2909,true,1,0.0213508558273 +bf16,16,64,16,64,16384,shuffle32_transform,2,2,100,21.3654851913,4194304,196.312134381,2097152,2909,true,1,0.0213654851913 +bf16,16,64,16,64,16384,baseline_split,2,3,100,1548.61679077,6881280,4.44350083314,2097152,2909,true,1,1.54861679077 +bf16,16,64,16,64,16384,control_optimized_split,2,4,100,1361.23428345,6881280,5.05517682274,2097152,2909,true,1,1.36123428345 +bf16,16,64,16,64,16384,optimized_split,2,5,100,576.572341919,6881280,11.934807655,2097152,2909,true,1,0.576572341919 +bf16,16,64,16,64,16384,control_shuffle32_split,2,6,100,851.890106201,6881280,8.07766160202,2097152,2909,true,1,0.851890106201 +bf16,16,64,16,64,16384,shuffle32_split,2,7,100,228.804111481,6881280,30.0749840353,2097152,2909,true,1,0.228804111481 +bf16,16,64,16,64,16384,baseline_fused,2,8,100,1525.86181641,2686976,1.76095631407,2097152,2909,true,1,1.52586181641 +bf16,16,64,16,64,16384,control_optimized_fused,2,9,100,1333.60275269,2686976,2.01482487539,2097152,2909,true,1,1.33360275269 +bf16,16,64,16,64,16384,optimized_fused,2,10,100,544.55039978,2686976,4.93430176726,2097152,2909,true,1,0.54455039978 +bf16,16,64,16,64,16384,control_shuffle32_fused,2,11,100,867.049179077,2686976,3.09898915176,2097152,2909,true,1,0.867049179077 +bf16,16,64,16,64,16384,shuffle32_fused,2,12,100,211.934394836,2686976,12.6783385117,2097152,2909,true,1,0.211934394836 +bf16,16,64,16,64,16384,baseline_transform,2,13,100,68.1810283661,4194304,61.5171712794,2097152,2909,true,1,0.0681810283661 +bf16,16,64,16,64,16384,control_optimized_transform,2,14,100,73.9152002335,4194304,56.7448100898,2097152,2909,true,1,0.0739152002335 +bf16,16,64,16,64,16384,control_shuffle32_transform,3,0,100,21.6701722145,4194304,193.551945895,2097152,2909,true,1,0.0216701722145 +bf16,16,64,16,64,16384,shuffle32_transform,3,1,100,21.6486859322,4194304,193.744045858,2097152,2909,true,1,0.0216486859322 +bf16,16,64,16,64,16384,baseline_split,3,2,100,1548.64028931,6881280,4.44343340898,2097152,2909,true,1,1.54864028931 +bf16,16,64,16,64,16384,control_optimized_split,3,3,100,1360.9866333,6881280,5.05609668135,2097152,2909,true,1,1.3609866333 +bf16,16,64,16,64,16384,optimized_split,3,4,100,576.570968628,6881280,11.9348360816,2097152,2909,true,1,0.576570968628 +bf16,16,64,16,64,16384,control_shuffle32_split,3,5,100,851.834030151,6881280,8.07819335273,2097152,2909,true,1,0.851834030151 +bf16,16,64,16,64,16384,shuffle32_split,3,6,100,228.777370453,6881280,30.0784994004,2097152,2909,true,1,0.228777370453 +bf16,16,64,16,64,16384,baseline_fused,3,7,100,1525.93139648,2686976,1.76087601723,2097152,2909,true,1,1.52593139648 +bf16,16,64,16,64,16384,control_optimized_fused,3,8,100,1333.71520996,2686976,2.01465498776,2097152,2909,true,1,1.33371520996 +bf16,16,64,16,64,16384,optimized_fused,3,9,100,544.655761719,2686976,4.93334724216,2097152,2909,true,1,0.544655761719 +bf16,16,64,16,64,16384,control_shuffle32_fused,3,10,100,867.250289917,2686976,3.09827051226,2097152,2909,true,1,0.867250289917 +bf16,16,64,16,64,16384,shuffle32_fused,3,11,100,212.097587585,2686976,12.6685835072,2097152,2909,true,1,0.212097587585 +bf16,16,64,16,64,16384,baseline_transform,3,12,100,68.1817150116,4194304,61.5165517513,2097152,2909,true,1,0.0681817150116 +bf16,16,64,16,64,16384,control_optimized_transform,3,13,100,73.9071989059,4194304,56.750953386,2097152,2909,true,1,0.0739071989059 +bf16,16,64,16,64,16384,optimized_transform,3,14,100,73.9682292938,4194304,56.7041287867,2097152,2909,true,1,0.0739682292938 +bf16,16,64,16,64,16384,shuffle32_transform,4,0,100,21.7391991615,4194304,192.937374042,2097152,2909,true,1,0.0217391991615 +bf16,16,64,16,64,16384,baseline_split,4,1,100,1548.45672607,6881280,4.4439601599,2097152,2909,true,1,1.54845672607 +bf16,16,64,16,64,16384,control_optimized_split,4,2,100,1361.20498657,6881280,5.05528562405,2097152,2909,true,1,1.36120498657 +bf16,16,64,16,64,16384,optimized_split,4,3,100,576.476325989,6881280,11.9367954759,2097152,2909,true,1,0.576476325989 +bf16,16,64,16,64,16384,control_shuffle32_split,4,4,100,851.856460571,6881280,8.07798064405,2097152,2909,true,1,0.851856460571 +bf16,16,64,16,64,16384,shuffle32_split,4,5,100,228.81280899,6881280,30.0738408412,2097152,2909,true,1,0.22881280899 +bf16,16,64,16,64,16384,baseline_fused,4,6,100,1525.859375,2686976,1.76095913164,2097152,2909,true,1,1.525859375 +bf16,16,64,16,64,16384,control_optimized_fused,4,7,100,1333.6882019,2686976,2.01469578584,2097152,2909,true,1,1.3336882019 +bf16,16,64,16,64,16384,optimized_fused,4,8,100,544.582633972,2686976,4.93400970281,2097152,2909,true,1,0.544582633972 +bf16,16,64,16,64,16384,control_shuffle32_fused,4,9,100,867.048950195,2686976,3.09898996982,2097152,2909,true,1,0.867048950195 +bf16,16,64,16,64,16384,shuffle32_fused,4,10,100,211.860351562,2686976,12.6827694761,2097152,2909,true,1,0.211860351562 +bf16,16,64,16,64,16384,baseline_transform,4,11,100,68.1851434708,4194304,61.5134585996,2097152,2909,true,1,0.0681851434708 +bf16,16,64,16,64,16384,control_optimized_transform,4,12,100,73.9156579971,4194304,56.7444586662,2097152,2909,true,1,0.0739156579971 +bf16,16,64,16,64,16384,optimized_transform,4,13,100,73.973941803,4194304,56.6997499088,2097152,2909,true,1,0.073973941803 +bf16,16,64,16,64,16384,control_shuffle32_transform,4,14,100,21.7101716995,4194304,193.195339864,2097152,2909,true,1,0.0217101716995 +bf16,1,1,1,128,1,baseline_transform,0,0,100,9.70742821693,512,0.0527431147116,256,2909,true,1,0.00970742821693 +bf16,1,1,1,128,1,control_optimized_transform,0,1,100,10.0107431412,512,0.0511450541463,256,2909,true,1,0.0100107431412 +bf16,1,1,1,128,1,optimized_transform,0,2,100,10.0918865204,512,0.0507338245397,256,2909,true,1,0.0100918865204 +bf16,1,1,1,128,1,control_shuffle32_transform,0,3,100,9.59314286709,512,0.0533714557464,256,2909,true,1,0.00959314286709 +bf16,1,1,1,128,1,shuffle32_transform,0,4,100,9.46125686169,512,0.0541154317534,256,2909,true,1,0.00946125686169 +bf16,1,1,1,128,1,baseline_split,0,5,100,79.1778326035,836,0.0105585107916,256,2909,true,1,0.0791778326035 +bf16,1,1,1,128,1,control_optimized_split,0,6,100,63.5140562057,836,0.0131624407248,256,2909,true,1,0.0635140562057 +bf16,1,1,1,128,1,optimized_split,0,7,100,34.1577148438,836,0.0244747051676,256,2909,true,1,0.0341577148437 +bf16,1,1,1,128,1,control_shuffle32_split,0,8,100,161.44411087,836,0.00517826259188,256,2909,true,1,0.16144411087 +bf16,1,1,1,128,1,shuffle32_split,0,9,100,45.5716562271,836,0.0183447359436,256,2909,true,1,0.0455716562271 +bf16,1,1,1,128,1,baseline_fused,0,10,100,74.5776033401,324,0.00434446784945,256,2909,true,1,0.0745776033401 +bf16,1,1,1,128,1,control_optimized_fused,0,11,100,58.1837701797,324,0.0055685631749,256,2909,true,1,0.0581837701797 +bf16,1,1,1,128,1,optimized_fused,0,12,100,28.9056015015,324,0.0112089001152,256,2909,true,1,0.0289056015015 +bf16,1,1,1,128,1,control_shuffle32_fused,0,13,100,157.17508316,324,0.00206139544185,256,2909,true,1,0.15717508316 +bf16,1,1,1,128,1,shuffle32_fused,0,14,100,40.9391975403,324,0.00791417564258,256,2909,true,1,0.0409391975403 +bf16,1,1,1,128,1,control_optimized_transform,1,0,100,10.001142025,512,0.0511941534997,256,2909,true,1,0.010001142025 +bf16,1,1,1,128,1,optimized_transform,1,1,100,9.9044573307,512,0.0516938972934,256,2909,true,1,0.0099044573307 +bf16,1,1,1,128,1,control_shuffle32_transform,1,2,100,9.43245708942,512,0.0542806603991,256,2909,true,1,0.00943245708942 +bf16,1,1,1,128,1,shuffle32_transform,1,3,100,9.48182821274,512,0.0539980253294,256,2909,true,1,0.00948182821274 +bf16,1,1,1,128,1,baseline_split,1,4,100,79.2345142365,836,0.0105509575979,256,2909,true,1,0.0792345142365 +bf16,1,1,1,128,1,control_optimized_split,1,5,100,63.5323429108,836,0.01315865214,256,2909,true,1,0.0635323429108 +bf16,1,1,1,128,1,optimized_split,1,6,100,34.1456007957,836,0.0244833882116,256,2909,true,1,0.0341456007957 +bf16,1,1,1,128,1,control_shuffle32_split,1,7,100,161.514282227,836,0.00517601284837,256,2909,true,1,0.161514282227 +bf16,1,1,1,128,1,shuffle32_split,1,8,100,45.5469751358,836,0.018354676628,256,2909,true,1,0.0455469751358 +bf16,1,1,1,128,1,baseline_fused,1,9,100,74.57508564,324,0.00434461452132,256,2909,true,1,0.07457508564 +bf16,1,1,1,128,1,control_optimized_fused,1,10,100,58.2038831711,324,0.00556663889672,256,2909,true,1,0.0582038831711 +bf16,1,1,1,128,1,optimized_fused,1,11,100,28.92182827,324,0.0112026112933,256,2909,true,1,0.02892182827 +bf16,1,1,1,128,1,control_shuffle32_fused,1,12,100,157.176904678,324,0.00206137155241,256,2909,true,1,0.157176904678 +bf16,1,1,1,128,1,shuffle32_fused,1,13,100,40.9620571136,324,0.00790975900212,256,2909,true,1,0.0409620571136 +bf16,1,1,1,128,1,baseline_transform,1,14,100,9.68022882938,512,0.0528913116646,256,2909,true,1,0.00968022882938 +bf16,1,1,1,128,1,optimized_transform,2,0,100,10.0596570969,512,0.0508963670501,256,2909,true,1,0.0100596570969 +bf16,1,1,1,128,1,control_shuffle32_transform,2,1,100,9.38308596611,512,0.0545662697591,256,2909,true,1,0.00938308596611 +bf16,1,1,1,128,1,shuffle32_transform,2,2,100,9.4450289011,512,0.0542084100918,256,2909,true,1,0.0094450289011 +bf16,1,1,1,128,1,baseline_split,2,3,100,79.2358827591,836,0.010550775367,256,2909,true,1,0.0792358827591 +bf16,1,1,1,128,1,control_optimized_split,2,4,100,63.5570287704,836,0.0131535412554,256,2909,true,1,0.0635570287704 +bf16,1,1,1,128,1,optimized_split,2,5,100,34.6905136108,836,0.0240988072237,256,2909,true,1,0.0346905136108 +bf16,1,1,1,128,1,control_shuffle32_split,2,6,100,161.418514252,836,0.00517908372454,256,2909,true,1,0.161418514252 +bf16,1,1,1,128,1,shuffle32_split,2,7,100,45.6233119965,836,0.0183239656092,256,2909,true,1,0.0456233119965 +bf16,1,1,1,128,1,baseline_fused,2,8,100,74.6075439453,324,0.00434272437969,256,2909,true,1,0.0746075439453 +bf16,1,1,1,128,1,control_optimized_fused,2,9,100,58.2194280624,324,0.00556515257506,256,2909,true,1,0.0582194280624 +bf16,1,1,1,128,1,optimized_fused,2,10,100,28.9684581757,324,0.0111845786902,256,2909,true,1,0.0289684581757 +bf16,1,1,1,128,1,control_shuffle32_fused,2,11,100,157.192459106,324,0.00206116757662,256,2909,true,1,0.157192459106 +bf16,1,1,1,128,1,shuffle32_fused,2,12,100,40.9385156631,324,0.0079143074621,256,2909,true,1,0.0409385156631 +bf16,1,1,1,128,1,baseline_transform,2,13,100,9.70788598061,512,0.0527406276735,256,2909,true,1,0.00970788598061 +bf16,1,1,1,128,1,control_optimized_transform,2,14,100,9.80274260044,512,0.0522302809396,256,2909,true,1,0.00980274260044 +bf16,1,1,1,128,1,control_shuffle32_transform,3,0,100,9.57348585129,512,0.053481042115,256,2909,true,1,0.00957348585129 +bf16,1,1,1,128,1,shuffle32_transform,3,1,100,9.50834333897,512,0.0538474455273,256,2909,true,1,0.00950834333897 +bf16,1,1,1,128,1,baseline_split,3,2,100,79.2482328415,836,0.0105491311292,256,2909,true,1,0.0792482328415 +bf16,1,1,1,128,1,control_optimized_split,3,3,100,64.2370271683,836,0.0130143008924,256,2909,true,1,0.0642370271683 +bf16,1,1,1,128,1,optimized_split,3,4,100,34.1154289246,836,0.0245050414535,256,2909,true,1,0.0341154289246 +bf16,1,1,1,128,1,control_shuffle32_split,3,5,100,161.330986023,836,0.00518189357549,256,2909,true,1,0.161330986023 +bf16,1,1,1,128,1,shuffle32_split,3,6,100,45.5350828171,836,0.0183594702871,256,2909,true,1,0.0455350828171 +bf16,1,1,1,128,1,baseline_fused,3,7,100,74.5789718628,324,0.00434438812855,256,2909,true,1,0.0745789718628 +bf16,1,1,1,128,1,control_optimized_fused,3,8,100,58.2018327713,324,0.00556683500455,256,2909,true,1,0.0582018327713 +bf16,1,1,1,128,1,optimized_fused,3,9,100,28.9152002335,324,0.0112051791924,256,2909,true,1,0.0289152002335 +bf16,1,1,1,128,1,control_shuffle32_fused,3,10,100,157.184228897,324,0.00206127549992,256,2909,true,1,0.157184228897 +bf16,1,1,1,128,1,shuffle32_fused,3,11,100,40.9433174133,324,0.00791337928798,256,2909,true,1,0.0409433174133 +bf16,1,1,1,128,1,baseline_transform,3,12,100,9.68502819538,512,0.0528651016467,256,2909,true,1,0.00968502819538 +bf16,1,1,1,128,1,control_optimized_transform,3,13,100,10.0493705273,512,0.0509484647432,256,2909,true,1,0.0100493705273 +bf16,1,1,1,128,1,optimized_transform,3,14,100,9.95611429214,512,0.05142568526,256,2909,true,1,0.00995611429214 +bf16,1,1,1,128,1,shuffle32_transform,4,0,100,9.55428600311,512,0.0535885151264,256,2909,true,1,0.00955428600311 +bf16,1,1,1,128,1,baseline_split,4,1,100,79.1865110397,836,0.0105573536329,256,2909,true,1,0.0791865110397 +bf16,1,1,1,128,1,control_optimized_split,4,2,100,63.5510873795,836,0.0131547709799,256,2909,true,1,0.0635510873795 +bf16,1,1,1,128,1,optimized_split,4,3,100,34.1437721252,836,0.0244846994917,256,2909,true,1,0.0341437721252 +bf16,1,1,1,128,1,control_shuffle32_split,4,4,100,161.335544586,836,0.00518174716021,256,2909,true,1,0.161335544586 +bf16,1,1,1,128,1,shuffle32_split,4,5,100,45.6075429916,836,0.0183303011994,256,2909,true,1,0.0456075429916 +bf16,1,1,1,128,1,baseline_fused,4,6,100,74.5760011673,324,0.00434456118495,256,2909,true,1,0.0745760011673 +bf16,1,1,1,128,1,control_optimized_fused,4,7,100,58.2000017166,324,0.00556701014508,256,2909,true,1,0.0582000017166 +bf16,1,1,1,128,1,optimized_fused,4,8,100,28.9149713516,324,0.0112052678891,256,2909,true,1,0.0289149713516 +bf16,1,1,1,128,1,control_shuffle32_fused,4,9,100,157.181711197,324,0.00206130851696,256,2909,true,1,0.157181711197 +bf16,1,1,1,128,1,shuffle32_fused,4,10,100,40.9104013443,324,0.00791974630787,256,2909,true,1,0.0409104013443 +bf16,1,1,1,128,1,baseline_transform,4,11,100,9.68502819538,512,0.0528651016467,256,2909,true,1,0.00968502819538 +bf16,1,1,1,128,1,control_optimized_transform,4,12,100,10.0900566578,512,0.0507430252738,256,2909,true,1,0.0100900566578 +bf16,1,1,1,128,1,optimized_transform,4,13,100,9.90971446037,512,0.0516664735445,256,2909,true,1,0.00990971446037 +bf16,1,1,1,128,1,control_shuffle32_transform,4,14,100,9.49051380157,512,0.0539486070728,256,2909,true,1,0.00949051380157 +bf16,1,17,1,128,17,baseline_transform,0,0,100,9.93759930134,8704,0.875865461674,4352,2909,true,1,0.00993759930134 +bf16,1,17,1,128,17,control_optimized_transform,0,1,100,10.2262866497,8704,0.85113984168,4352,2909,true,1,0.0102262866497 +bf16,1,17,1,128,17,optimized_transform,0,2,100,10.1677715778,8704,0.856038113501,4352,2909,true,1,0.0101677715778 +bf16,1,17,1,128,17,control_shuffle32_transform,0,3,100,9.64297175407,8704,0.902626308775,4352,2909,true,1,0.00964297175407 +bf16,1,17,1,128,17,shuffle32_transform,0,4,100,9.6303999424,8704,0.903804624113,4352,2909,true,1,0.0096303999424 +bf16,1,17,1,128,17,baseline_split,0,5,100,80.5385112762,14212,0.176462164184,4352,2909,true,1,0.0805385112762 +bf16,1,17,1,128,17,control_optimized_split,0,6,100,64.8363447189,14212,0.219198044887,4352,2909,true,1,0.0648363447189 +bf16,1,17,1,128,17,optimized_split,0,7,100,35.6939435005,14212,0.39816278635,4352,2909,true,1,0.0356939435005 +bf16,1,17,1,128,17,control_shuffle32_split,0,8,100,163.362293243,14212,0.0869968198771,4352,2909,true,1,0.163362293243 +bf16,1,17,1,128,17,shuffle32_split,0,9,100,45.5906295776,14212,0.311730724749,4352,2909,true,1,0.0455906295776 +bf16,1,17,1,128,17,baseline_fused,0,10,100,75.6518888474,5508,0.0728071708971,4352,2909,true,1,0.0756518888474 +bf16,1,17,1,128,17,control_optimized_fused,0,11,100,59.2544031143,5508,0.0929551174345,4352,2909,true,1,0.0592544031143 +bf16,1,17,1,128,17,optimized_fused,0,12,100,29.3590855598,5508,0.187608023035,4352,2909,true,1,0.0293590855598 +bf16,1,17,1,128,17,control_shuffle32_fused,0,13,100,158.852119446,5508,0.0346737583308,4352,2909,true,1,0.158852119446 +bf16,1,17,1,128,17,shuffle32_fused,0,14,100,40.5465173721,5508,0.135843972725,4352,2909,true,1,0.0405465173721 +bf16,1,17,1,128,17,control_optimized_transform,1,0,100,10.2349722385,8704,0.850417548494,4352,2909,true,1,0.0102349722385 +bf16,1,17,1,128,17,optimized_transform,1,1,100,10.3291428089,8704,0.842664310197,4352,2909,true,1,0.0103291428089 +bf16,1,17,1,128,17,control_shuffle32_transform,1,2,100,9.53942835331,8704,0.912423646117,4352,2909,true,1,0.00953942835331 +bf16,1,17,1,128,17,shuffle32_transform,1,3,100,9.73439991474,8704,0.894148594288,4352,2909,true,1,0.00973439991474 +bf16,1,17,1,128,17,baseline_split,1,4,100,80.5364513397,14212,0.176466677679,4352,2909,true,1,0.0805364513397 +bf16,1,17,1,128,17,control_optimized_split,1,5,100,64.8548603058,14212,0.219135465453,4352,2909,true,1,0.0648548603058 +bf16,1,17,1,128,17,optimized_split,1,6,100,35.0125718117,14212,0.405911341687,4352,2909,true,1,0.0350125718117 +bf16,1,17,1,128,17,control_shuffle32_split,1,7,100,163.358402252,14212,0.0869988920316,4352,2909,true,1,0.163358402252 +bf16,1,17,1,128,17,shuffle32_split,1,8,100,45.6052541733,14212,0.311630759605,4352,2909,true,1,0.0456052541733 +bf16,1,17,1,128,17,baseline_fused,1,9,100,75.6447982788,5508,0.0728139954805,4352,2909,true,1,0.0756447982788 +bf16,1,17,1,128,17,control_optimized_fused,1,10,100,59.2329120636,5508,0.09298884367,4352,2909,true,1,0.0592329120636 +bf16,1,17,1,128,17,optimized_fused,1,11,100,29.3728017807,5508,0.187520415694,4352,2909,true,1,0.0293728017807 +bf16,1,17,1,128,17,control_shuffle32_fused,1,12,100,158.867425919,5508,0.0346704176023,4352,2909,true,1,0.158867425919 +bf16,1,17,1,128,17,shuffle32_fused,1,13,100,40.5401134491,5508,0.135865431332,4352,2909,true,1,0.0405401134491 +bf16,1,17,1,128,17,baseline_transform,1,14,100,9.93668556213,8704,0.875946002877,4352,2909,true,1,0.00993668556213 +bf16,1,17,1,128,17,optimized_transform,2,0,100,10.2934861183,8704,0.845583303844,4352,2909,true,1,0.0102934861183 +bf16,1,17,1,128,17,control_shuffle32_transform,2,1,100,9.57622826099,8704,0.90891734854,4352,2909,true,1,0.00957622826099 +bf16,1,17,1,128,17,shuffle32_transform,2,2,100,9.59291398525,8704,0.907336395738,4352,2909,true,1,0.00959291398525 +bf16,1,17,1,128,17,baseline_split,2,3,100,80.5728054047,14212,0.176387056757,4352,2909,true,1,0.0805728054047 +bf16,1,17,1,128,17,control_optimized_split,2,4,100,64.7945165634,14212,0.219339548372,4352,2909,true,1,0.0647945165634 +bf16,1,17,1,128,17,optimized_split,2,5,100,35.6827425957,14212,0.398287770675,4352,2909,true,1,0.0356827425957 +bf16,1,17,1,128,17,control_shuffle32_split,2,6,100,163.265380859,14212,0.0870484601524,4352,2909,true,1,0.163265380859 +bf16,1,17,1,128,17,shuffle32_split,2,7,100,45.6134843826,14212,0.311574530917,4352,2909,true,1,0.0456134843826 +bf16,1,17,1,128,17,baseline_fused,2,8,100,75.6381750107,5508,0.072820371449,4352,2909,true,1,0.0756381750107 +bf16,1,17,1,128,17,control_optimized_fused,2,9,100,59.2427444458,5508,0.0929734105252,4352,2909,true,1,0.0592427444458 +bf16,1,17,1,128,17,optimized_fused,2,10,100,29.3092560768,5508,0.187926980663,4352,2909,true,1,0.0293092560768 +bf16,1,17,1,128,17,control_shuffle32_fused,2,11,100,158.845949173,5508,0.0346751052115,4352,2909,true,1,0.158845949173 +bf16,1,17,1,128,17,shuffle32_fused,2,12,100,40.4973697662,5508,0.136008832964,4352,2909,true,1,0.0404973697662 +bf16,1,17,1,128,17,baseline_transform,2,13,100,9.93348538876,8704,0.876228197794,4352,2909,true,1,0.00993348538876 +bf16,1,17,1,128,17,control_optimized_transform,2,14,100,10.2249133587,8704,0.851254156849,4352,2909,true,1,0.0102249133587 +bf16,1,17,1,128,17,control_shuffle32_transform,3,0,100,9.62628602982,8704,0.904190876215,4352,2909,true,1,0.00962628602982 +bf16,1,17,1,128,17,shuffle32_transform,3,1,100,9.56891477108,8704,0.909612031064,4352,2909,true,1,0.00956891477108 +bf16,1,17,1,128,17,baseline_split,3,2,100,80.4946327209,14212,0.176558355751,4352,2909,true,1,0.0804946327209 +bf16,1,17,1,128,17,control_optimized_split,3,3,100,64.8084545135,14212,0.219292376383,4352,2909,true,1,0.0648084545135 +bf16,1,17,1,128,17,optimized_split,3,4,100,35.0292563438,14212,0.405718004987,4352,2909,true,1,0.0350292563438 +bf16,1,17,1,128,17,control_shuffle32_split,3,5,100,163.375301361,14212,0.0869898930964,4352,2909,true,1,0.163375301361 +bf16,1,17,1,128,17,shuffle32_split,3,6,100,45.5387449265,14212,0.312085895713,4352,2909,true,1,0.0455387449265 +bf16,1,17,1,128,17,baseline_fused,3,7,100,75.6509685516,5508,0.072808056598,4352,2909,true,1,0.0756509685516 +bf16,1,17,1,128,17,control_optimized_fused,3,8,100,59.2237710953,5508,0.0930031961514,4352,2909,true,1,0.0592237710953 +bf16,1,17,1,128,17,optimized_fused,3,9,100,29.3234300613,5508,0.187836142923,4352,2909,true,1,0.0293234300613 +bf16,1,17,1,128,17,control_shuffle32_fused,3,10,100,158.866043091,5508,0.0346707193862,4352,2909,true,1,0.158866043091 +bf16,1,17,1,128,17,shuffle32_fused,3,11,100,40.5355453491,5508,0.135880742508,4352,2909,true,1,0.0405355453491 +bf16,1,17,1,128,17,baseline_transform,3,12,100,9.97531473637,8704,0.872553922361,4352,2909,true,1,0.00997531473637 +bf16,1,17,1,128,17,control_optimized_transform,3,13,100,10.1097142696,8704,0.860954104919,4352,2909,true,1,0.0101097142696 +bf16,1,17,1,128,17,optimized_transform,3,14,100,10.3282284737,8704,0.842738909407,4352,2909,true,1,0.0103282284737 +bf16,1,17,1,128,17,shuffle32_transform,4,0,100,9.59908604622,8704,0.906752992742,4352,2909,true,1,0.00959908604622 +bf16,1,17,1,128,17,baseline_split,4,1,100,80.5844593048,14212,0.176361548152,4352,2909,true,1,0.0805844593048 +bf16,1,17,1,128,17,control_optimized_split,4,2,100,64.8587417603,14212,0.219122351348,4352,2909,true,1,0.0648587417603 +bf16,1,17,1,128,17,optimized_split,4,3,100,34.9430847168,14212,0.406718528578,4352,2909,true,1,0.0349430847168 +bf16,1,17,1,128,17,control_shuffle32_split,4,4,100,163.399085999,14212,0.0869772307057,4352,2909,true,1,0.163399085999 +bf16,1,17,1,128,17,shuffle32_split,4,5,100,45.587887764,14212,0.311749473316,4352,2909,true,1,0.045587887764 +bf16,1,17,1,128,17,baseline_fused,4,6,100,75.6466293335,5508,0.0728122329908,4352,2909,true,1,0.0756466293335 +bf16,1,17,1,128,17,control_optimized_fused,4,7,100,59.2422866821,5508,0.0929741289284,4352,2909,true,1,0.0592422866821 +bf16,1,17,1,128,17,optimized_fused,4,8,100,29.3471980095,5508,0.187684016655,4352,2909,true,1,0.0293471980095 +bf16,1,17,1,128,17,control_shuffle32_fused,4,9,100,158.866977692,5508,0.0346705154213,4352,2909,true,1,0.158866977692 +bf16,1,17,1,128,17,shuffle32_fused,4,10,100,40.5291414261,5508,0.135902212734,4352,2909,true,1,0.0405291414261 +bf16,1,17,1,128,17,baseline_transform,4,11,100,9.9718862772,8704,0.872853917308,4352,2909,true,1,0.0099718862772 +bf16,1,17,1,128,17,control_optimized_transform,4,12,100,10.2861714363,8704,0.846184613381,4352,2909,true,1,0.0102861714363 +bf16,1,17,1,128,17,optimized_transform,4,13,100,10.3172564507,8704,0.843635131261,4352,2909,true,1,0.0103172564507 +bf16,1,17,1,128,17,control_shuffle32_transform,4,14,100,9.6594285965,8704,0.901088497425,4352,2909,true,1,0.0096594285965 +bf16,1,257,1,128,257,baseline_transform,0,0,100,10.361828804,131584,12.6989166188,65792,2909,true,1,0.010361828804 +bf16,1,257,1,128,257,control_optimized_transform,0,1,100,10.423771143,131584,12.6234544289,65792,2909,true,1,0.010423771143 +bf16,1,257,1,128,257,optimized_transform,0,2,100,10.604571104,131584,12.4082340256,65792,2909,true,1,0.010604571104 +bf16,1,257,1,128,257,control_shuffle32_transform,0,3,100,10.8427429199,131584,12.1356746141,65792,2909,true,1,0.0108427429199 +bf16,1,257,1,128,257,shuffle32_transform,0,4,100,11.0795426369,131584,11.8763025075,65792,2909,true,1,0.0110795426369 +bf16,1,257,1,128,257,baseline_split,0,5,100,81.9604587555,214852,2.62141041256,65792,2909,true,1,0.0819604587555 +bf16,1,257,1,128,257,control_optimized_split,0,6,100,65.8013725281,214852,3.26515985526,65792,2909,true,1,0.0658013725281 +bf16,1,257,1,128,257,optimized_split,0,7,100,35.2674269676,214852,6.09208038333,65792,2909,true,1,0.0352674269676 +bf16,1,257,1,128,257,control_shuffle32_split,0,8,100,177.696685791,214852,1.20909401908,65792,2909,true,1,0.177696685791 +bf16,1,257,1,128,257,shuffle32_split,0,9,100,51.9014835358,214852,4.13961192173,65792,2909,true,1,0.0519014835358 +bf16,1,257,1,128,257,baseline_fused,0,10,100,76.7846870422,83268,1.08443497275,65792,2909,true,1,0.0767846870422 +bf16,1,257,1,128,257,control_optimized_fused,0,11,100,60.229716301,83268,1.38250692704,65792,2909,true,1,0.060229716301 +bf16,1,257,1,128,257,optimized_fused,0,12,100,29.5833158493,83268,2.81469462126,65792,2909,true,1,0.0295833158493 +bf16,1,257,1,128,257,control_shuffle32_fused,0,13,100,171.222400665,83268,0.4863148728,65792,2909,true,1,0.171222400665 +bf16,1,257,1,128,257,shuffle32_fused,0,14,100,44.8637723923,83268,1.85601868857,65792,2909,true,1,0.0448637723923 +bf16,1,257,1,128,257,control_optimized_transform,1,0,100,10.4075419903,131584,12.643138997,65792,2909,true,1,0.0104075419903 +bf16,1,257,1,128,257,optimized_transform,1,1,100,10.3956568241,131584,12.6575936688,65792,2909,true,1,0.0103956568241 +bf16,1,257,1,128,257,control_shuffle32_transform,1,2,100,10.8932578564,131584,12.0793982604,65792,2909,true,1,0.0108932578564 +bf16,1,257,1,128,257,shuffle32_transform,1,3,100,10.8027422428,131584,12.1806109081,65792,2909,true,1,0.0108027422428 +bf16,1,257,1,128,257,baseline_split,1,4,100,81.9014835358,214852,2.62329802495,65792,2909,true,1,0.0819014835358 +bf16,1,257,1,128,257,control_optimized_split,1,5,100,66.5485715866,214852,3.2284990478,65792,2909,true,1,0.0665485715866 +bf16,1,257,1,128,257,optimized_split,1,6,100,35.2121138573,214852,6.10165015571,65792,2909,true,1,0.0352121138573 +bf16,1,257,1,128,257,control_shuffle32_split,1,7,100,176.957931519,214852,1.21414167851,65792,2909,true,1,0.176957931519 +bf16,1,257,1,128,257,shuffle32_split,1,8,100,51.7462825775,214852,4.1520277264,65792,2909,true,1,0.0517462825775 +bf16,1,257,1,128,257,baseline_fused,1,9,100,76.7707395554,83268,1.08463198977,65792,2909,true,1,0.0767707395554 +bf16,1,257,1,128,257,control_optimized_fused,1,10,100,60.2454900742,83268,1.38214495222,65792,2909,true,1,0.0602454900742 +bf16,1,257,1,128,257,optimized_fused,1,11,100,29.5599985123,83268,2.81691489143,65792,2909,true,1,0.0295599985123 +bf16,1,257,1,128,257,control_shuffle32_fused,1,12,100,171.220569611,83268,0.486320073513,65792,2909,true,1,0.171220569611 +bf16,1,257,1,128,257,shuffle32_fused,1,13,100,44.8422861099,83268,1.85690800411,65792,2909,true,1,0.0448422861099 +bf16,1,257,1,128,257,baseline_transform,1,14,100,10.3549718857,131584,12.7073256647,65792,2909,true,1,0.0103549718857 +bf16,1,257,1,128,257,optimized_transform,2,0,100,10.4813718796,131584,12.5540818045,65792,2909,true,1,0.0104813718796 +bf16,1,257,1,128,257,control_shuffle32_transform,2,1,100,10.8443439007,131584,12.1338829905,65792,2909,true,1,0.0108443439007 +bf16,1,257,1,128,257,shuffle32_transform,2,2,100,11.0683429241,131584,11.8883197695,65792,2909,true,1,0.0110683429241 +bf16,1,257,1,128,257,baseline_split,2,3,100,81.917257309,214852,2.62279288953,65792,2909,true,1,0.081917257309 +bf16,1,257,1,128,257,control_optimized_split,2,4,100,65.9156608582,214852,3.25949853499,65792,2909,true,1,0.0659156608582 +bf16,1,257,1,128,257,optimized_split,2,5,100,35.2870869637,214852,6.08868621605,65792,2909,true,1,0.0352870869637 +bf16,1,257,1,128,257,control_shuffle32_split,2,6,100,176.98469162,214852,1.21395810018,65792,2909,true,1,0.17698469162 +bf16,1,257,1,128,257,shuffle32_split,2,7,100,52.3675441742,214852,4.10277020601,65792,2909,true,1,0.0523675441742 +bf16,1,257,1,128,257,baseline_fused,2,8,100,76.7867422104,83268,1.08440594825,65792,2909,true,1,0.0767867422104 +bf16,1,257,1,128,257,control_optimized_fused,2,9,100,60.2486848831,83268,1.38207166117,65792,2909,true,1,0.0602486848831 +bf16,1,257,1,128,257,optimized_fused,2,10,100,29.5552015305,83268,2.81737209317,65792,2909,true,1,0.0295552015305 +bf16,1,257,1,128,257,control_shuffle32_fused,2,11,100,171.244564056,83268,0.486251931317,65792,2909,true,1,0.171244564056 +bf16,1,257,1,128,257,shuffle32_fused,2,12,100,44.8594284058,83268,1.85619841713,65792,2909,true,1,0.0448594284058 +bf16,1,257,1,128,257,baseline_transform,2,13,100,10.2774858475,131584,12.8031312281,65792,2909,true,1,0.0102774858475 +bf16,1,257,1,128,257,control_optimized_transform,2,14,100,10.3721141815,131584,12.6863238967,65792,2909,true,1,0.0103721141815 +bf16,1,257,1,128,257,control_shuffle32_transform,3,0,100,10.9158849716,131584,12.0543593435,65792,2909,true,1,0.0109158849716 +bf16,1,257,1,128,257,shuffle32_transform,3,1,100,10.9380578995,131584,12.0299235211,65792,2909,true,1,0.0109380578995 +bf16,1,257,1,128,257,baseline_split,3,2,100,81.9590854645,214852,2.62145433642,65792,2909,true,1,0.0819590854645 +bf16,1,257,1,128,257,control_optimized_split,3,3,100,65.8441162109,214852,3.2630402284,65792,2909,true,1,0.0658441162109 +bf16,1,257,1,128,257,optimized_split,3,4,100,35.376226902,214852,6.07334413009,65792,2909,true,1,0.035376226902 +bf16,1,257,1,128,257,control_shuffle32_split,3,5,100,177.686862946,214852,1.20916085994,65792,2909,true,1,0.177686862946 +bf16,1,257,1,128,257,shuffle32_split,3,6,100,51.8162250519,214852,4.14642324455,65792,2909,true,1,0.0518162250519 +bf16,1,257,1,128,257,baseline_fused,3,7,100,76.7695999146,83268,1.08464809108,65792,2909,true,1,0.0767695999146 +bf16,1,257,1,128,257,control_optimized_fused,3,8,100,60.2397680283,83268,1.38227623919,65792,2909,true,1,0.0602397680283 +bf16,1,257,1,128,257,optimized_fused,3,9,100,29.5743989944,83268,2.8155432682,65792,2909,true,1,0.0295743989944 +bf16,1,257,1,128,257,control_shuffle32_fused,3,10,100,171.222629547,83268,0.486314222718,65792,2909,true,1,0.171222629547 +bf16,1,257,1,128,257,shuffle32_fused,3,11,100,44.8502874374,83268,1.85657673022,65792,2909,true,1,0.0448502874374 +bf16,1,257,1,128,257,baseline_transform,3,12,100,10.4057145119,131584,12.6453594176,65792,2909,true,1,0.0104057145119 +bf16,1,257,1,128,257,control_optimized_transform,3,13,100,10.2829706669,131584,12.7963021837,65792,2909,true,1,0.0102829706669 +bf16,1,257,1,128,257,optimized_transform,3,14,100,10.3652572632,131584,12.6947162679,65792,2909,true,1,0.0103652572632 +bf16,1,257,1,128,257,shuffle32_transform,4,0,100,10.8491432667,131584,12.1285152906,65792,2909,true,1,0.0108491432667 +bf16,1,257,1,128,257,baseline_split,4,1,100,81.9634246826,214852,2.62131555425,65792,2909,true,1,0.0819634246826 +bf16,1,257,1,128,257,control_optimized_split,4,2,100,65.9595441818,214852,3.25732996892,65792,2909,true,1,0.0659595441818 +bf16,1,257,1,128,257,optimized_split,4,3,100,35.3090286255,214852,6.08490259754,65792,2909,true,1,0.0353090286255 +bf16,1,257,1,128,257,control_shuffle32_split,4,4,100,177.03086853,214852,1.21364145013,65792,2909,true,1,0.17703086853 +bf16,1,257,1,128,257,shuffle32_split,4,5,100,51.8002271652,214852,4.14770381826,65792,2909,true,1,0.0518002271652 +bf16,1,257,1,128,257,baseline_fused,4,6,100,76.7789745331,83268,1.08451565688,65792,2909,true,1,0.0767789745331 +bf16,1,257,1,128,257,control_optimized_fused,4,7,100,60.2415990829,83268,1.38223422465,65792,2909,true,1,0.0602415990829 +bf16,1,257,1,128,257,optimized_fused,4,8,100,29.5938301086,83268,2.81369460101,65792,2909,true,1,0.0295938301086 +bf16,1,257,1,128,257,control_shuffle32_fused,4,9,100,171.226501465,83268,0.486303225772,65792,2909,true,1,0.171226501465 +bf16,1,257,1,128,257,shuffle32_fused,4,10,100,44.8742818832,83268,1.85558401172,65792,2909,true,1,0.0448742818832 +bf16,1,257,1,128,257,baseline_transform,4,11,100,10.3675436974,131584,12.6919166045,65792,2909,true,1,0.0103675436974 +bf16,1,257,1,128,257,control_optimized_transform,4,12,100,10.6203424931,131584,12.3898075873,65792,2909,true,1,0.0106203424931 +bf16,1,257,1,128,257,optimized_transform,4,13,100,10.4914295673,131584,12.5420467397,65792,2909,true,1,0.0104914295673 +bf16,1,257,1,128,257,control_shuffle32_transform,4,14,100,10.9549713135,131584,12.0113504851,65792,2909,true,1,0.0109549713135 +bf16,4,64,16,128,4096,baseline_transform,0,0,100,25.9613704681,2097152,80.7797108621,1048576,2909,true,1,0.0259613704681 +bf16,4,64,16,128,4096,control_optimized_transform,0,1,100,27.8116583824,2097152,75.4054997787,1048576,2909,true,1,0.0278116583824 +bf16,4,64,16,128,4096,optimized_transform,0,2,100,27.8548574448,2097152,75.2885561938,1048576,2909,true,1,0.0278548574448 +bf16,4,64,16,128,4096,control_shuffle32_transform,0,3,100,15.9668576717,2097152,131.344065508,1048576,2909,true,1,0.0159668576717 +bf16,4,64,16,128,4096,shuffle32_transform,0,4,100,16.1602294445,2097152,129.772414878,1048576,2909,true,1,0.0161602294445 +bf16,4,64,16,128,4096,baseline_split,0,5,100,528.315429688,3424256,6.48146127783,1048576,2909,true,1,0.528315429688 +bf16,4,64,16,128,4096,control_optimized_split,0,6,100,403.23223114,3424256,8.49201957472,1048576,2909,true,1,0.40323223114 +bf16,4,64,16,128,4096,optimized_split,0,7,100,168.902626038,3424256,20.2735509822,1048576,2909,true,1,0.168902626038 +bf16,4,64,16,128,4096,control_shuffle32_split,0,8,100,453.661270142,3424256,7.54804570144,1048576,2909,true,1,0.453661270142 +bf16,4,64,16,128,4096,shuffle32_split,0,9,100,106.035203934,3424256,32.2935767836,1048576,2909,true,1,0.106035203934 +bf16,4,64,16,128,4096,baseline_fused,0,10,100,517.451858521,1327104,2.56469075943,1048576,2909,true,1,0.517451858521 +bf16,4,64,16,128,4096,control_optimized_fused,0,11,100,391.784667969,1327104,3.38733010375,1048576,2909,true,1,0.391784667969 +bf16,4,64,16,128,4096,optimized_fused,0,12,100,154.900798798,1327104,8.56744452128,1048576,2909,true,1,0.154900798798 +bf16,4,64,16,128,4096,control_shuffle32_fused,0,13,100,435.903778076,1327104,3.04448840948,1048576,2909,true,1,0.435903778076 +bf16,4,64,16,128,4096,shuffle32_fused,0,14,100,92.9282283783,1327104,14.2809566389,1048576,2909,true,1,0.0929282283783 +bf16,4,64,16,128,4096,control_optimized_transform,1,0,100,27.8347420692,2097152,75.3429650896,1048576,2909,true,1,0.0278347420692 +bf16,4,64,16,128,4096,optimized_transform,1,1,100,27.8267431259,2097152,75.3646228202,1048576,2909,true,1,0.0278267431259 +bf16,4,64,16,128,4096,control_shuffle32_transform,1,2,100,16.1330282688,2097152,129.991218329,1048576,2909,true,1,0.0161330282688 +bf16,4,64,16,128,4096,shuffle32_transform,1,3,100,16.128000021,2097152,130.031745863,1048576,2909,true,1,0.016128000021 +bf16,4,64,16,128,4096,baseline_split,1,4,100,527.53578186,3424256,6.49104026257,1048576,2909,true,1,0.52753578186 +bf16,4,64,16,128,4096,control_optimized_split,1,5,100,403.058738708,3424256,8.495674876,1048576,2909,true,1,0.403058738708 +bf16,4,64,16,128,4096,optimized_split,1,6,100,168.810749054,3424256,20.2845850705,1048576,2909,true,1,0.168810749054 +bf16,4,64,16,128,4096,control_shuffle32_split,1,7,100,445.848693848,3424256,7.68030959214,1048576,2909,true,1,0.445848693848 +bf16,4,64,16,128,4096,shuffle32_split,1,8,100,105.875654221,3424256,32.3422417099,1048576,2909,true,1,0.105875654221 +bf16,4,64,16,128,4096,baseline_fused,1,9,100,518.21346283,1327104,2.5609215028,1048576,2909,true,1,0.51821346283 +bf16,4,64,16,128,4096,control_optimized_fused,1,10,100,390.362281799,1327104,3.3996727191,1048576,2909,true,1,0.390362281799 +bf16,4,64,16,128,4096,optimized_fused,1,11,100,153.117027283,1327104,8.66725290813,1048576,2909,true,1,0.153117027283 +bf16,4,64,16,128,4096,control_shuffle32_fused,1,12,100,438.263778687,1327104,3.0280941856,1048576,2909,true,1,0.438263778687 +bf16,4,64,16,128,4096,shuffle32_fused,1,13,100,92.9762363434,1327104,14.2735827153,1048576,2909,true,1,0.0929762363434 +bf16,4,64,16,128,4096,baseline_transform,1,14,100,25.9504008293,2097152,80.8138577047,1048576,2909,true,1,0.0259504008293 +bf16,4,64,16,128,4096,optimized_transform,2,0,100,27.8443431854,2097152,75.316985789,1048576,2909,true,1,0.0278443431854 +bf16,4,64,16,128,4096,control_shuffle32_transform,2,1,100,16.1357712746,2097152,129.969120429,1048576,2909,true,1,0.0161357712746 +bf16,4,64,16,128,4096,shuffle32_transform,2,2,100,16.2754285336,2097152,128.853872921,1048576,2909,true,1,0.0162754285336 +bf16,4,64,16,128,4096,baseline_split,2,3,100,525.956573486,3424256,6.51052990421,1048576,2909,true,1,0.525956573486 +bf16,4,64,16,128,4096,control_optimized_split,2,4,100,402.262611389,3424256,8.51248886436,1048576,2909,true,1,0.402262611389 +bf16,4,64,16,128,4096,optimized_split,2,5,100,169.055995941,3424256,20.2551585404,1048576,2909,true,1,0.169055995941 +bf16,4,64,16,128,4096,control_shuffle32_split,2,6,100,455.670166016,3424256,7.51476891705,1048576,2909,true,1,0.455670166016 +bf16,4,64,16,128,4096,shuffle32_split,2,7,100,105.677709579,3424256,32.4028218782,1048576,2909,true,1,0.105677709579 +bf16,4,64,16,128,4096,baseline_fused,2,8,100,520.449142456,1327104,2.54992061998,1048576,2909,true,1,0.520449142456 +bf16,4,64,16,128,4096,control_optimized_fused,2,9,100,391.827430725,1327104,3.38696042169,1048576,2909,true,1,0.391827430725 +bf16,4,64,16,128,4096,optimized_fused,2,10,100,153.762512207,1327104,8.63086834984,1048576,2909,true,1,0.153762512207 +bf16,4,64,16,128,4096,control_shuffle32_fused,2,11,100,437.704467773,1327104,3.03196356837,1048576,2909,true,1,0.437704467773 +bf16,4,64,16,128,4096,shuffle32_fused,2,12,100,92.8985118866,1327104,14.2855248491,1048576,2909,true,1,0.0928985118866 +bf16,4,64,16,128,4096,baseline_transform,2,13,100,25.9604549408,2097152,80.7825596579,1048576,2909,true,1,0.0259604549408 +bf16,4,64,16,128,4096,control_optimized_transform,2,14,100,27.9236555099,2097152,75.1030608887,1048576,2909,true,1,0.0279236555099 +bf16,4,64,16,128,4096,control_shuffle32_transform,3,0,100,16.2203419209,2097152,129.291479195,1048576,2909,true,1,0.0162203419209 +bf16,4,64,16,128,4096,shuffle32_transform,3,1,100,16.1428570747,2097152,129.912071345,1048576,2909,true,1,0.0161428570747 +bf16,4,64,16,128,4096,baseline_split,3,2,100,527.397956848,3424256,6.4927365674,1048576,2909,true,1,0.527397956848 +bf16,4,64,16,128,4096,control_optimized_split,3,3,100,402.727088928,3424256,8.50267114912,1048576,2909,true,1,0.402727088928 +bf16,4,64,16,128,4096,optimized_split,3,4,100,169.2162323,3424256,20.2359782715,1048576,2909,true,1,0.1692162323 +bf16,4,64,16,128,4096,control_shuffle32_split,3,5,100,452.676773071,3424256,7.56446145175,1048576,2909,true,1,0.452676773071 +bf16,4,64,16,128,4096,shuffle32_split,3,6,100,105.848464966,3424256,32.3505494492,1048576,2909,true,1,0.105848464966 +bf16,4,64,16,128,4096,baseline_fused,3,7,100,517.896881104,1327104,2.5624869514,1048576,2909,true,1,0.517896881104 +bf16,4,64,16,128,4096,control_optimized_fused,3,8,100,393.31817627,1327104,3.37412324187,1048576,2909,true,1,0.39331817627 +bf16,4,64,16,128,4096,optimized_fused,3,9,100,154.254179001,1327104,8.6033584866,1048576,2909,true,1,0.154254179001 +bf16,4,64,16,128,4096,control_shuffle32_fused,3,10,100,437.277030945,1327104,3.03492730257,1048576,2909,true,1,0.437277030945 +bf16,4,64,16,128,4096,shuffle32_fused,3,11,100,92.8470897675,1327104,14.293436696,1048576,2909,true,1,0.0928470897675 +bf16,4,64,16,128,4096,baseline_transform,3,12,100,25.9549713135,2097152,80.7996269644,1048576,2909,true,1,0.0259549713135 +bf16,4,64,16,128,4096,control_optimized_transform,3,13,100,27.7622866631,2097152,75.5395989334,1048576,2909,true,1,0.0277622866631 +bf16,4,64,16,128,4096,optimized_transform,3,14,100,27.8196573257,2097152,75.3838185512,1048576,2909,true,1,0.0278196573257 +bf16,4,64,16,128,4096,shuffle32_transform,4,0,100,15.9039998055,2097152,131.863180687,1048576,2909,true,1,0.0159039998055 +bf16,4,64,16,128,4096,baseline_split,4,1,100,527.946510315,3424256,6.48599040452,1048576,2909,true,1,0.527946510315 +bf16,4,64,16,128,4096,control_optimized_split,4,2,100,402.093276978,3424256,8.51607374721,1048576,2909,true,1,0.402093276978 +bf16,4,64,16,128,4096,optimized_split,4,3,100,168.553829193,3424256,20.3155040523,1048576,2909,true,1,0.168553829193 +bf16,4,64,16,128,4096,control_shuffle32_split,4,4,100,452.725715637,3424256,7.56364368474,1048576,2909,true,1,0.452725715637 +bf16,4,64,16,128,4096,shuffle32_split,4,5,100,105.681600571,3424256,32.4016288693,1048576,2909,true,1,0.105681600571 +bf16,4,64,16,128,4096,baseline_fused,4,6,100,520.173721313,1327104,2.5512707498,1048576,2909,true,1,0.520173721313 +bf16,4,64,16,128,4096,control_optimized_fused,4,7,100,391.698722839,1327104,3.38807333958,1048576,2909,true,1,0.391698722839 +bf16,4,64,16,128,4096,optimized_fused,4,8,100,154.923887253,1327104,8.56616770682,1048576,2909,true,1,0.154923887253 +bf16,4,64,16,128,4096,control_shuffle32_fused,4,9,100,437.646827698,1327104,3.03236289174,1048576,2909,true,1,0.437646827698 +bf16,4,64,16,128,4096,shuffle32_fused,4,10,100,92.8747463226,1327104,14.2891803482,1048576,2909,true,1,0.0928747463226 +bf16,4,64,16,128,4096,baseline_transform,4,11,100,25.8971428871,2097152,80.980052863,1048576,2909,true,1,0.0258971428871 +bf16,4,64,16,128,4096,control_optimized_transform,4,12,100,27.8185153008,2097152,75.38691326,1048576,2909,true,1,0.0278185153008 +bf16,4,64,16,128,4096,optimized_transform,4,13,100,27.8219413757,2097152,75.3776298957,1048576,2909,true,1,0.0278219413757 +bf16,4,64,16,128,4096,control_shuffle32_transform,4,14,100,15.7723426819,2097152,132.963887629,1048576,2909,true,1,0.0157723426819 +bf16,16,64,16,128,16384,baseline_transform,0,0,100,78.9076566696,8388608,106.309176499,4194304,2909,true,1,0.0789076566696 +bf16,16,64,16,128,16384,control_optimized_transform,0,1,100,87.1465206146,8388608,96.2586680551,4194304,2909,true,1,0.0871465206146 +bf16,16,64,16,128,16384,optimized_transform,0,2,100,87.2093677521,8388608,96.1892995698,4194304,2909,true,1,0.0872093677521 +bf16,16,64,16,128,16384,control_shuffle32_transform,0,3,100,26.6795420647,8388608,314.420988924,4194304,2909,true,1,0.0266795420647 +bf16,16,64,16,128,16384,shuffle32_transform,0,4,100,26.6175985336,8388608,315.152698295,4194304,2909,true,1,0.0266175985336 +bf16,16,64,16,128,16384,baseline_split,0,5,100,1884.61868286,13697024,7.26779593377,4194304,2909,true,1,1.88461868286 +bf16,16,64,16,128,16384,control_optimized_split,0,6,100,1452.8678894,13697024,9.42757707008,4194304,2909,true,1,1.4528678894 +bf16,16,64,16,128,16384,optimized_split,0,7,100,601.861495972,13697024,22.7577675124,4194304,2909,true,1,0.601861495972 +bf16,16,64,16,128,16384,control_shuffle32_split,0,8,100,1612.77923584,13697024,8.49280775423,4194304,2909,true,1,1.61277923584 +bf16,16,64,16,128,16384,shuffle32_split,0,9,100,288.826980591,13697024,47.4229380232,4194304,2909,true,1,0.288826980591 +bf16,16,64,16,128,16384,baseline_fused,0,10,100,1864.70840454,5308416,2.84678075514,4194304,2909,true,1,1.86470840454 +bf16,16,64,16,128,16384,control_optimized_fused,0,11,100,1421.76559448,5308416,3.73367875872,4194304,2909,true,1,1.42176559448 +bf16,16,64,16,128,16384,optimized_fused,0,12,100,566.301269531,5308416,9.37383736468,4194304,2909,true,1,0.566301269531 +bf16,16,64,16,128,16384,control_shuffle32_fused,0,13,100,1496.89041138,5308416,3.54629568047,4194304,2909,true,1,1.49689041138 +bf16,16,64,16,128,16384,shuffle32_fused,0,14,100,265.197029114,5308416,20.0168758215,4194304,2909,true,1,0.265197029114 +bf16,16,64,16,128,16384,control_optimized_transform,1,0,100,87.1634292603,8388608,96.2399950437,4194304,2909,true,1,0.0871634292603 +bf16,16,64,16,128,16384,optimized_transform,1,1,100,87.2073173523,8388608,96.1915611521,4194304,2909,true,1,0.0872073173523 +bf16,16,64,16,128,16384,control_shuffle32_transform,1,2,100,26.6022872925,8388608,315.334087921,4194304,2909,true,1,0.0266022872925 +bf16,16,64,16,128,16384,shuffle32_transform,1,3,100,26.5510845184,8388608,315.942197923,4194304,2909,true,1,0.0265510845184 +bf16,16,64,16,128,16384,baseline_split,1,4,100,1884.60144043,13697024,7.26786242765,4194304,2909,true,1,1.88460144043 +bf16,16,64,16,128,16384,control_optimized_split,1,5,100,1452.90206909,13697024,9.42735528525,4194304,2909,true,1,1.45290206909 +bf16,16,64,16,128,16384,optimized_split,1,6,100,601.767997742,13697024,22.7613034449,4194304,2909,true,1,0.601767997742 +bf16,16,64,16,128,16384,control_shuffle32_split,1,7,100,1615.72631836,13697024,8.4773168849,4194304,2909,true,1,1.61572631836 +bf16,16,64,16,128,16384,shuffle32_split,1,8,100,288.874511719,13697024,47.4151350997,4194304,2909,true,1,0.288874511719 +bf16,16,64,16,128,16384,baseline_fused,1,9,100,1864.82315063,5308416,2.84660558734,4194304,2909,true,1,1.86482315063 +bf16,16,64,16,128,16384,control_optimized_fused,1,10,100,1421.45004272,5308416,3.73450760874,4194304,2909,true,1,1.42145004272 +bf16,16,64,16,128,16384,optimized_fused,1,11,100,566.259880066,5308416,9.3745225238,4194304,2909,true,1,0.566259880066 +bf16,16,64,16,128,16384,control_shuffle32_fused,1,12,100,1497.93365479,5308416,3.54382584505,4194304,2909,true,1,1.49793365479 +bf16,16,64,16,128,16384,shuffle32_fused,1,13,100,265.167312622,5308416,20.0191190517,4194304,2909,true,1,0.265167312622 +bf16,16,64,16,128,16384,baseline_transform,1,14,100,78.9213752747,8388608,106.290697176,4194304,2909,true,1,0.0789213752747 +bf16,16,64,16,128,16384,optimized_transform,2,0,100,87.1609115601,8388608,96.2427749992,4194304,2909,true,1,0.0871609115601 +bf16,16,64,16,128,16384,control_shuffle32_transform,2,1,100,26.6093707085,8388608,315.250145969,4194304,2909,true,1,0.0266093707085 +bf16,16,64,16,128,16384,shuffle32_transform,2,2,100,26.5775990486,8388608,315.627005459,4194304,2909,true,1,0.0265775990486 +bf16,16,64,16,128,16384,baseline_split,2,3,100,1884.49310303,13697024,7.26828024894,4194304,2909,true,1,1.88449310303 +bf16,16,64,16,128,16384,control_optimized_split,2,4,100,1452.57827759,13697024,9.42945671936,4194304,2909,true,1,1.45257827759 +bf16,16,64,16,128,16384,optimized_split,2,5,100,601.787414551,13697024,22.7605690462,4194304,2909,true,1,0.601787414551 +bf16,16,64,16,128,16384,control_shuffle32_split,2,6,100,1604.17572021,13697024,8.53835638291,4194304,2909,true,1,1.60417572021 +bf16,16,64,16,128,16384,shuffle32_split,2,7,100,288.94903183,13697024,47.4029067108,4194304,2909,true,1,0.28894903183 +bf16,16,64,16,128,16384,baseline_fused,2,8,100,1864.63058472,5308416,2.84689956472,4194304,2909,true,1,1.86463058472 +bf16,16,64,16,128,16384,control_optimized_fused,2,9,100,1421.43997192,5308416,3.73453406746,4194304,2909,true,1,1.42143997192 +bf16,16,64,16,128,16384,optimized_fused,2,10,100,566.235427856,5308416,9.3749273515,4194304,2909,true,1,0.566235427856 +bf16,16,64,16,128,16384,control_shuffle32_fused,2,11,100,1498.66760254,5308416,3.54209031476,4194304,2909,true,1,1.49866760254 +bf16,16,64,16,128,16384,shuffle32_fused,2,12,100,265.194740295,5308416,20.0170485813,4194304,2909,true,1,0.265194740295 +bf16,16,64,16,128,16384,baseline_transform,2,13,100,78.9010238647,8388608,106.318113367,4194304,2909,true,1,0.0789010238647 +bf16,16,64,16,128,16384,control_optimized_transform,2,14,100,87.1638870239,8388608,96.2394896145,4194304,2909,true,1,0.0871638870239 +bf16,16,64,16,128,16384,control_shuffle32_transform,3,0,100,26.5648007393,8388608,315.779067283,4194304,2909,true,1,0.0265648007393 +bf16,16,64,16,128,16384,shuffle32_transform,3,1,100,26.6909718513,8388608,314.286345462,4194304,2909,true,1,0.0266909718513 +bf16,16,64,16,128,16384,baseline_split,3,2,100,1884.54315186,13697024,7.26808722131,4194304,2909,true,1,1.88454315186 +bf16,16,64,16,128,16384,control_optimized_split,3,3,100,1453.09234619,13697024,9.42612080774,4194304,2909,true,1,1.45309234619 +bf16,16,64,16,128,16384,optimized_split,3,4,100,601.860580444,13697024,22.7578021307,4194304,2909,true,1,0.601860580444 +bf16,16,64,16,128,16384,control_shuffle32_split,3,5,100,1609.26742554,13697024,8.51134111251,4194304,2909,true,1,1.60926742554 +bf16,16,64,16,128,16384,shuffle32_split,3,6,100,288.967761993,13697024,47.3998341736,4194304,2909,true,1,0.288967761993 +bf16,16,64,16,128,16384,baseline_fused,3,7,100,1864.79156494,5308416,2.84665380292,4194304,2909,true,1,1.86479156494 +bf16,16,64,16,128,16384,control_optimized_fused,3,8,100,1421.60751343,5308416,3.73409393933,4194304,2909,true,1,1.42160751343 +bf16,16,64,16,128,16384,optimized_fused,3,9,100,566.204338074,5308416,9.37544212052,4194304,2909,true,1,0.566204338074 +bf16,16,64,16,128,16384,control_shuffle32_fused,3,10,100,1497.54898071,5308416,3.54473614444,4194304,2909,true,1,1.49754898071 +bf16,16,64,16,128,16384,shuffle32_fused,3,11,100,265.172576904,5308416,20.0187216264,4194304,2909,true,1,0.265172576904 +bf16,16,64,16,128,16384,baseline_transform,3,12,100,78.9277744293,8388608,106.282079542,4194304,2909,true,1,0.0789277744293 +bf16,16,64,16,128,16384,control_optimized_transform,3,13,100,87.1019458771,8388608,96.3079287785,4194304,2909,true,1,0.0871019458771 +bf16,16,64,16,128,16384,optimized_transform,3,14,100,87.2178268433,8388608,96.1799703526,4194304,2909,true,1,0.0872178268433 +bf16,16,64,16,128,16384,shuffle32_transform,4,0,100,26.6646838188,8388608,314.596192364,4194304,2909,true,1,0.0266646838188 +bf16,16,64,16,128,16384,baseline_split,4,1,100,1884.36477661,13697024,7.26877522336,4194304,2909,true,1,1.88436477661 +bf16,16,64,16,128,16384,control_optimized_split,4,2,100,1452.80639648,13697024,9.42797611103,4194304,2909,true,1,1.45280639648 +bf16,16,64,16,128,16384,optimized_split,4,3,100,601.746520996,13697024,22.7621158114,4194304,2909,true,1,0.601746520996 +bf16,16,64,16,128,16384,control_shuffle32_split,4,4,100,1602.70446777,13697024,8.54619443286,4194304,2909,true,1,1.60270446777 +bf16,16,64,16,128,16384,shuffle32_split,4,5,100,288.835201263,13697024,47.421588297,4194304,2909,true,1,0.288835201263 +bf16,16,64,16,128,16384,baseline_fused,4,6,100,1864.49554443,5308416,2.84710575783,4194304,2909,true,1,1.86449554443 +bf16,16,64,16,128,16384,control_optimized_fused,4,7,100,1421.44119263,5308416,3.73453086032,4194304,2909,true,1,1.42144119263 +bf16,16,64,16,128,16384,optimized_fused,4,8,100,566.221961975,5308416,9.37515030587,4194304,2909,true,1,0.566221961975 +bf16,16,64,16,128,16384,control_shuffle32_fused,4,9,100,1496.5586853,5308416,3.54708174971,4194304,2909,true,1,1.4965586853 +bf16,16,64,16,128,16384,shuffle32_fused,4,10,100,265.24684906,5308416,20.0131161551,4194304,2909,true,1,0.26524684906 +bf16,16,64,16,128,16384,baseline_transform,4,11,100,78.938741684,8388608,106.267313376,4194304,2909,true,1,0.078938741684 +bf16,16,64,16,128,16384,control_optimized_transform,4,12,100,87.1803379059,8388608,96.2213292756,4194304,2909,true,1,0.0871803379059 +bf16,16,64,16,128,16384,optimized_transform,4,13,100,87.2139453888,8388608,96.1842508397,4194304,2909,true,1,0.0872139453888 +bf16,16,64,16,128,16384,control_shuffle32_transform,4,14,100,26.697371006,8388608,314.211013441,4194304,2909,true,1,0.026697371006 +bf16,1,1,1,256,1,baseline_transform,0,0,100,10.1556575298,1024,0.100830497385,512,2909,true,1,0.0101556575298 +bf16,1,1,1,256,1,control_optimized_transform,0,1,100,10.9501719475,1024,0.0935145132799,512,2909,true,1,0.0109501719475 +bf16,1,1,1,256,1,optimized_transform,0,2,100,10.8274281025,1024,0.0945746293863,512,2909,true,1,0.0108274281025 +bf16,1,1,1,256,1,control_shuffle32_transform,0,3,100,11.8980562687,1024,0.0860644778336,512,2909,true,1,0.0118980562687 +bf16,1,1,1,256,1,shuffle32_transform,0,4,100,11.8706285954,1024,0.0862633340581,512,2909,true,1,0.0118706285954 +bf16,1,1,1,256,1,baseline_split,0,5,100,100.727767944,1668,0.0165594853737,512,2909,true,1,0.100727767944 +bf16,1,1,1,256,1,control_optimized_split,0,6,100,67.5764608383,1668,0.0246831511936,512,2909,true,1,0.0675764608383 +bf16,1,1,1,256,1,optimized_split,0,7,100,35.8237719536,1668,0.0465612611135,512,2909,true,1,0.0358237719536 +bf16,1,1,1,256,1,control_shuffle32_split,0,8,100,304.967098236,1668,0.00546944247313,512,2909,true,1,0.304967098236 +bf16,1,1,1,256,1,shuffle32_split,0,9,100,68.6870861053,1668,0.0242840407794,512,2909,true,1,0.0686870861053 +bf16,1,1,1,256,1,baseline_fused,0,10,100,96.012802124,644,0.00670743885975,512,2909,true,1,0.096012802124 +bf16,1,1,1,256,1,control_optimized_fused,0,11,100,61.1766862869,644,0.0105268859608,512,2909,true,1,0.0611766862869 +bf16,1,1,1,256,1,optimized_fused,0,12,100,30.0173711777,644,0.0214542438173,512,2909,true,1,0.0300173711777 +bf16,1,1,1,256,1,control_shuffle32_fused,0,13,100,300.689601898,644,0.00214174349873,512,2909,true,1,0.300689601898 +bf16,1,1,1,256,1,shuffle32_fused,0,14,100,61.5355396271,644,0.0104654969129,512,2909,true,1,0.0615355396271 +bf16,1,1,1,256,1,control_optimized_transform,1,0,100,10.8687996864,1024,0.0942146354283,512,2909,true,1,0.0108687996864 +bf16,1,1,1,256,1,optimized_transform,1,1,100,10.7595431805,1024,0.0951713267771,512,2909,true,1,0.0107595431805 +bf16,1,1,1,256,1,control_shuffle32_transform,1,2,100,12.0070850849,1024,0.0852829802369,512,2909,true,1,0.0120070850849 +bf16,1,1,1,256,1,shuffle32_transform,1,3,100,11.8827426434,1024,0.0861753915518,512,2909,true,1,0.0118827426434 +bf16,1,1,1,256,1,baseline_split,1,4,100,100.664224625,1668,0.0165699383889,512,2909,true,1,0.100664224625 +bf16,1,1,1,256,1,control_optimized_split,1,5,100,67.6656007767,1668,0.0246506346039,512,2909,true,1,0.0676656007767 +bf16,1,1,1,256,1,optimized_split,1,6,100,36.3238859177,1668,0.0459201970786,512,2909,true,1,0.0363238859177 +bf16,1,1,1,256,1,control_shuffle32_split,1,7,100,305.012111664,1668,0.00546863529747,512,2909,true,1,0.305012111664 +bf16,1,1,1,256,1,shuffle32_split,1,8,100,68.1001138687,1668,0.0244933511156,512,2909,true,1,0.0681001138687 +bf16,1,1,1,256,1,baseline_fused,1,9,100,96.008682251,644,0.00670772668576,512,2909,true,1,0.096008682251 +bf16,1,1,1,256,1,control_optimized_fused,1,10,100,61.1545133591,644,0.0105307027172,512,2909,true,1,0.0611545133591 +bf16,1,1,1,256,1,optimized_fused,1,11,100,30.004799366,644,0.021463233003,512,2909,true,1,0.030004799366 +bf16,1,1,1,256,1,control_shuffle32_fused,1,12,100,300.697841644,644,0.0021416848105,512,2909,true,1,0.300697841644 +bf16,1,1,1,256,1,shuffle32_fused,1,13,100,61.5490293503,644,0.0104632031861,512,2909,true,1,0.0615490293503 +bf16,1,1,1,256,1,baseline_transform,1,14,100,10.1746284962,1024,0.100642495241,512,2909,true,1,0.0101746284962 +bf16,1,1,1,256,1,optimized_transform,2,0,100,10.8436572552,1024,0.0944330843278,512,2909,true,1,0.0108436572552 +bf16,1,1,1,256,1,control_shuffle32_transform,2,1,100,11.9131433964,1024,0.0859554834462,512,2909,true,1,0.0119131433964 +bf16,1,1,1,256,1,shuffle32_transform,2,2,100,11.9723427296,1024,0.0855304615922,512,2909,true,1,0.0119723427296 +bf16,1,1,1,256,1,baseline_split,2,3,100,100.683193207,1668,0.0165668166342,512,2909,true,1,0.100683193207 +bf16,1,1,1,256,1,control_optimized_split,2,4,100,67.5579452515,1668,0.0246899160978,512,2909,true,1,0.0675579452515 +bf16,1,1,1,256,1,optimized_split,2,5,100,35.7398867607,1668,0.0466705451858,512,2909,true,1,0.0357398867607 +bf16,1,1,1,256,1,control_shuffle32_split,2,6,100,304.991550446,1668,0.00546900396933,512,2909,true,1,0.304991550446 +bf16,1,1,1,256,1,shuffle32_split,2,7,100,68.038854599,1668,0.0245154038796,512,2909,true,1,0.068038854599 +bf16,1,1,1,256,1,baseline_fused,2,8,100,96.0089206696,644,0.00670771002849,512,2909,true,1,0.0960089206696 +bf16,1,1,1,256,1,control_optimized_fused,2,9,100,61.1851406097,644,0.0105254313969,512,2909,true,1,0.0611851406097 +bf16,1,1,1,256,1,optimized_fused,2,10,100,30.0203418732,644,0.0214521207893,512,2909,true,1,0.0300203418732 +bf16,1,1,1,256,1,control_shuffle32_fused,2,11,100,300.695323944,644,0.00214170274267,512,2909,true,1,0.300695323944 +bf16,1,1,1,256,1,shuffle32_fused,2,12,100,61.5513134003,644,0.0104628149169,512,2909,true,1,0.0615513134003 +bf16,1,1,1,256,1,baseline_transform,2,13,100,10.1535999775,1024,0.100850929943,512,2909,true,1,0.0101535999775 +bf16,1,1,1,256,1,control_optimized_transform,2,14,100,11.0331428051,1024,0.0928112703777,512,2909,true,1,0.0110331428051 +bf16,1,1,1,256,1,control_shuffle32_transform,3,0,100,11.8946290016,1024,0.0860892760809,512,2909,true,1,0.0118946290016 +bf16,1,1,1,256,1,shuffle32_transform,3,1,100,11.9769144058,1024,0.0854978139864,512,2909,true,1,0.0119769144058 +bf16,1,1,1,256,1,baseline_split,3,2,100,100.556344986,1668,0.0165877150789,512,2909,true,1,0.100556344986 +bf16,1,1,1,256,1,control_optimized_split,3,3,100,67.604341507,1668,0.024672971629,512,2909,true,1,0.067604341507 +bf16,1,1,1,256,1,optimized_split,3,4,100,35.7341718674,1668,0.046678009111,512,2909,true,1,0.0357341718674 +bf16,1,1,1,256,1,control_shuffle32_split,3,5,100,304.898509979,1668,0.0054706728482,512,2909,true,1,0.304898509979 +bf16,1,1,1,256,1,shuffle32_split,3,6,100,68.7195396423,1668,0.0242725723816,512,2909,true,1,0.0687195396423 +bf16,1,1,1,256,1,baseline_fused,3,7,100,96.0130214691,644,0.00670742353637,512,2909,true,1,0.0960130214691 +bf16,1,1,1,256,1,control_optimized_fused,3,8,100,61.1869716644,644,0.0105251164175,512,2909,true,1,0.0611869716644 +bf16,1,1,1,256,1,optimized_fused,3,9,100,29.991543293,644,0.0214727196166,512,2909,true,1,0.029991543293 +bf16,1,1,1,256,1,control_shuffle32_fused,3,10,100,300.689144135,644,0.00214174675928,512,2909,true,1,0.300689144135 +bf16,1,1,1,256,1,shuffle32_fused,3,11,100,61.5545129776,644,0.0104622710643,512,2909,true,1,0.0615545129776 +bf16,1,1,1,256,1,baseline_transform,3,12,100,10.069257021,1024,0.101695685975,512,2909,true,1,0.010069257021 +bf16,1,1,1,256,1,control_optimized_transform,3,13,100,10.9110856056,1024,0.0938495065489,512,2909,true,1,0.0109110856056 +bf16,1,1,1,256,1,optimized_transform,3,14,100,10.8800005913,1024,0.094117641944,512,2909,true,1,0.0108800005913 +bf16,1,1,1,256,1,shuffle32_transform,4,0,100,11.9522285461,1024,0.0856743992174,512,2909,true,1,0.0119522285461 +bf16,1,1,1,256,1,baseline_split,4,1,100,100.666513443,1668,0.0165695616442,512,2909,true,1,0.100666513443 +bf16,1,1,1,256,1,control_optimized_split,4,2,100,67.0226287842,1668,0.0248871169373,512,2909,true,1,0.0670226287842 +bf16,1,1,1,256,1,optimized_split,4,3,100,35.8374857903,1668,0.0465434436378,512,2909,true,1,0.0358374857903 +bf16,1,1,1,256,1,control_shuffle32_split,4,4,100,304.950180054,1668,0.00546974590966,512,2909,true,1,0.304950180054 +bf16,1,1,1,256,1,shuffle32_split,4,5,100,68.1078863144,1668,0.0244905559439,512,2909,true,1,0.0681078863144 +bf16,1,1,1,256,1,baseline_fused,4,6,100,96.0091400146,644,0.00670769470388,512,2909,true,1,0.0960091400146 +bf16,1,1,1,256,1,control_optimized_fused,4,7,100,61.1851406097,644,0.0105254313969,512,2909,true,1,0.0611851406097 +bf16,1,1,1,256,1,optimized_fused,4,8,100,30.0379419327,644,0.0214395513995,512,2909,true,1,0.0300379419327 +bf16,1,1,1,256,1,control_shuffle32_fused,4,9,100,300.690975189,644,0.00214173371713,512,2909,true,1,0.300690975189 +bf16,1,1,1,256,1,shuffle32_fused,4,10,100,61.5375995636,644,0.0104651465863,512,2909,true,1,0.0615375995636 +bf16,1,1,1,256,1,baseline_transform,4,11,100,10.1730287075,1024,0.100658322063,512,2909,true,1,0.0101730287075 +bf16,1,1,1,256,1,control_optimized_transform,4,12,100,10.9572565556,1024,0.0934540498169,512,2909,true,1,0.0109572565556 +bf16,1,1,1,256,1,optimized_transform,4,13,100,10.9670853615,1024,0.0933702954111,512,2909,true,1,0.0109670853615 +bf16,1,1,1,256,1,control_shuffle32_transform,4,14,100,11.9003427029,1024,0.0860479421112,512,2909,true,1,0.0119003427029 +bf16,1,17,1,256,17,baseline_transform,0,0,100,10.4523432255,17408,1.66546387011,8704,2909,true,1,0.0104523432255 +bf16,1,17,1,256,17,control_optimized_transform,0,1,100,11.0658276081,17408,1.57313132072,8704,2909,true,1,0.0110658276081 +bf16,1,17,1,256,17,optimized_transform,0,2,100,11.0923433304,17408,1.56937082468,8704,2909,true,1,0.0110923433304 +bf16,1,17,1,256,17,control_shuffle32_transform,0,3,100,12.0610284805,17408,1.44332633225,8704,2909,true,1,0.0120610284805 +bf16,1,17,1,256,17,shuffle32_transform,0,4,100,12.1353149414,17408,1.43449099459,8704,2909,true,1,0.0121353149414 +bf16,1,17,1,256,17,baseline_split,0,5,100,101.364116669,28356,0.279743965931,8704,2909,true,1,0.101364116669 +bf16,1,17,1,256,17,control_optimized_split,0,6,100,67.4580526352,28356,0.420350112289,8704,2909,true,1,0.0674580526352 +bf16,1,17,1,256,17,optimized_split,0,7,100,36.4230847359,28356,0.77851725645,8704,2909,true,1,0.0364230847359 +bf16,1,17,1,256,17,control_shuffle32_split,0,8,100,305.452575684,28356,0.0928327415035,8704,2909,true,1,0.305452575684 +bf16,1,17,1,256,17,shuffle32_split,0,9,100,68.2998847961,28356,0.415169075096,8704,2909,true,1,0.0682998847961 +bf16,1,17,1,256,17,baseline_fused,0,10,100,96.5366840363,10948,0.113407665794,8704,2909,true,1,0.0965366840363 +bf16,1,17,1,256,17,control_optimized_fused,0,11,100,61.5917682648,10948,0.177751025964,8704,2909,true,1,0.0615917682648 +bf16,1,17,1,256,17,optimized_fused,0,12,100,30.2806854248,10948,0.361550600537,8704,2909,true,1,0.0302806854248 +bf16,1,17,1,256,17,control_shuffle32_fused,0,13,100,301.216220856,10948,0.0363459841867,8704,2909,true,1,0.301216220856 +bf16,1,17,1,256,17,shuffle32_fused,0,14,100,61.4454889297,10948,0.178174186432,8704,2909,true,1,0.0614454889297 +bf16,1,17,1,256,17,control_optimized_transform,1,0,100,11.1220562458,17408,1.56517820224,8704,2909,true,1,0.0111220562458 +bf16,1,17,1,256,17,optimized_transform,1,1,100,11.140114069,17408,1.56264109076,8704,2909,true,1,0.011140114069 +bf16,1,17,1,256,17,control_shuffle32_transform,1,2,100,12.159999609,17408,1.4315789934,8704,2909,true,1,0.012159999609 +bf16,1,17,1,256,17,shuffle32_transform,1,3,100,12.1634292603,17408,1.43117533942,8704,2909,true,1,0.0121634292603 +bf16,1,17,1,256,17,baseline_split,1,4,100,101.304683685,28356,0.279908084883,8704,2909,true,1,0.101304683685 +bf16,1,17,1,256,17,control_optimized_split,1,5,100,67.4774885178,28356,0.420229036718,8704,2909,true,1,0.0674774885178 +bf16,1,17,1,256,17,optimized_split,1,6,100,36.4697146416,28356,0.77752185008,8704,2909,true,1,0.0364697146416 +bf16,1,17,1,256,17,control_shuffle32_split,1,7,100,305.587882996,28356,0.0927916372928,8704,2909,true,1,0.305587882996 +bf16,1,17,1,256,17,shuffle32_split,1,8,100,68.1885719299,28356,0.415846808306,8704,2909,true,1,0.0681885719299 +bf16,1,17,1,256,17,baseline_fused,1,9,100,96.5268516541,10948,0.113419217683,8704,2909,true,1,0.0965268516541 +bf16,1,17,1,256,17,control_optimized_fused,1,10,100,61.6031980515,10948,0.177718046243,8704,2909,true,1,0.0616031980515 +bf16,1,17,1,256,17,optimized_fused,1,11,100,30.304684639,10948,0.361264277468,8704,2909,true,1,0.030304684639 +bf16,1,17,1,256,17,control_shuffle32_fused,1,12,100,301.20639801,10948,0.0363471694902,8704,2909,true,1,0.30120639801 +bf16,1,17,1,256,17,shuffle32_fused,1,13,100,61.4715433121,10948,0.178098668264,8704,2909,true,1,0.0614715433121 +bf16,1,17,1,256,17,baseline_transform,1,14,100,10.4132568836,17408,1.67171521788,8704,2909,true,1,0.0104132568836 +bf16,1,17,1,256,17,optimized_transform,2,0,100,11.0765707493,17408,1.57160554417,8704,2909,true,1,0.0110765707493 +bf16,1,17,1,256,17,control_shuffle32_transform,2,1,100,12.1529150009,17408,1.43241354018,8704,2909,true,1,0.0121529150009 +bf16,1,17,1,256,17,shuffle32_transform,2,2,100,12.187885046,17408,1.42830359281,8704,2909,true,1,0.012187885046 +bf16,1,17,1,256,17,baseline_split,2,3,100,101.418972015,28356,0.27959265842,8704,2909,true,1,0.101418972015 +bf16,1,17,1,256,17,control_optimized_split,2,4,100,67.4356555939,28356,0.420489720909,8704,2909,true,1,0.0674356555939 +bf16,1,17,1,256,17,optimized_split,2,5,100,36.5115427971,28356,0.776631109717,8704,2909,true,1,0.0365115427971 +bf16,1,17,1,256,17,control_shuffle32_split,2,6,100,305.578746796,28356,0.0927944115792,8704,2909,true,1,0.305578746796 +bf16,1,17,1,256,17,shuffle32_split,2,7,100,68.3211421967,28356,0.415039899631,8704,2909,true,1,0.0683211421967 +bf16,1,17,1,256,17,baseline_fused,2,8,100,96.5373706818,10948,0.113406859154,8704,2909,true,1,0.0965373706818 +bf16,1,17,1,256,17,control_optimized_fused,2,9,100,61.6340589523,10948,0.177629060719,8704,2909,true,1,0.0616340589523 +bf16,1,17,1,256,17,optimized_fused,2,10,100,30.2866292,10948,0.361479645942,8704,2909,true,1,0.0302866292 +bf16,1,17,1,256,17,control_shuffle32_fused,2,11,100,301.226291656,10948,0.0363447690432,8704,2909,true,1,0.301226291656 +bf16,1,17,1,256,17,shuffle32_fused,2,12,100,61.4706277847,10948,0.178101320818,8704,2909,true,1,0.0614706277847 +bf16,1,17,1,256,17,baseline_transform,2,13,100,10.5149722099,17408,1.65554408062,8704,2909,true,1,0.0105149722099 +bf16,1,17,1,256,17,control_optimized_transform,2,14,100,11.0914289951,17408,1.56950019764,8704,2909,true,1,0.0110914289951 +bf16,1,17,1,256,17,control_shuffle32_transform,3,0,100,12.2315430641,17408,1.42320555213,8704,2909,true,1,0.0122315430641 +bf16,1,17,1,256,17,shuffle32_transform,3,1,100,12.1245718002,17408,1.43576204478,8704,2909,true,1,0.0121245718002 +bf16,1,17,1,256,17,baseline_split,3,2,100,101.378746033,28356,0.279703597743,8704,2909,true,1,0.101378746033 +bf16,1,17,1,256,17,control_optimized_split,3,3,100,68.0480003357,28356,0.416705852635,8704,2909,true,1,0.0680480003357 +bf16,1,17,1,256,17,optimized_split,3,4,100,36.5391993523,28356,0.776043276883,8704,2909,true,1,0.0365391993523 +bf16,1,17,1,256,17,control_shuffle32_split,3,5,100,305.496921539,28356,0.0928192659262,8704,2909,true,1,0.305496921539 +bf16,1,17,1,256,17,shuffle32_split,3,6,100,68.2358884811,28356,0.415558449244,8704,2909,true,1,0.0682358884811 +bf16,1,17,1,256,17,baseline_fused,3,7,100,96.5421772003,10948,0.113401212998,8704,2909,true,1,0.0965421772003 +bf16,1,17,1,256,17,control_optimized_fused,3,8,100,61.6054821014,10948,0.177711457269,8704,2909,true,1,0.0616054821014 +bf16,1,17,1,256,17,optimized_fused,3,9,100,30.298743248,10948,0.361335119097,8704,2909,true,1,0.030298743248 +bf16,1,17,1,256,17,control_shuffle32_fused,3,10,100,301.209831238,10948,0.0363467552006,8704,2909,true,1,0.301209831238 +bf16,1,17,1,256,17,shuffle32_fused,3,11,100,61.4594316483,10948,0.178133765744,8704,2909,true,1,0.0614594316483 +bf16,1,17,1,256,17,baseline_transform,3,12,100,10.403200388,17408,1.67333122028,8704,2909,true,1,0.010403200388 +bf16,1,17,1,256,17,control_optimized_transform,3,13,100,11.1181724072,17408,1.56572495573,8704,2909,true,1,0.0111181724072 +bf16,1,17,1,256,17,optimized_transform,3,14,100,11.1780571938,17408,1.55733681607,8704,2909,true,1,0.0111780571938 +bf16,1,17,1,256,17,shuffle32_transform,4,0,100,12.0381724834,17408,1.44606667033,8704,2909,true,1,0.0120381724834 +bf16,1,17,1,256,17,baseline_split,4,1,100,101.380567551,28356,0.279698572272,8704,2909,true,1,0.101380567551 +bf16,1,17,1,256,17,control_optimized_split,4,2,100,67.4374866486,28356,0.420478303822,8704,2909,true,1,0.0674374866486 +bf16,1,17,1,256,17,optimized_split,4,3,100,36.5087985992,28356,0.776689485493,8704,2909,true,1,0.0365087985992 +bf16,1,17,1,256,17,control_shuffle32_split,4,4,100,305.523204803,28356,0.0928112809573,8704,2909,true,1,0.305523204803 +bf16,1,17,1,256,17,shuffle32_split,4,5,100,68.2150840759,28356,0.415685187288,8704,2909,true,1,0.0682150840759 +bf16,1,17,1,256,17,baseline_fused,4,6,100,96.5282344818,10948,0.113417592881,8704,2909,true,1,0.0965282344818 +bf16,1,17,1,256,17,control_optimized_fused,4,7,100,61.583313942,10948,0.177775428102,8704,2909,true,1,0.061583313942 +bf16,1,17,1,256,17,optimized_fused,4,8,100,30.277941227,10948,0.361583369158,8704,2909,true,1,0.030277941227 +bf16,1,17,1,256,17,control_shuffle32_fused,4,9,100,301.221942902,10948,0.0363452937543,8704,2909,true,1,0.301221942902 +bf16,1,17,1,256,17,shuffle32_fused,4,10,100,61.445941925,10948,0.178172872886,8704,2909,true,1,0.061445941925 +bf16,1,17,1,256,17,baseline_transform,4,11,100,10.5654859543,17408,1.64762889992,8704,2909,true,1,0.0105654859543 +bf16,1,17,1,256,17,control_optimized_transform,4,12,100,11.1506283283,17408,1.56116762997,8704,2909,true,1,0.0111506283283 +bf16,1,17,1,256,17,optimized_transform,4,13,100,11.1846864223,17408,1.5564137735,8704,2909,true,1,0.0111846864223 +bf16,1,17,1,256,17,control_shuffle32_transform,4,14,100,12.1769142151,17408,1.42959042763,8704,2909,true,1,0.0121769142151 +bf16,1,257,1,256,257,baseline_transform,0,0,100,10.9467434883,263168,24.0407569869,131584,2909,true,1,0.0109467434883 +bf16,1,257,1,256,257,control_optimized_transform,0,1,100,11.497143507,263168,22.8898595412,131584,2909,true,1,0.011497143507 +bf16,1,257,1,256,257,optimized_transform,0,2,100,11.3942861557,263168,23.0964885736,131584,2909,true,1,0.0113942861557 +bf16,1,257,1,256,257,control_shuffle32_transform,0,3,100,14.05782938,263168,18.7203865466,131584,2909,true,1,0.01405782938 +bf16,1,257,1,256,257,shuffle32_transform,0,4,100,14.2061710358,263168,18.5249071926,131584,2909,true,1,0.0142061710358 +bf16,1,257,1,256,257,baseline_split,0,5,100,104.163885117,428676,4.11539949302,131584,2909,true,1,0.104163885117 +bf16,1,257,1,256,257,control_optimized_split,0,6,100,69.6304035187,428676,6.15644859627,131584,2909,true,1,0.0696304035187 +bf16,1,257,1,256,257,optimized_split,0,7,100,36.9894862175,428676,11.5891309622,131584,2909,true,1,0.0369894862175 +bf16,1,257,1,256,257,control_shuffle32_split,0,8,100,328.979187012,428676,1.303048998,131584,2909,true,1,0.328979187012 +bf16,1,257,1,256,257,shuffle32_split,0,9,100,77.4621725082,428676,5.53400435489,131584,2909,true,1,0.0774621725082 +bf16,1,257,1,256,257,baseline_fused,0,10,100,98.8765716553,165508,1.6738848974,131584,2909,true,1,0.0988765716553 +bf16,1,257,1,256,257,control_optimized_fused,0,11,100,63.7691402435,165508,2.59542467356,131584,2909,true,1,0.0637691402435 +bf16,1,257,1,256,257,optimized_fused,0,12,100,30.7446861267,165508,5.38330426656,131584,2909,true,1,0.0307446861267 +bf16,1,257,1,256,257,control_shuffle32_fused,0,13,100,321.57119751,165508,0.514685398698,131584,2909,true,1,0.32157119751 +bf16,1,257,1,256,257,shuffle32_fused,0,14,100,68.3602285385,165508,2.42111536983,131584,2909,true,1,0.0683602285385 +bf16,1,257,1,256,257,control_optimized_transform,1,0,100,11.365942955,263168,23.154084183,131584,2909,true,1,0.011365942955 +bf16,1,257,1,256,257,optimized_transform,1,1,100,11.4093708992,263168,23.0659518676,131584,2909,true,1,0.0114093708992 +bf16,1,257,1,256,257,control_shuffle32_transform,1,2,100,14.0354287624,263168,18.7502643813,131584,2909,true,1,0.0140354287624 +bf16,1,257,1,256,257,shuffle32_transform,1,3,100,14.1844570637,263168,18.5532656498,131584,2909,true,1,0.0141844570637 +bf16,1,257,1,256,257,baseline_split,1,4,100,104.13851738,428676,4.11640198829,131584,2909,true,1,0.10413851738 +bf16,1,257,1,256,257,control_optimized_split,1,5,100,70.3574848175,428676,6.09282723952,131584,2909,true,1,0.0703574848175 +bf16,1,257,1,256,257,optimized_split,1,6,100,37.1410274506,428676,11.5418454853,131584,2909,true,1,0.0371410274506 +bf16,1,257,1,256,257,control_shuffle32_split,1,7,100,328.385391235,428676,1.30540520815,131584,2909,true,1,0.328385391235 +bf16,1,257,1,256,257,shuffle32_split,1,8,100,76.8614816666,428676,5.57725392102,131584,2909,true,1,0.0768614816666 +bf16,1,257,1,256,257,baseline_fused,1,9,100,98.8735961914,165508,1.67393527064,131584,2909,true,1,0.0988735961914 +bf16,1,257,1,256,257,control_optimized_fused,1,10,100,63.7732601166,165508,2.59525700423,131584,2909,true,1,0.0637732601166 +bf16,1,257,1,256,257,optimized_fused,1,11,100,30.7396554947,165508,5.38418525961,131584,2909,true,1,0.0307396554947 +bf16,1,257,1,256,257,control_shuffle32_fused,1,12,100,321.57875061,165508,0.514673309993,131584,2909,true,1,0.32157875061 +bf16,1,257,1,256,257,shuffle32_fused,1,13,100,68.3620548248,165508,2.42105068995,131584,2909,true,1,0.0683620548248 +bf16,1,257,1,256,257,baseline_transform,1,14,100,10.8893716335,263168,24.167418365,131584,2909,true,1,0.0108893716335 +bf16,1,257,1,256,257,optimized_transform,2,0,100,11.4626288414,263168,22.9587822864,131584,2909,true,1,0.0114626288414 +bf16,1,257,1,256,257,control_shuffle32_transform,2,1,100,14.093028307,263168,18.6736302708,131584,2909,true,1,0.014093028307 +bf16,1,257,1,256,257,shuffle32_transform,2,2,100,14.2486858368,263168,18.4696331307,131584,2909,true,1,0.0142486858368 +bf16,1,257,1,256,257,baseline_split,2,3,100,104.069490433,428676,4.11913230494,131584,2909,true,1,0.104069490433 +bf16,1,257,1,256,257,control_optimized_split,2,4,100,70.2431964874,428676,6.10274049924,131584,2909,true,1,0.0702431964874 +bf16,1,257,1,256,257,optimized_split,2,5,100,37.0882296562,428676,11.5582761424,131584,2909,true,1,0.0370882296562 +bf16,1,257,1,256,257,control_shuffle32_split,2,6,100,328.461723328,428676,1.30510184157,131584,2909,true,1,0.328461723328 +bf16,1,257,1,256,257,shuffle32_split,2,7,100,76.7714309692,428676,5.58379588068,131584,2909,true,1,0.0767714309692 +bf16,1,257,1,256,257,baseline_fused,2,8,100,98.8745117188,165508,1.67391977086,131584,2909,true,1,0.0988745117188 +bf16,1,257,1,256,257,control_optimized_fused,2,9,100,63.7668561935,165508,2.59551763847,131584,2909,true,1,0.0637668561935 +bf16,1,257,1,256,257,optimized_fused,2,10,100,30.7421708107,165508,5.38374472704,131584,2909,true,1,0.0307421708107 +bf16,1,257,1,256,257,control_shuffle32_fused,2,11,100,321.570281982,165508,0.514686864034,131584,2909,true,1,0.321570281982 +bf16,1,257,1,256,257,shuffle32_fused,2,12,100,68.3643436432,165508,2.42096963388,131584,2909,true,1,0.0683643436432 +bf16,1,257,1,256,257,baseline_transform,2,13,100,10.8473145962,263168,24.261119899,131584,2909,true,1,0.0108473145962 +bf16,1,257,1,256,257,control_optimized_transform,2,14,100,11.2854850292,263168,23.3191572466,131584,2909,true,1,0.0112854850292 +bf16,1,257,1,256,257,control_shuffle32_transform,3,0,100,14.1250288486,263168,18.6313247796,131584,2909,true,1,0.0141250288486 +bf16,1,257,1,256,257,shuffle32_transform,3,1,100,14.4242286682,263168,18.2448577358,131584,2909,true,1,0.0144242286682 +bf16,1,257,1,256,257,baseline_split,3,2,100,104.169368744,428676,4.1151828524,131584,2909,true,1,0.104169368744 +bf16,1,257,1,256,257,control_optimized_split,3,3,100,70.3051424026,428676,6.09736336988,131584,2909,true,1,0.0703051424026 +bf16,1,257,1,256,257,optimized_split,3,4,100,37.932343483,428676,11.3010681819,131584,2909,true,1,0.037932343483 +bf16,1,257,1,256,257,control_shuffle32_split,3,5,100,328.375549316,428676,1.30544433315,131584,2909,true,1,0.328375549316 +bf16,1,257,1,256,257,shuffle32_split,3,6,100,77.5124549866,428676,5.53041443564,131584,2909,true,1,0.0775124549866 +bf16,1,257,1,256,257,baseline_fused,3,7,100,98.8790893555,165508,1.67384227625,131584,2909,true,1,0.0988790893555 +bf16,1,257,1,256,257,control_optimized_fused,3,8,100,63.7684583664,165508,2.59545242648,131584,2909,true,1,0.0637684583664 +bf16,1,257,1,256,257,optimized_fused,3,9,100,30.7517719269,165508,5.38206384964,131584,2909,true,1,0.0307517719269 +bf16,1,257,1,256,257,control_shuffle32_fused,3,10,100,321.57623291,165508,0.514677339498,131584,2909,true,1,0.32157623291 +bf16,1,257,1,256,257,shuffle32_fused,3,11,100,68.3689165115,165508,2.42080770685,131584,2909,true,1,0.0683689165115 +bf16,1,257,1,256,257,baseline_transform,3,12,100,10.8710861206,263168,24.2080687321,131584,2909,true,1,0.0108710861206 +bf16,1,257,1,256,257,control_optimized_transform,3,13,100,11.4075434208,263168,23.0696470127,131584,2909,true,1,0.0114075434208 +bf16,1,257,1,256,257,optimized_transform,3,14,100,11.5609145164,263168,22.7635970862,131584,2909,true,1,0.0115609145164 +bf16,1,257,1,256,257,shuffle32_transform,4,0,100,14.2582857609,263168,18.4571977595,131584,2909,true,1,0.0142582857609 +bf16,1,257,1,256,257,baseline_split,4,1,100,104.171199799,428676,4.11511051835,131584,2909,true,1,0.104171199799 +bf16,1,257,1,256,257,control_optimized_split,4,2,100,69.7266244888,428676,6.14795285363,131584,2909,true,1,0.0697266244888 +bf16,1,257,1,256,257,optimized_split,4,3,100,37.1213722229,428676,11.5479567249,131584,2909,true,1,0.0371213722229 +bf16,1,257,1,256,257,control_shuffle32_split,4,4,100,328.495330811,428676,1.30496832007,131584,2909,true,1,0.328495330811 +bf16,1,257,1,256,257,shuffle32_split,4,5,100,76.8639993668,428676,5.57707123662,131584,2909,true,1,0.0768639993668 +bf16,1,257,1,256,257,baseline_fused,4,6,100,98.8793182373,165508,1.6738384017,131584,2909,true,1,0.0988793182373 +bf16,1,257,1,256,257,control_optimized_fused,4,7,100,63.7705135345,165508,2.59536878138,131584,2909,true,1,0.0637705135345 +bf16,1,257,1,256,257,optimized_fused,4,8,100,30.7348585129,165508,5.38502560312,131584,2909,true,1,0.0307348585129 +bf16,1,257,1,256,257,control_shuffle32_fused,4,9,100,321.577148438,165508,0.514675874216,131584,2909,true,1,0.321577148438 +bf16,1,257,1,256,257,shuffle32_fused,4,10,100,68.3664035797,165508,2.42089668805,131584,2909,true,1,0.0683664035797 +bf16,1,257,1,256,257,baseline_transform,4,11,100,10.9636569023,263168,24.0036697924,131584,2909,true,1,0.0109636569023 +bf16,1,257,1,256,257,control_optimized_transform,4,12,100,11.3844573498,263168,23.1164289974,131584,2909,true,1,0.0113844573498 +bf16,1,257,1,256,257,optimized_transform,4,13,100,11.4319992065,263168,23.0202955096,131584,2909,true,1,0.0114319992065 +bf16,1,257,1,256,257,control_shuffle32_transform,4,14,100,14.1433131695,263168,18.6072384063,131584,2909,true,1,0.0141433131695 +bf16,4,64,16,256,4096,baseline_transform,0,0,100,49.2642307281,4194304,85.1389322031,2097152,2909,true,1,0.0492642307281 +bf16,4,64,16,256,4096,control_optimized_transform,0,1,100,37.3540568352,4194304,112.285099809,2097152,2909,true,1,0.0373540568352 +bf16,4,64,16,256,4096,optimized_transform,0,2,100,37.4143981934,4194304,112.10400815,2097152,2909,true,1,0.0374143981934 +bf16,4,64,16,256,4096,control_shuffle32_transform,0,3,100,21.9444584846,4194304,191.132718218,2097152,2909,true,1,0.0219444584846 +bf16,4,64,16,256,4096,shuffle32_transform,0,4,100,22.2027420998,4194304,188.909278915,2097152,2909,true,1,0.0222027420998 +bf16,4,64,16,256,4096,baseline_split,0,5,100,1144.91424561,6832128,5.96737094173,2097152,2909,true,1,1.14491424561 +bf16,4,64,16,256,4096,control_optimized_split,0,6,100,500.236778259,6832128,13.6577882653,2097152,2909,true,1,0.500236778259 +bf16,4,64,16,256,4096,optimized_split,0,7,100,188.881607056,6832128,36.1714838544,2097152,2909,true,1,0.188881607056 +bf16,4,64,16,256,4096,control_shuffle32_split,0,8,100,836.410293579,6832128,8.1683930153,2097152,2909,true,1,0.836410293579 +bf16,4,64,16,256,4096,shuffle32_split,0,9,100,149.506521225,6832128,45.6978594915,2097152,2909,true,1,0.149506521225 +bf16,4,64,16,256,4096,baseline_fused,0,10,100,1137.3588562,2637824,2.31925393258,2097152,2909,true,1,1.1373588562 +bf16,4,64,16,256,4096,control_optimized_fused,0,11,100,485.297584534,2637824,5.43547729077,2097152,2909,true,1,0.485297584534 +bf16,4,64,16,256,4096,optimized_fused,0,12,100,172.086162567,2637824,15.328507305,2097152,2909,true,1,0.172086162567 +bf16,4,64,16,256,4096,control_shuffle32_fused,0,13,100,835.176239014,2637824,3.15840403112,2097152,2909,true,1,0.835176239014 +bf16,4,64,16,256,4096,shuffle32_fused,0,14,100,131.349716187,2637824,20.0824491791,2097152,2909,true,1,0.131349716187 +bf16,4,64,16,256,4096,control_optimized_transform,1,0,100,37.3787426949,4194304,112.210943911,2097152,2909,true,1,0.0373787426949 +bf16,4,64,16,256,4096,optimized_transform,1,1,100,37.4139428139,4194304,112.105372611,2097152,2909,true,1,0.0374139428139 +bf16,4,64,16,256,4096,control_shuffle32_transform,1,2,100,21.9026279449,4194304,191.497751345,2097152,2909,true,1,0.0219026279449 +bf16,4,64,16,256,4096,shuffle32_transform,1,3,100,23.5085725784,4194304,178.415936825,2097152,2909,true,1,0.0235085725784 +bf16,4,64,16,256,4096,baseline_split,1,4,100,1143.74465942,6832128,5.97347313818,2097152,2909,true,1,1.14374465942 +bf16,4,64,16,256,4096,control_optimized_split,1,5,100,498.094863892,6832128,13.7165196738,2097152,2909,true,1,0.498094863892 +bf16,4,64,16,256,4096,optimized_split,1,6,100,188.90789032,6832128,36.1664512183,2097152,2909,true,1,0.18890789032 +bf16,4,64,16,256,4096,control_shuffle32_split,1,7,100,838.395843506,6832128,8.14904803372,2097152,2909,true,1,0.838395843506 +bf16,4,64,16,256,4096,shuffle32_split,1,8,100,149.851198196,6832128,45.5927485548,2097152,2909,true,1,0.149851198196 +bf16,4,64,16,256,4096,baseline_fused,1,9,100,1138.92074585,2637824,2.31607336122,2097152,2909,true,1,1.13892074585 +bf16,4,64,16,256,4096,control_optimized_fused,1,10,100,485.821266174,2637824,5.42961822312,2097152,2909,true,1,0.485821266174 +bf16,4,64,16,256,4096,optimized_fused,1,11,100,171.618518829,2637824,15.3702759935,2097152,2909,true,1,0.171618518829 +bf16,4,64,16,256,4096,control_shuffle32_fused,1,12,100,836.191329956,2637824,3.15456989986,2097152,2909,true,1,0.836191329956 +bf16,4,64,16,256,4096,shuffle32_fused,1,13,100,130.926170349,2637824,20.1474158525,2097152,2909,true,1,0.130926170349 +bf16,4,64,16,256,4096,baseline_transform,1,14,100,49.2676591873,4194304,85.1330075182,2097152,2909,true,1,0.0492676591873 +bf16,4,64,16,256,4096,optimized_transform,2,0,100,37.408914566,4194304,112.120441041,2097152,2909,true,1,0.037408914566 +bf16,4,64,16,256,4096,control_shuffle32_transform,2,1,100,23.5035443306,4194304,178.454106368,2097152,2909,true,1,0.0235035443306 +bf16,4,64,16,256,4096,shuffle32_transform,2,2,100,22.7748584747,4194304,184.163778873,2097152,2909,true,1,0.0227748584747 +bf16,4,64,16,256,4096,baseline_split,2,3,100,1145.43609619,6832128,5.96465226015,2097152,2909,true,1,1.14543609619 +bf16,4,64,16,256,4096,control_optimized_split,2,4,100,500.980796814,6832128,13.6375047576,2097152,2909,true,1,0.500980796814 +bf16,4,64,16,256,4096,optimized_split,2,5,100,188.722515106,6832128,36.2019761985,2097152,2909,true,1,0.188722515106 +bf16,4,64,16,256,4096,control_shuffle32_split,2,6,100,835.800018311,6832128,8.17435732271,2097152,2909,true,1,0.835800018311 +bf16,4,64,16,256,4096,shuffle32_split,2,7,100,149.506969452,6832128,45.6977224878,2097152,2909,true,1,0.149506969452 +bf16,4,64,16,256,4096,baseline_fused,2,8,100,1136.26495361,2637824,2.32148671981,2097152,2909,true,1,1.13626495361 +bf16,4,64,16,256,4096,control_optimized_fused,2,9,100,485.288009644,2637824,5.43558453451,2097152,2909,true,1,0.485288009644 +bf16,4,64,16,256,4096,optimized_fused,2,10,100,172.018051147,2637824,15.3345767052,2097152,2909,true,1,0.172018051147 +bf16,4,64,16,256,4096,control_shuffle32_fused,2,11,100,831.909942627,2637824,3.17080475282,2097152,2909,true,1,0.831909942627 +bf16,4,64,16,256,4096,shuffle32_fused,2,12,100,131.205253601,2637824,20.1045608129,2097152,2909,true,1,0.131205253601 +bf16,4,64,16,256,4096,baseline_transform,2,13,100,49.2784023285,4194304,85.1144477461,2097152,2909,true,1,0.0492784023285 +bf16,4,64,16,256,4096,control_optimized_transform,2,14,100,37.3314285278,4194304,112.353161007,2097152,2909,true,1,0.0373314285278 +bf16,4,64,16,256,4096,control_shuffle32_transform,3,0,100,22.5455999374,4194304,186.0364777,2097152,2909,true,1,0.0225455999374 +bf16,4,64,16,256,4096,shuffle32_transform,3,1,100,23.5417127609,4194304,178.16477682,2097152,2909,true,1,0.0235417127609 +bf16,4,64,16,256,4096,baseline_split,3,2,100,1143.47473145,6832128,5.97488323276,2097152,2909,true,1,1.14347473145 +bf16,4,64,16,256,4096,control_optimized_split,3,3,100,502.082061768,6832128,13.6075923046,2097152,2909,true,1,0.502082061768 +bf16,4,64,16,256,4096,optimized_split,3,4,100,188.897609711,6832128,36.16841955,2097152,2909,true,1,0.188897609711 +bf16,4,64,16,256,4096,control_shuffle32_split,3,5,100,836.438140869,6832128,8.16812106739,2097152,2909,true,1,0.836438140869 +bf16,4,64,16,256,4096,shuffle32_split,3,6,100,149.562740326,6832128,45.6806821345,2097152,2909,true,1,0.149562740326 +bf16,4,64,16,256,4096,baseline_fused,3,7,100,1136.04202271,2637824,2.32194227615,2097152,2909,true,1,1.13604202271 +bf16,4,64,16,256,4096,control_optimized_fused,3,8,100,486.244544983,2637824,5.42489170772,2097152,2909,true,1,0.486244544983 +bf16,4,64,16,256,4096,optimized_fused,3,9,100,171.134395599,2637824,15.4137570695,2097152,2909,true,1,0.171134395599 +bf16,4,64,16,256,4096,control_shuffle32_fused,3,10,100,838.488693237,2637824,3.14592673852,2097152,2909,true,1,0.838488693237 +bf16,4,64,16,256,4096,shuffle32_fused,3,11,100,131.062860489,2637824,20.1264033927,2097152,2909,true,1,0.131062860489 +bf16,4,64,16,256,4096,baseline_transform,3,12,100,49.2804574966,4194304,85.1108981747,2097152,2909,true,1,0.0492804574966 +bf16,4,64,16,256,4096,control_optimized_transform,3,13,100,37.3794269562,4194304,112.208889797,2097152,2909,true,1,0.0373794269562 +bf16,4,64,16,256,4096,optimized_transform,3,14,100,37.3963427544,4194304,112.158133418,2097152,2909,true,1,0.0373963427544 +bf16,4,64,16,256,4096,shuffle32_transform,4,0,100,22.247543335,4194304,188.528860776,2097152,2909,true,1,0.022247543335 +bf16,4,64,16,256,4096,baseline_split,4,1,100,1143.80302429,6832128,5.9731683296,2097152,2909,true,1,1.14380302429 +bf16,4,64,16,256,4096,control_optimized_split,4,2,100,500.656433105,6832128,13.6463401811,2097152,2909,true,1,0.500656433105 +bf16,4,64,16,256,4096,optimized_split,4,3,100,188.991317749,6832128,36.1504860719,2097152,2909,true,1,0.188991317749 +bf16,4,64,16,256,4096,control_shuffle32_split,4,4,100,841.160736084,6832128,8.12226213958,2097152,2909,true,1,0.841160736084 +bf16,4,64,16,256,4096,shuffle32_split,4,5,100,149.150629044,6832128,45.8069003383,2097152,2909,true,1,0.149150629044 +bf16,4,64,16,256,4096,baseline_fused,4,6,100,1136.56959534,2637824,2.32086447748,2097152,2909,true,1,1.13656959534 +bf16,4,64,16,256,4096,control_optimized_fused,4,7,100,486.344680786,2637824,5.42377475114,2097152,2909,true,1,0.486344680786 +bf16,4,64,16,256,4096,optimized_fused,4,8,100,171.624450684,2637824,15.3697447508,2097152,2909,true,1,0.171624450684 +bf16,4,64,16,256,4096,control_shuffle32_fused,4,9,100,830.93170166,2637824,3.1745376843,2097152,2909,true,1,0.83093170166 +bf16,4,64,16,256,4096,shuffle32_fused,4,10,100,131.476564407,2637824,20.0630736884,2097152,2909,true,1,0.131476564407 +bf16,4,64,16,256,4096,baseline_transform,4,11,100,49.2555427551,4194304,85.1539494926,2097152,2909,true,1,0.0492555427551 +bf16,4,64,16,256,4096,control_optimized_transform,4,12,100,37.3584008217,4194304,112.272043443,2097152,2909,true,1,0.0373584008217 +bf16,4,64,16,256,4096,optimized_transform,4,13,100,37.4031996727,4194304,112.137572098,2097152,2909,true,1,0.0374031996727 +bf16,4,64,16,256,4096,control_shuffle32_transform,4,14,100,22.4870872498,4194304,186.52055526,2097152,2909,true,1,0.0224870872498 +bf16,16,64,16,256,16384,baseline_transform,0,0,100,170.586051941,16777216,98.3504560256,8388608,2909,true,1,0.170586051941 +bf16,16,64,16,256,16384,control_optimized_transform,0,1,100,120.669717789,16777216,139.034186103,8388608,2909,true,1,0.120669717789 +bf16,16,64,16,256,16384,optimized_transform,0,2,100,120.715427399,16777216,138.981539987,8388608,2909,true,1,0.120715427399 +bf16,16,64,16,256,16384,control_shuffle32_transform,0,3,100,43.1316566467,16777216,388.976851444,8388608,2909,true,1,0.0431316566467 +bf16,16,64,16,256,16384,shuffle32_transform,0,4,100,43.2210254669,16777216,388.172557656,8388608,2909,true,1,0.0432210254669 +bf16,16,64,16,256,16384,baseline_split,0,5,100,4271.57745361,27328512,6.39775640188,8388608,2909,true,1,4.27157745361 +bf16,16,64,16,256,16384,control_optimized_split,0,6,100,1869.96688843,27328512,14.6144363139,8388608,2909,true,1,1.86996688843 +bf16,16,64,16,256,16384,optimized_split,0,7,100,677.89276123,27328512,40.313916246,8388608,2909,true,1,0.67789276123 +bf16,16,64,16,256,16384,control_shuffle32_split,0,8,100,3194.63317871,27328512,8.55450703452,8388608,2909,true,1,3.19463317871 +bf16,16,64,16,256,16384,shuffle32_split,0,9,100,436.181716919,27328512,62.6539603563,8388608,2909,true,1,0.436181716919 +bf16,16,64,16,256,16384,baseline_fused,0,10,100,4271.00463867,10551296,2.47044826514,8388608,2909,true,1,4.27100463867 +bf16,16,64,16,256,16384,control_optimized_fused,0,11,100,1823.57376099,10551296,5.78605386069,8388608,2909,true,1,1.82357376099 +bf16,16,64,16,256,16384,optimized_fused,0,12,100,631.560020447,10551296,16.7067193274,8388608,2909,true,1,0.631560020447 +bf16,16,64,16,256,16384,control_shuffle32_fused,0,13,100,3042.06512451,10551296,3.46846486454,8388608,2909,true,1,3.04206512451 +bf16,16,64,16,256,16384,shuffle32_fused,0,14,100,391.467437744,10551296,26.9531894167,8388608,2909,true,1,0.391467437744 +bf16,16,64,16,256,16384,control_optimized_transform,1,0,100,120.656452179,16777216,139.049472258,8388608,2909,true,1,0.120656452179 +bf16,16,64,16,256,16384,optimized_transform,1,1,100,120.705833435,16777216,138.992586543,8388608,2909,true,1,0.120705833435 +bf16,16,64,16,256,16384,control_shuffle32_transform,1,2,100,43.074054718,16777216,389.497021115,8388608,2909,true,1,0.043074054718 +bf16,16,64,16,256,16384,shuffle32_transform,1,3,100,43.2075405121,16777216,388.293705246,8388608,2909,true,1,0.0432075405121 +bf16,16,64,16,256,16384,baseline_split,1,4,100,4270.74249268,27328512,6.39900720937,8388608,2909,true,1,4.27074249268 +bf16,16,64,16,256,16384,control_optimized_split,1,5,100,1870.0617981,27328512,14.6136945997,8388608,2909,true,1,1.8700617981 +bf16,16,64,16,256,16384,optimized_split,1,6,100,677.907409668,27328512,40.3130451301,8388608,2909,true,1,0.677907409668 +bf16,16,64,16,256,16384,control_shuffle32_split,1,7,100,3193.41308594,27328512,8.55777541601,8388608,2909,true,1,3.19341308594 +bf16,16,64,16,256,16384,shuffle32_split,1,8,100,436.150856018,27328512,62.6583935877,8388608,2909,true,1,0.436150856018 +bf16,16,64,16,256,16384,baseline_fused,1,9,100,4274.03991699,10551296,2.46869383649,8388608,2909,true,1,4.27403991699 +bf16,16,64,16,256,16384,control_optimized_fused,1,10,100,1823.59680176,10551296,5.78598075508,8388608,2909,true,1,1.82359680176 +bf16,16,64,16,256,16384,optimized_fused,1,11,100,631.534385681,10551296,16.7073974739,8388608,2909,true,1,0.631534385681 +bf16,16,64,16,256,16384,control_shuffle32_fused,1,12,100,3042.20245361,10551296,3.46830829338,8388608,2909,true,1,3.04220245361 +bf16,16,64,16,256,16384,shuffle32_fused,1,13,100,390.743103027,10551296,27.0031535253,8388608,2909,true,1,0.390743103027 +bf16,16,64,16,256,16384,baseline_transform,1,14,100,170.59337616,16777216,98.3462334686,8388608,2909,true,1,0.17059337616 +bf16,16,64,16,256,16384,optimized_transform,2,0,100,120.739202499,16777216,138.954172735,8388608,2909,true,1,0.120739202499 +bf16,16,64,16,256,16384,control_shuffle32_transform,2,1,100,43.1279993057,16777216,389.009837462,8388608,2909,true,1,0.0431279993057 +bf16,16,64,16,256,16384,shuffle32_transform,2,2,100,43.2532548904,16777216,387.88331751,8388608,2909,true,1,0.0432532548904 +bf16,16,64,16,256,16384,baseline_split,2,3,100,4271.18286133,27328512,6.3983474572,8388608,2909,true,1,4.27118286133 +bf16,16,64,16,256,16384,control_optimized_split,2,4,100,1870.20401001,27328512,14.6125833619,8388608,2909,true,1,1.87020401001 +bf16,16,64,16,256,16384,optimized_split,2,5,100,677.864227295,27328512,40.3156132151,8388608,2909,true,1,0.677864227295 +bf16,16,64,16,256,16384,control_shuffle32_split,2,6,100,3177.91595459,27328512,8.59950747298,8388608,2909,true,1,3.17791595459 +bf16,16,64,16,256,16384,shuffle32_split,2,7,100,436.267433167,27328512,62.6416503328,8388608,2909,true,1,0.436267433167 +bf16,16,64,16,256,16384,baseline_fused,2,8,100,4272.66418457,10551296,2.46948871809,8388608,2909,true,1,4.27266418457 +bf16,16,64,16,256,16384,control_optimized_fused,2,9,100,1823.71109009,10551296,5.78561815923,8388608,2909,true,1,1.82371109009 +bf16,16,64,16,256,16384,optimized_fused,2,10,100,631.759300232,10551296,16.7014494225,8388608,2909,true,1,0.631759300232 +bf16,16,64,16,256,16384,control_shuffle32_fused,2,11,100,3041.7868042,10551296,3.46878222545,8388608,2909,true,1,3.0417868042 +bf16,16,64,16,256,16384,shuffle32_fused,2,12,100,390.812339783,10551296,26.9983696161,8388608,2909,true,1,0.390812339783 +bf16,16,64,16,256,16384,baseline_transform,2,13,100,170.602054596,16777216,98.3412306477,8388608,2909,true,1,0.170602054596 +bf16,16,64,16,256,16384,control_optimized_transform,2,14,100,120.666284561,16777216,139.038141938,8388608,2909,true,1,0.120666284561 +bf16,16,64,16,256,16384,control_shuffle32_transform,3,0,100,43.0411434174,16777216,389.79484902,8388608,2909,true,1,0.0430411434174 +bf16,16,64,16,256,16384,shuffle32_transform,3,1,100,43.1604576111,16777216,388.717287272,8388608,2909,true,1,0.0431604576111 +bf16,16,64,16,256,16384,baseline_split,3,2,100,4267.68096924,27328512,6.40359769087,8388608,2909,true,1,4.26768096924 +bf16,16,64,16,256,16384,control_optimized_split,3,3,100,1869.95773315,27328512,14.6145078659,8388608,2909,true,1,1.86995773315 +bf16,16,64,16,256,16384,optimized_split,3,4,100,677.932357788,27328512,40.3115615976,8388608,2909,true,1,0.677932357788 +bf16,16,64,16,256,16384,control_shuffle32_split,3,5,100,3183.23791504,27328512,8.58513021314,8388608,2909,true,1,3.18323791504 +bf16,16,64,16,256,16384,shuffle32_split,3,6,100,436.178512573,27328512,62.654420638,8388608,2909,true,1,0.436178512573 +bf16,16,64,16,256,16384,baseline_fused,3,7,100,4273.49853516,10551296,2.46900657932,8388608,2909,true,1,4.27349853516 +bf16,16,64,16,256,16384,control_optimized_fused,3,8,100,1823.73077393,10551296,5.78555571406,8388608,2909,true,1,1.82373077393 +bf16,16,64,16,256,16384,optimized_fused,3,9,100,631.684799194,10551296,16.7034191949,8388608,2909,true,1,0.631684799194 +bf16,16,64,16,256,16384,control_shuffle32_fused,3,10,100,3041.39465332,10551296,3.46922948276,8388608,2909,true,1,3.04139465332 +bf16,16,64,16,256,16384,shuffle32_fused,3,11,100,390.950393677,10551296,26.9888358489,8388608,2909,true,1,0.390950393677 +bf16,16,64,16,256,16384,baseline_transform,3,12,100,170.626983643,16777216,98.3268627379,8388608,2909,true,1,0.170626983643 +bf16,16,64,16,256,16384,control_optimized_transform,3,13,100,120.652799606,16777216,139.053681761,8388608,2909,true,1,0.120652799606 +bf16,16,64,16,256,16384,optimized_transform,3,14,100,120.739421844,16777216,138.9539203,8388608,2909,true,1,0.120739421844 +bf16,16,64,16,256,16384,shuffle32_transform,4,0,100,43.1442308426,16777216,388.863485855,8388608,2909,true,1,0.0431442308426 +bf16,16,64,16,256,16384,baseline_split,4,1,100,4270.3994751,27328512,6.39952120624,8388608,2909,true,1,4.2703994751 +bf16,16,64,16,256,16384,control_optimized_split,4,2,100,1869.96032715,27328512,14.6144875927,8388608,2909,true,1,1.86996032715 +bf16,16,64,16,256,16384,optimized_split,4,3,100,677.828979492,27328512,40.3177096684,8388608,2909,true,1,0.677828979492 +bf16,16,64,16,256,16384,control_shuffle32_split,4,4,100,3198.7008667,27328512,8.54362853511,8388608,2909,true,1,3.1987008667 +bf16,16,64,16,256,16384,shuffle32_split,4,5,100,436.265144348,27328512,62.6419789755,8388608,2909,true,1,0.436265144348 +bf16,16,64,16,256,16384,baseline_fused,4,6,100,4273.56536865,10551296,2.46896796698,8388608,2909,true,1,4.27356536865 +bf16,16,64,16,256,16384,control_optimized_fused,4,7,100,1823.49029541,10551296,5.78631870241,8388608,2909,true,1,1.82349029541 +bf16,16,64,16,256,16384,optimized_fused,4,8,100,631.736221313,10551296,16.7020595686,8388608,2909,true,1,0.631736221313 +bf16,16,64,16,256,16384,control_shuffle32_fused,4,9,100,3041.95220947,10551296,3.46859361141,8388608,2909,true,1,3.04195220947 +bf16,16,64,16,256,16384,shuffle32_fused,4,10,100,391.255989075,10551296,26.9677558801,8388608,2909,true,1,0.391255989075 +bf16,16,64,16,256,16384,baseline_transform,4,11,100,170.586280823,16777216,98.3503240652,8388608,2909,true,1,0.170586280823 +bf16,16,64,16,256,16384,control_optimized_transform,4,12,100,120.645027161,16777216,139.062640167,8388608,2909,true,1,0.120645027161 +bf16,16,64,16,256,16384,optimized_transform,4,13,100,120.749263763,16777216,138.942594572,8388608,2909,true,1,0.120749263763 +bf16,16,64,16,256,16384,control_shuffle32_transform,4,14,100,43.0390834808,16777216,389.81350538,8388608,2909,true,1,0.0430390834808 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round1.log b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round1.log new file mode 100644 index 00000000..22c2e112 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round1.log @@ -0,0 +1,32 @@ +DEVICE name=MTT S4000 warp=128 runtime=50100 driver=50100 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=256 +PASS requested validation/benchmark operations completed diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round2.csv b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round2.csv new file mode 100644 index 00000000..c4b11fb3 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round2.csv @@ -0,0 +1,2251 @@ +dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale,kernel_ms +fp16,1,1,1,64,1,baseline_transform,0,0,100,8.97965729237,256,0.0285088830971,128,2909,true,1,0.00897965729237 +fp16,1,1,1,64,1,control_optimized_transform,0,1,100,9.36365664005,256,0.0273397466226,128,2909,true,1,0.00936365664005 +fp16,1,1,1,64,1,optimized_transform,0,2,100,9.20000016689,256,0.0278260864517,128,2909,true,1,0.00920000016689 +fp16,1,1,1,64,1,control_shuffle32_transform,0,3,100,7.8560000658,256,0.0325865577719,128,2909,true,1,0.0078560000658 +fp16,1,1,1,64,1,shuffle32_transform,0,4,100,7.83794283867,256,0.0326616314088,128,2909,true,1,0.00783794283867 +fp16,1,1,1,64,1,baseline_split,0,5,100,69.8665142059,420,0.0060114634997,128,2909,true,1,0.0698665142059 +fp16,1,1,1,64,1,control_optimized_split,0,6,100,63.9938306808,420,0.00656313265719,128,2909,true,1,0.0639938306808 +fp16,1,1,1,64,1,optimized_split,0,7,100,33.6784005165,420,0.0124709010392,128,2909,true,1,0.0336784005165 +fp16,1,1,1,64,1,control_shuffle32_split,0,8,100,94.518404007,420,0.00444357905122,128,2909,true,1,0.094518404007 +fp16,1,1,1,64,1,shuffle32_split,0,9,100,34.7908568382,420,0.0120721372846,128,2909,true,1,0.0347908568382 +fp16,1,1,1,64,1,baseline_fused,0,10,100,65.4260587692,164,0.00250664648131,128,2909,true,1,0.0654260587692 +fp16,1,1,1,64,1,control_optimized_fused,0,11,100,58.6470890045,164,0.00279638772842,128,2909,true,1,0.0586470890045 +fp16,1,1,1,64,1,optimized_fused,0,12,100,28.5202288628,164,0.00575030448701,128,2909,true,1,0.0285202288628 +fp16,1,1,1,64,1,control_shuffle32_fused,0,13,100,89.2297172546,164,0.00183795270282,128,2909,true,1,0.0892297172546 +fp16,1,1,1,64,1,shuffle32_fused,0,14,100,29.703772068,164,0.00552118430024,128,2909,true,1,0.029703772068 +fp16,1,1,1,64,1,control_optimized_transform,1,0,100,9.30902838707,256,0.0275001846976,128,2909,true,1,0.00930902838707 +fp16,1,1,1,64,1,optimized_transform,1,1,100,9.27062869072,256,0.0276140926943,128,2909,true,1,0.00927062869072 +fp16,1,1,1,64,1,control_shuffle32_transform,1,2,100,7.9439997673,256,0.0322255799973,128,2909,true,1,0.0079439997673 +fp16,1,1,1,64,1,shuffle32_transform,1,3,100,7.93439984322,256,0.0322645701072,128,2909,true,1,0.00793439984322 +fp16,1,1,1,64,1,baseline_split,1,4,100,69.8861694336,420,0.00600977279774,128,2909,true,1,0.0698861694336 +fp16,1,1,1,64,1,control_optimized_split,1,5,100,64.056687355,420,0.00655669247571,128,2909,true,1,0.064056687355 +fp16,1,1,1,64,1,optimized_split,1,6,100,33.642513752,420,0.0124842038587,128,2909,true,1,0.033642513752 +fp16,1,1,1,64,1,control_shuffle32_split,1,7,100,93.8781738281,420,0.00447388336259,128,2909,true,1,0.0938781738281 +fp16,1,1,1,64,1,shuffle32_split,1,8,100,35.2859425545,420,0.0119027570073,128,2909,true,1,0.0352859425545 +fp16,1,1,1,64,1,baseline_fused,1,9,100,65.4400014877,164,0.00250611241246,128,2909,true,1,0.0654400014877 +fp16,1,1,1,64,1,control_optimized_fused,1,10,100,58.639087677,164,0.00279676929667,128,2909,true,1,0.058639087677 +fp16,1,1,1,64,1,optimized_fused,1,11,100,28.5152006149,164,0.00575131847097,128,2909,true,1,0.0285152006149 +fp16,1,1,1,64,1,control_shuffle32_fused,1,12,100,89.2290306091,164,0.00183796684644,128,2909,true,1,0.0892290306091 +fp16,1,1,1,64,1,shuffle32_fused,1,13,100,29.694173336,164,0.00552296903989,128,2909,true,1,0.029694173336 +fp16,1,1,1,64,1,baseline_transform,1,14,100,8.8948571682,256,0.0287806757499,128,2909,true,1,0.0088948571682 +fp16,1,1,1,64,1,optimized_transform,2,0,100,9.33371424675,256,0.0274274520552,128,2909,true,1,0.00933371424675 +fp16,1,1,1,64,1,control_shuffle32_transform,2,1,100,7.90582835674,256,0.0323811735404,128,2909,true,1,0.00790582835674 +fp16,1,1,1,64,1,shuffle32_transform,2,2,100,7.86285758018,256,0.032558137724,128,2909,true,1,0.00786285758018 +fp16,1,1,1,64,1,baseline_split,2,3,100,69.8756551743,420,0.00601067709423,128,2909,true,1,0.0698756551743 +fp16,1,1,1,64,1,control_optimized_split,2,4,100,64.1417121887,420,0.00654800106932,128,2909,true,1,0.0641417121887 +fp16,1,1,1,64,1,optimized_split,2,5,100,33.6763429642,420,0.0124716629845,128,2909,true,1,0.0336763429642 +fp16,1,1,1,64,1,control_shuffle32_split,2,6,100,93.8806915283,420,0.00447376338161,128,2909,true,1,0.0938806915283 +fp16,1,1,1,64,1,shuffle32_split,2,7,100,34.8464012146,420,0.0120528945705,128,2909,true,1,0.0348464012146 +fp16,1,1,1,64,1,baseline_fused,2,8,100,65.4546260834,164,0.00250555246914,128,2909,true,1,0.0654546260834 +fp16,1,1,1,64,1,control_optimized_fused,2,9,100,58.6541700363,164,0.00279605013418,128,2909,true,1,0.0586541700363 +fp16,1,1,1,64,1,optimized_fused,2,10,100,28.5046839714,164,0.00575344038771,128,2909,true,1,0.0285046839714 +fp16,1,1,1,64,1,control_shuffle32_fused,2,11,100,89.2258262634,164,0.00183803285291,128,2909,true,1,0.0892258262634 +fp16,1,1,1,64,1,shuffle32_fused,2,12,100,29.7024011612,164,0.00552143912911,128,2909,true,1,0.0297024011612 +fp16,1,1,1,64,1,baseline_transform,2,13,100,9.13462817669,256,0.0280252239115,128,2909,true,1,0.00913462817669 +fp16,1,1,1,64,1,control_optimized_transform,2,14,100,9.18354272842,256,0.0278759524043,128,2909,true,1,0.00918354272842 +fp16,1,1,1,64,1,control_shuffle32_transform,3,0,100,7.81485676765,256,0.0327581179811,128,2909,true,1,0.00781485676765 +fp16,1,1,1,64,1,shuffle32_transform,3,1,100,7.85805702209,256,0.0325780277848,128,2909,true,1,0.00785805702209 +fp16,1,1,1,64,1,baseline_split,3,2,100,69.8660612106,420,0.00601150247663,128,2909,true,1,0.0698660612106 +fp16,1,1,1,64,1,control_optimized_split,3,3,100,64.0475416183,420,0.00655762874558,128,2909,true,1,0.0640475416183 +fp16,1,1,1,64,1,optimized_split,3,4,100,33.6614847183,420,0.0124771680012,128,2909,true,1,0.0336614847183 +fp16,1,1,1,64,1,control_shuffle32_split,3,5,100,93.8912010193,420,0.00447326262142,128,2909,true,1,0.0938912010193 +fp16,1,1,1,64,1,shuffle32_split,3,6,100,34.8034286499,420,0.0120677765465,128,2909,true,1,0.0348034286499 +fp16,1,1,1,64,1,baseline_fused,3,7,100,65.4365730286,164,0.00250624371677,128,2909,true,1,0.0654365730286 +fp16,1,1,1,64,1,control_optimized_fused,3,8,100,58.6685705185,164,0.00279536383025,128,2909,true,1,0.0586685705185 +fp16,1,1,1,64,1,optimized_fused,3,9,100,28.5053730011,164,0.00575330131599,128,2909,true,1,0.0285053730011 +fp16,1,1,1,64,1,control_shuffle32_fused,3,10,100,89.2303943634,164,0.00183793875585,128,2909,true,1,0.0892303943634 +fp16,1,1,1,64,1,shuffle32_fused,3,11,100,29.7149705887,164,0.0055191035613,128,2909,true,1,0.0297149705887 +fp16,1,1,1,64,1,baseline_transform,3,12,100,9.00685727596,256,0.0284227885661,128,2909,true,1,0.00900685727596 +fp16,1,1,1,64,1,control_optimized_transform,3,13,100,9.16114330292,256,0.0279441104167,128,2909,true,1,0.00916114330292 +fp16,1,1,1,64,1,optimized_transform,3,14,100,9.27268564701,256,0.0276079670707,128,2909,true,1,0.00927268564701 +fp16,1,1,1,64,1,shuffle32_transform,4,0,100,7.86925733089,256,0.0325316594992,128,2909,true,1,0.00786925733089 +fp16,1,1,1,64,1,baseline_split,4,1,100,69.8623991013,420,0.00601181759291,128,2909,true,1,0.0698623991013 +fp16,1,1,1,64,1,control_optimized_split,4,2,100,64.0146303177,420,0.0065610001638,128,2909,true,1,0.0640146303177 +fp16,1,1,1,64,1,optimized_split,4,3,100,34.6164560318,420,0.0121329577937,128,2909,true,1,0.0346164560318 +fp16,1,1,1,64,1,control_shuffle32_split,4,4,100,93.8863945007,420,0.00447349163032,128,2909,true,1,0.0938863945007 +fp16,1,1,1,64,1,shuffle32_split,4,5,100,34.8219442368,420,0.0120613598467,128,2909,true,1,0.0348219442368 +fp16,1,1,1,64,1,baseline_fused,4,6,100,65.4052591324,164,0.00250744362419,128,2909,true,1,0.0654052591324 +fp16,1,1,1,64,1,control_optimized_fused,4,7,100,58.6064004898,164,0.00279832916933,128,2909,true,1,0.0586064004898 +fp16,1,1,1,64,1,optimized_fused,4,8,100,28.5193133354,164,0.00575048908335,128,2909,true,1,0.0285193133354 +fp16,1,1,1,64,1,control_shuffle32_fused,4,9,100,89.2322349548,164,0.00183790084472,128,2909,true,1,0.0892322349548 +fp16,1,1,1,64,1,shuffle32_fused,4,10,100,29.6815991402,164,0.00552530876876,128,2909,true,1,0.0296815991402 +fp16,1,1,1,64,1,baseline_transform,4,11,100,9.00548577309,256,0.0284271172539,128,2909,true,1,0.00900548577309 +fp16,1,1,1,64,1,control_optimized_transform,4,12,100,9.12000000477,256,0.0280701754239,128,2909,true,1,0.00912000000477 +fp16,1,1,1,64,1,optimized_transform,4,13,100,9.25165712833,256,0.0276707184939,128,2909,true,1,0.00925165712833 +fp16,1,1,1,64,1,control_shuffle32_transform,4,14,100,7.87154316902,256,0.032522212545,128,2909,true,1,0.00787154316902 +fp16,1,17,1,64,17,baseline_transform,0,0,100,9.16022837162,4352,0.475097325464,2176,2909,true,1,0.00916022837162 +fp16,1,17,1,64,17,control_optimized_transform,0,1,100,9.31519985199,4352,0.467193411752,2176,2909,true,1,0.00931519985199 +fp16,1,17,1,64,17,optimized_transform,0,2,100,9.41394329071,4352,0.462292990897,2176,2909,true,1,0.00941394329071 +fp16,1,17,1,64,17,control_shuffle32_transform,0,3,100,7.95199990273,4352,0.547283708908,2176,2909,true,1,0.00795199990273 +fp16,1,17,1,64,17,shuffle32_transform,0,4,100,7.92320013046,4352,0.549273011957,2176,2909,true,1,0.00792320013046 +fp16,1,17,1,64,17,baseline_split,0,5,100,70.2941703796,7140,0.10157314556,2176,2909,true,1,0.0702941703796 +fp16,1,17,1,64,17,control_optimized_split,0,6,100,64.4694852829,7140,0.110750069877,2176,2909,true,1,0.0644694852829 +fp16,1,17,1,64,17,optimized_split,0,7,100,34.9430847168,7140,0.204332275123,2176,2909,true,1,0.0349430847168 +fp16,1,17,1,64,17,control_shuffle32_split,0,8,100,93.7440013885,7140,0.0761648734238,2176,2909,true,1,0.0937440013885 +fp16,1,17,1,64,17,shuffle32_split,0,9,100,34.8118853569,7140,0.205102364517,2176,2909,true,1,0.0348118853569 +fp16,1,17,1,64,17,baseline_fused,0,10,100,65.6667423248,2788,0.0424568038751,2176,2909,true,1,0.0656667423248 +fp16,1,17,1,64,17,control_optimized_fused,0,11,100,58.9005708694,2788,0.0473340064255,2176,2909,true,1,0.0589005708694 +fp16,1,17,1,64,17,optimized_fused,0,12,100,28.99954319,2788,0.0961394454296,2176,2909,true,1,0.02899954319 +fp16,1,17,1,64,17,control_shuffle32_fused,0,13,100,89.0758800507,2788,0.0312991575095,2176,2909,true,1,0.0890758800507 +fp16,1,17,1,64,17,shuffle32_fused,0,14,100,29.6169137955,2788,0.0941353990917,2176,2909,true,1,0.0296169137955 +fp16,1,17,1,64,17,control_optimized_transform,1,0,100,9.30377185345,4352,0.467767274236,2176,2909,true,1,0.00930377185345 +fp16,1,17,1,64,17,optimized_transform,1,1,100,9.54651474953,4352,0.455873176147,2176,2909,true,1,0.00954651474953 +fp16,1,17,1,64,17,control_shuffle32_transform,1,2,100,7.88160026073,4352,0.552172129521,2176,2909,true,1,0.00788160026073 +fp16,1,17,1,64,17,shuffle32_transform,1,3,100,7.90788531303,4352,0.550336762323,2176,2909,true,1,0.00790788531303 +fp16,1,17,1,64,17,baseline_split,1,4,100,70.2857112885,7140,0.101585370186,2176,2909,true,1,0.0702857112885 +fp16,1,17,1,64,17,control_optimized_split,1,5,100,64.4505167007,7140,0.110782664989,2176,2909,true,1,0.0644505167007 +fp16,1,17,1,64,17,optimized_split,1,6,100,34.2852568626,7140,0.208252778406,2176,2909,true,1,0.0342852568626 +fp16,1,17,1,64,17,control_shuffle32_split,1,7,100,94.3835449219,7140,0.0756487797307,2176,2909,true,1,0.0943835449219 +fp16,1,17,1,64,17,shuffle32_split,1,8,100,34.760684967,7140,0.205404467915,2176,2909,true,1,0.034760684967 +fp16,1,17,1,64,17,baseline_fused,1,9,100,65.7094860077,2788,0.0424291859424,2176,2909,true,1,0.0657094860077 +fp16,1,17,1,64,17,control_optimized_fused,1,10,100,58.9051437378,2788,0.0473303318367,2176,2909,true,1,0.0589051437378 +fp16,1,17,1,64,17,optimized_fused,1,11,100,28.983771801,2788,0.0961917592763,2176,2909,true,1,0.028983771801 +fp16,1,17,1,64,17,control_shuffle32_fused,1,12,100,89.0301704407,2788,0.0313152270315,2176,2909,true,1,0.0890301704407 +fp16,1,17,1,64,17,shuffle32_fused,1,13,100,29.5849132538,2788,0.0942372207106,2176,2909,true,1,0.0295849132538 +fp16,1,17,1,64,17,baseline_transform,1,14,100,9.20205712318,4352,0.472937729221,2176,2909,true,1,0.00920205712318 +fp16,1,17,1,64,17,optimized_transform,2,0,100,9.4843429327,4352,0.458861518492,2176,2909,true,1,0.0094843429327 +fp16,1,17,1,64,17,control_shuffle32_transform,2,1,100,7.92548596859,4352,0.549114592752,2176,2909,true,1,0.00792548596859 +fp16,1,17,1,64,17,shuffle32_transform,2,2,100,7.96411395073,4352,0.546451247047,2176,2909,true,1,0.00796411395073 +fp16,1,17,1,64,17,baseline_split,2,3,100,70.3988552094,7140,0.101422103794,2176,2909,true,1,0.0703988552094 +fp16,1,17,1,64,17,control_optimized_split,2,4,100,64.5604610443,7140,0.110594005751,2176,2909,true,1,0.0645604610443 +fp16,1,17,1,64,17,optimized_split,2,5,100,34.2822861671,7140,0.208270824332,2176,2909,true,1,0.0342822861671 +fp16,1,17,1,64,17,control_shuffle32_split,2,6,100,93.7476539612,7140,0.0761619059071,2176,2909,true,1,0.0937476539612 +fp16,1,17,1,64,17,shuffle32_split,2,7,100,34.8301720619,7140,0.204994680684,2176,2909,true,1,0.0348301720619 +fp16,1,17,1,64,17,baseline_fused,2,8,100,65.7280015945,2788,0.042417233635,2176,2909,true,1,0.0657280015945 +fp16,1,17,1,64,17,control_optimized_fused,2,9,100,58.906288147,2788,0.0473294123209,2176,2909,true,1,0.058906288147 +fp16,1,17,1,64,17,optimized_fused,2,10,100,28.9913153648,2788,0.0961667301023,2176,2909,true,1,0.0289913153648 +fp16,1,17,1,64,17,control_shuffle32_fused,2,11,100,89.0701770782,2788,0.0313011615274,2176,2909,true,1,0.0890701770782 +fp16,1,17,1,64,17,shuffle32_fused,2,12,100,29.5746278763,2788,0.0942699942553,2176,2909,true,1,0.0295746278763 +fp16,1,17,1,64,17,baseline_transform,2,13,100,9.2971432209,4352,0.468100780702,2176,2909,true,1,0.0092971432209 +fp16,1,17,1,64,17,control_optimized_transform,2,14,100,9.40777122974,4352,0.462596282767,2176,2909,true,1,0.00940777122974 +fp16,1,17,1,64,17,control_shuffle32_transform,3,0,100,7.99497127533,4352,0.544342168361,2176,2909,true,1,0.00799497127533 +fp16,1,17,1,64,17,shuffle32_transform,3,1,100,8.03062856197,4352,0.541925201298,2176,2909,true,1,0.00803062856197 +fp16,1,17,1,64,17,baseline_split,3,2,100,70.2608013153,7140,0.101621385842,2176,2909,true,1,0.0702608013153 +fp16,1,17,1,64,17,control_optimized_split,3,3,100,64.395198822,7140,0.110877831432,2176,2909,true,1,0.064395198822 +fp16,1,17,1,64,17,optimized_split,3,4,100,34.265370369,7140,0.208373641467,2176,2909,true,1,0.034265370369 +fp16,1,17,1,64,17,control_shuffle32_split,3,5,100,93.7563419342,7140,0.0761548483303,2176,2909,true,1,0.0937563419342 +fp16,1,17,1,64,17,shuffle32_split,3,6,100,34.7787427902,7140,0.205297817781,2176,2909,true,1,0.0347787427902 +fp16,1,17,1,64,17,baseline_fused,3,7,100,65.7229709625,2788,0.0424204803765,2176,2909,true,1,0.0657229709625 +fp16,1,17,1,64,17,control_optimized_fused,3,8,100,58.9046859741,2788,0.0473306996531,2176,2909,true,1,0.0589046859741 +fp16,1,17,1,64,17,optimized_fused,3,9,100,28.99954319,2788,0.0961394454296,2176,2909,true,1,0.02899954319 +fp16,1,17,1,64,17,control_shuffle32_fused,3,10,100,89.073600769,2788,0.0312999584156,2176,2909,true,1,0.089073600769 +fp16,1,17,1,64,17,shuffle32_fused,3,11,100,29.5682263374,2788,0.094290403766,2176,2909,true,1,0.0295682263374 +fp16,1,17,1,64,17,baseline_transform,3,12,100,9.22537088394,4352,0.471742551573,2176,2909,true,1,0.00922537088394 +fp16,1,17,1,64,17,control_optimized_transform,3,13,100,9.3364572525,4352,0.466129698054,2176,2909,true,1,0.0093364572525 +fp16,1,17,1,64,17,optimized_transform,3,14,100,9.36297118664,4352,0.464809718331,2176,2909,true,1,0.00936297118664 +fp16,1,17,1,64,17,shuffle32_transform,4,0,100,8.07748556137,4352,0.538781526372,2176,2909,true,1,0.00807748556137 +fp16,1,17,1,64,17,baseline_split,4,1,100,70.3357696533,7140,0.101513071303,2176,2909,true,1,0.0703357696533 +fp16,1,17,1,64,17,control_optimized_split,4,2,100,64.4157743454,7140,0.110842415116,2176,2909,true,1,0.0644157743454 +fp16,1,17,1,64,17,optimized_split,4,3,100,34.2507433891,7140,0.208462628647,2176,2909,true,1,0.0342507433891 +fp16,1,17,1,64,17,control_shuffle32_split,4,4,100,93.8306236267,7140,0.0760945597932,2176,2909,true,1,0.0938306236267 +fp16,1,17,1,64,17,shuffle32_split,4,5,100,34.702398777,7140,0.205749465502,2176,2909,true,1,0.034702398777 +fp16,1,17,1,64,17,baseline_fused,4,6,100,65.7263994217,2788,0.0424182676144,2176,2909,true,1,0.0657263994217 +fp16,1,17,1,64,17,control_optimized_fused,4,7,100,58.8585138321,2788,0.0473678286875,2176,2909,true,1,0.0588585138321 +fp16,1,17,1,64,17,optimized_fused,4,8,100,29.0511989594,2788,0.0959685004361,2176,2909,true,1,0.0290511989594 +fp16,1,17,1,64,17,control_shuffle32_fused,4,9,100,89.0756607056,2788,0.0312992345823,2176,2909,true,1,0.0890756607056 +fp16,1,17,1,64,17,shuffle32_fused,4,10,100,29.5746278763,2788,0.0942699942553,2176,2909,true,1,0.0295746278763 +fp16,1,17,1,64,17,baseline_transform,4,11,100,9.1979432106,4352,0.473149257432,2176,2909,true,1,0.0091979432106 +fp16,1,17,1,64,17,control_optimized_transform,4,12,100,9.38171386719,4352,0.463881126797,2176,2909,true,1,0.00938171386719 +fp16,1,17,1,64,17,optimized_transform,4,13,100,9.47268545628,4352,0.459426212354,2176,2909,true,1,0.00947268545628 +fp16,1,17,1,64,17,control_shuffle32_transform,4,14,100,7.9113137722,4352,0.550098267533,2176,2909,true,1,0.0079113137722 +fp16,1,257,1,64,257,baseline_transform,0,0,100,9.46880042553,65792,6.94829303009,32896,2909,true,1,0.00946880042553 +fp16,1,257,1,64,257,control_optimized_transform,0,1,100,9.61805701256,65792,6.8404668338,32896,2909,true,1,0.00961805701256 +fp16,1,257,1,64,257,optimized_transform,0,2,100,9.5236569643,65792,6.90827066185,32896,2909,true,1,0.0095236569643 +fp16,1,257,1,64,257,control_shuffle32_transform,0,3,100,8.88091444969,65792,7.40824611843,32896,2909,true,1,0.00888091444969 +fp16,1,257,1,64,257,shuffle32_transform,0,4,100,8.76274287701,65792,7.50815137719,32896,2909,true,1,0.00876274287701 +fp16,1,257,1,64,257,baseline_split,0,5,100,71.0491418839,107940,1.51923017137,32896,2909,true,1,0.0710491418839 +fp16,1,257,1,64,257,control_optimized_split,0,6,100,64.3958854675,107940,1.67619404899,32896,2909,true,1,0.0643958854675 +fp16,1,257,1,64,257,optimized_split,0,7,100,34.6847987175,107940,3.11202613223,32896,2909,true,1,0.0346847987175 +fp16,1,257,1,64,257,control_shuffle32_split,0,8,100,102.72023201,107940,1.05081538357,32896,2909,true,1,0.10272023201 +fp16,1,257,1,64,257,shuffle32_split,0,9,100,39.0500569344,107940,2.76414449744,32896,2909,true,1,0.0390500569344 +fp16,1,257,1,64,257,baseline_fused,0,10,100,65.9881162643,42148,0.63872106655,32896,2909,true,1,0.0659881162643 +fp16,1,257,1,64,257,control_optimized_fused,0,11,100,59.3058300018,42148,0.710688982832,32896,2909,true,1,0.0593058300018 +fp16,1,257,1,64,257,optimized_fused,0,12,100,29.3462872505,42148,1.43622938194,32896,2909,true,1,0.0293462872505 +fp16,1,257,1,64,257,control_shuffle32_fused,0,13,100,96.6580581665,42148,0.436052625094,32896,2909,true,1,0.0966580581665 +fp16,1,257,1,64,257,shuffle32_fused,0,14,100,33.231086731,42148,1.26833047445,32896,2909,true,1,0.033231086731 +fp16,1,257,1,64,257,control_optimized_transform,1,0,100,9.70560014248,65792,6.77876679795,32896,2909,true,1,0.00970560014248 +fp16,1,257,1,64,257,optimized_transform,1,1,100,9.53577160835,65792,6.89949410516,32896,2909,true,1,0.00953577160835 +fp16,1,257,1,64,257,control_shuffle32_transform,1,2,100,8.85257124901,65792,7.43196503585,32896,2909,true,1,0.00885257124901 +fp16,1,257,1,64,257,shuffle32_transform,1,3,100,8.82102847099,65792,7.4585407151,32896,2909,true,1,0.00882102847099 +fp16,1,257,1,64,257,baseline_split,1,4,100,71.0587453842,107940,1.51902484932,32896,2909,true,1,0.0710587453842 +fp16,1,257,1,64,257,control_optimized_split,1,5,100,64.4550848007,107940,1.67465453399,32896,2909,true,1,0.0644550848007 +fp16,1,257,1,64,257,optimized_split,1,6,100,34.7309732437,107940,3.10788872061,32896,2909,true,1,0.0347309732437 +fp16,1,257,1,64,257,control_shuffle32_split,1,7,100,102.770738602,107940,1.05029896125,32896,2909,true,1,0.102770738602 +fp16,1,257,1,64,257,shuffle32_split,1,8,100,39.0797710419,107940,2.76204279407,32896,2909,true,1,0.0390797710419 +fp16,1,257,1,64,257,baseline_fused,1,9,100,65.9791994095,42148,0.638807387438,32896,2909,true,1,0.0659791994095 +fp16,1,257,1,64,257,control_optimized_fused,1,10,100,59.2678880692,42148,0.7111439495,32896,2909,true,1,0.0592678880692 +fp16,1,257,1,64,257,optimized_fused,1,11,100,29.2592000961,42148,1.44050417857,32896,2909,true,1,0.0292592000961 +fp16,1,257,1,64,257,control_shuffle32_fused,1,12,100,96.69257164,42148,0.435896980348,32896,2909,true,1,0.09669257164 +fp16,1,257,1,64,257,shuffle32_fused,1,13,100,33.1940579414,42148,1.26974532835,32896,2909,true,1,0.0331940579414 +fp16,1,257,1,64,257,baseline_transform,1,14,100,9.49554264545,65792,6.92872460865,32896,2909,true,1,0.00949554264545 +fp16,1,257,1,64,257,optimized_transform,2,0,100,9.51885700226,65792,6.91175421423,32896,2909,true,1,0.00951885700226 +fp16,1,257,1,64,257,control_shuffle32_transform,2,1,100,8.84320020676,65792,7.43984060767,32896,2909,true,1,0.00884320020676 +fp16,1,257,1,64,257,shuffle32_transform,2,2,100,8.92022848129,65792,7.3755958312,32896,2909,true,1,0.00892022848129 +fp16,1,257,1,64,257,baseline_split,2,3,100,71.0779428482,107940,1.51861457542,32896,2909,true,1,0.0710779428482 +fp16,1,257,1,64,257,control_optimized_split,2,4,100,64.4582843781,107940,1.67457140756,32896,2909,true,1,0.0644582843781 +fp16,1,257,1,64,257,optimized_split,2,5,100,34.7138285637,107940,3.10942366389,32896,2909,true,1,0.0347138285637 +fp16,1,257,1,64,257,control_shuffle32_split,2,6,100,102.747430801,107940,1.05053721692,32896,2909,true,1,0.102747430801 +fp16,1,257,1,64,257,shuffle32_split,2,7,100,39.0742850304,107940,2.7624305836,32896,2909,true,1,0.0390742850304 +fp16,1,257,1,64,257,baseline_fused,2,8,100,65.984454155,42148,0.6387565153,32896,2909,true,1,0.065984454155 +fp16,1,257,1,64,257,control_optimized_fused,2,9,100,59.3003463745,42148,0.710754701732,32896,2909,true,1,0.0593003463745 +fp16,1,257,1,64,257,optimized_fused,2,10,100,29.301712513,42148,1.43841422174,32896,2909,true,1,0.029301712513 +fp16,1,257,1,64,257,control_shuffle32_fused,2,11,100,96.693944931,42148,0.435890789543,32896,2909,true,1,0.096693944931 +fp16,1,257,1,64,257,shuffle32_fused,2,12,100,33.2093715668,42148,1.2691598188,32896,2909,true,1,0.0332093715668 +fp16,1,257,1,64,257,baseline_transform,2,13,100,9.5490282774,65792,6.8899157159,32896,2909,true,1,0.0095490282774 +fp16,1,257,1,64,257,control_optimized_transform,2,14,100,9.69988584518,65792,6.78276023554,32896,2909,true,1,0.00969988584518 +fp16,1,257,1,64,257,control_shuffle32_transform,3,0,100,8.78034293652,65792,7.49310140568,32896,2909,true,1,0.00878034293652 +fp16,1,257,1,64,257,shuffle32_transform,3,1,100,8.79657149315,65792,7.47927758573,32896,2909,true,1,0.00879657149315 +fp16,1,257,1,64,257,baseline_split,3,2,100,71.0198831558,107940,1.51985606289,32896,2909,true,1,0.0710198831558 +fp16,1,257,1,64,257,control_optimized_split,3,3,100,64.4114303589,107940,1.67578952056,32896,2909,true,1,0.0644114303589 +fp16,1,257,1,64,257,optimized_split,3,4,100,34.743771553,107940,3.10674389035,32896,2909,true,1,0.034743771553 +fp16,1,257,1,64,257,control_shuffle32_split,3,5,100,102.096681595,107940,1.05723318637,32896,2909,true,1,0.102096681595 +fp16,1,257,1,64,257,shuffle32_split,3,6,100,39.6532559395,107940,2.7220967722,32896,2909,true,1,0.0396532559395 +fp16,1,257,1,64,257,baseline_fused,3,7,100,65.9437704086,42148,0.639150593586,32896,2909,true,1,0.0659437704086 +fp16,1,257,1,64,257,control_optimized_fused,3,8,100,59.30788517,42148,0.710664355662,32896,2909,true,1,0.05930788517 +fp16,1,257,1,64,257,optimized_fused,3,9,100,29.352915287,42148,1.43590507409,32896,2909,true,1,0.029352915287 +fp16,1,257,1,64,257,control_shuffle32_fused,3,10,100,96.6969108582,42148,0.435877419723,32896,2909,true,1,0.0966969108582 +fp16,1,257,1,64,257,shuffle32_fused,3,11,100,33.2185125351,42148,1.26881057529,32896,2909,true,1,0.0332185125351 +fp16,1,257,1,64,257,baseline_transform,3,12,100,9.60777163506,65792,6.84778973721,32896,2909,true,1,0.00960777163506 +fp16,1,257,1,64,257,control_optimized_transform,3,13,100,9.56068575382,65792,6.88151474633,32896,2909,true,1,0.00956068575382 +fp16,1,257,1,64,257,optimized_transform,3,14,100,9.52914297581,65792,6.90429350961,32896,2909,true,1,0.00952914297581 +fp16,1,257,1,64,257,shuffle32_transform,4,0,100,8.84548604488,65792,7.43791801447,32896,2909,true,1,0.00884548604488 +fp16,1,257,1,64,257,baseline_split,4,1,100,71.0011434555,107940,1.52025720639,32896,2909,true,1,0.0710011434555 +fp16,1,257,1,64,257,control_optimized_split,4,2,100,65.6175994873,107940,1.64498550455,32896,2909,true,1,0.0656175994873 +fp16,1,257,1,64,257,optimized_split,4,3,100,34.7309732437,107940,3.10788872061,32896,2909,true,1,0.0347309732437 +fp16,1,257,1,64,257,control_shuffle32_split,4,4,100,102.111768723,107940,1.05707697898,32896,2909,true,1,0.102111768723 +fp16,1,257,1,64,257,shuffle32_split,4,5,100,39.1165733337,107940,2.75944416396,32896,2909,true,1,0.0391165733337 +fp16,1,257,1,64,257,baseline_fused,4,6,100,65.9739398956,42148,0.638858313854,32896,2909,true,1,0.0659739398956 +fp16,1,257,1,64,257,control_optimized_fused,4,7,100,59.3097114563,42148,0.710642472625,32896,2909,true,1,0.0593097114563 +fp16,1,257,1,64,257,optimized_fused,4,8,100,29.2964577675,42148,1.43867222224,32896,2909,true,1,0.0292964577675 +fp16,1,257,1,64,257,control_shuffle32_fused,4,9,100,96.6976070404,42148,0.435874281588,32896,2909,true,1,0.0966976070404 +fp16,1,257,1,64,257,shuffle32_fused,4,10,100,33.1974864006,42148,1.26961419583,32896,2909,true,1,0.0331974864006 +fp16,1,257,1,64,257,baseline_transform,4,11,100,9.52137172222,65792,6.9099287287,32896,2909,true,1,0.00952137172222 +fp16,1,257,1,64,257,control_optimized_transform,4,12,100,9.56799983978,65792,6.87625429575,32896,2909,true,1,0.00956799983978 +fp16,1,257,1,64,257,optimized_transform,4,13,100,9.52319979668,65792,6.90860229804,32896,2909,true,1,0.00952319979668 +fp16,1,257,1,64,257,control_shuffle32_transform,4,14,100,8.73919963837,65792,7.52837819509,32896,2909,true,1,0.00873919963837 +fp16,4,64,16,64,4096,baseline_transform,0,0,100,22.7426290512,1048576,46.1061910494,524288,2909,true,1,0.0227426290512 +fp16,4,64,16,64,4096,control_optimized_transform,0,1,100,23.9334869385,1048576,43.8120865002,524288,2909,true,1,0.0239334869385 +fp16,4,64,16,64,4096,optimized_transform,0,2,100,23.9567995071,1048576,43.7694525802,524288,2909,true,1,0.0239567995071 +fp16,4,64,16,64,4096,control_shuffle32_transform,0,3,100,10.9737145901,1048576,95.5534237193,524288,2909,true,1,0.0109737145901 +fp16,4,64,16,64,4096,shuffle32_transform,0,4,100,10.8262860775,1048576,96.8546362523,524288,2909,true,1,0.0108262860775 +fp16,4,64,16,64,4096,baseline_split,0,5,100,438.530311584,1720320,3.92292152801,524288,2909,true,1,0.438530311584 +fp16,4,64,16,64,4096,control_optimized_split,0,6,100,382.921829224,1720320,4.49261407606,524288,2909,true,1,0.382921829224 +fp16,4,64,16,64,4096,optimized_split,0,7,100,162.066059113,1720320,10.6149307845,524288,2909,true,1,0.162066059113 +fp16,4,64,16,64,4096,control_shuffle32_split,0,8,100,248.774185181,1720320,6.91518695459,524288,2909,true,1,0.248774185181 +fp16,4,64,16,64,4096,shuffle32_split,0,9,100,71.3005685806,1720320,24.1277178324,524288,2909,true,1,0.0713005685806 +fp16,4,64,16,64,4096,baseline_fused,0,10,100,433.897590637,671744,1.54816254917,524288,2909,true,1,0.433897590637 +fp16,4,64,16,64,4096,control_optimized_fused,0,11,100,372.053718567,671744,1.80550271769,524288,2909,true,1,0.372053718567 +fp16,4,64,16,64,4096,optimized_fused,0,12,100,147.934169769,671744,4.54083056705,524288,2909,true,1,0.147934169769 +fp16,4,64,16,64,4096,control_shuffle32_fused,0,13,100,248.180103302,671744,2.7066795084,524288,2909,true,1,0.248180103302 +fp16,4,64,16,64,4096,shuffle32_fused,0,14,100,69.8285722733,671744,9.61990168396,524288,2909,true,1,0.0698285722733 +fp16,4,64,16,64,4096,control_optimized_transform,1,0,100,23.9270853996,1048576,43.8238081441,524288,2909,true,1,0.0239270853996 +fp16,4,64,16,64,4096,optimized_transform,1,1,100,23.9862847328,1048576,43.7156488252,524288,2909,true,1,0.0239862847328 +fp16,4,64,16,64,4096,control_shuffle32_transform,1,2,100,11.0566866398,1048576,94.8363677258,524288,2909,true,1,0.0110566866398 +fp16,4,64,16,64,4096,shuffle32_transform,1,3,100,11.245714426,1048576,93.2422752593,524288,2909,true,1,0.011245714426 +fp16,4,64,16,64,4096,baseline_split,1,4,100,438.254852295,1720320,3.92538722844,524288,2909,true,1,0.438254852295 +fp16,4,64,16,64,4096,control_optimized_split,1,5,100,381.6040802,1720320,4.50812789815,524288,2909,true,1,0.3816040802 +fp16,4,64,16,64,4096,optimized_split,1,6,100,162.126617432,1720320,10.6109658442,524288,2909,true,1,0.162126617432 +fp16,4,64,16,64,4096,control_shuffle32_split,1,7,100,249.012584686,1720320,6.90856649742,524288,2909,true,1,0.249012584686 +fp16,4,64,16,64,4096,shuffle32_split,1,8,100,72.3741722107,1720320,23.769805546,524288,2909,true,1,0.0723741722107 +fp16,4,64,16,64,4096,baseline_fused,1,9,100,433.837242126,671744,1.54837790483,524288,2909,true,1,0.433837242126 +fp16,4,64,16,64,4096,control_optimized_fused,1,10,100,371.584663391,671744,1.8077818225,524288,2909,true,1,0.371584663391 +fp16,4,64,16,64,4096,optimized_fused,1,11,100,147.560453415,671744,4.55233082072,524288,2909,true,1,0.147560453415 +fp16,4,64,16,64,4096,control_shuffle32_fused,1,12,100,247.728691101,671744,2.71161163051,524288,2909,true,1,0.247728691101 +fp16,4,64,16,64,4096,shuffle32_fused,1,13,100,69.2089128494,671744,9.70603311544,524288,2909,true,1,0.0692089128494 +fp16,4,64,16,64,4096,baseline_transform,1,14,100,22.7430868149,1048576,46.1052630426,524288,2909,true,1,0.0227430868149 +fp16,4,64,16,64,4096,optimized_transform,2,0,100,24.1625142097,1048576,43.3968084156,524288,2909,true,1,0.0241625142097 +fp16,4,64,16,64,4096,control_shuffle32_transform,2,1,100,10.9732568264,1048576,95.5574098546,524288,2909,true,1,0.0109732568264 +fp16,4,64,16,64,4096,shuffle32_transform,2,2,100,11.2080001831,1048576,93.5560298777,524288,2909,true,1,0.0112080001831 +fp16,4,64,16,64,4096,baseline_split,2,3,100,437.118644714,1720320,3.93559053315,524288,2909,true,1,0.437118644714 +fp16,4,64,16,64,4096,control_optimized_split,2,4,100,381.392250061,1720320,4.51063177011,524288,2909,true,1,0.381392250061 +fp16,4,64,16,64,4096,optimized_split,2,5,100,162.082958221,1720320,10.6138240496,524288,2909,true,1,0.162082958221 +fp16,4,64,16,64,4096,control_shuffle32_split,2,6,100,247.370052338,1720320,6.95443924494,524288,2909,true,1,0.247370052338 +fp16,4,64,16,64,4096,shuffle32_split,2,7,100,70.7430839539,1720320,24.3178541824,524288,2909,true,1,0.0707430839539 +fp16,4,64,16,64,4096,baseline_fused,2,8,100,434.00844574,671744,1.54776711512,524288,2909,true,1,0.43400844574 +fp16,4,64,16,64,4096,control_optimized_fused,2,9,100,372.027435303,671744,1.80563027416,524288,2909,true,1,0.372027435303 +fp16,4,64,16,64,4096,optimized_fused,2,10,100,147.455768585,671744,4.55556270497,524288,2909,true,1,0.147455768585 +fp16,4,64,16,64,4096,control_shuffle32_fused,2,11,100,248.198394775,671744,2.70648003428,524288,2909,true,1,0.248198394775 +fp16,4,64,16,64,4096,shuffle32_fused,2,12,100,69.9654865265,671744,9.60107666436,524288,2909,true,1,0.0699654865265 +fp16,4,64,16,64,4096,baseline_transform,2,13,100,22.7556562424,1048576,46.0797961101,524288,2909,true,1,0.0227556562424 +fp16,4,64,16,64,4096,control_optimized_transform,2,14,100,24.1645693779,1048576,43.3931175682,524288,2909,true,1,0.0241645693779 +fp16,4,64,16,64,4096,control_shuffle32_transform,3,0,100,11.4507424831,1048576,91.5727518581,524288,2909,true,1,0.0114507424831 +fp16,4,64,16,64,4096,shuffle32_transform,3,1,100,11.2610292435,1048576,93.1154672747,524288,2909,true,1,0.0112610292435 +fp16,4,64,16,64,4096,baseline_split,3,2,100,437.115859985,1720320,3.93561560557,524288,2909,true,1,0.437115859985 +fp16,4,64,16,64,4096,control_optimized_split,3,3,100,380.504226685,1720320,4.5211587135,524288,2909,true,1,0.380504226685 +fp16,4,64,16,64,4096,optimized_split,3,4,100,162.909030914,1720320,10.5600038889,524288,2909,true,1,0.162909030914 +fp16,4,64,16,64,4096,control_shuffle32_split,3,5,100,249.335765839,1720320,6.89961183151,524288,2909,true,1,0.249335765839 +fp16,4,64,16,64,4096,shuffle32_split,3,6,100,70.339884758,1720320,24.4572479173,524288,2909,true,1,0.070339884758 +fp16,4,64,16,64,4096,baseline_fused,3,7,100,433.848686218,671744,1.5483370616,524288,2909,true,1,0.433848686218 +fp16,4,64,16,64,4096,control_optimized_fused,3,8,100,372.009124756,671744,1.80571914853,524288,2909,true,1,0.372009124756 +fp16,4,64,16,64,4096,optimized_fused,3,9,100,148.069486618,671744,4.53668082022,524288,2909,true,1,0.148069486618 +fp16,4,64,16,64,4096,control_shuffle32_fused,3,10,100,247.899665833,671744,2.70974145021,524288,2909,true,1,0.247899665833 +fp16,4,64,16,64,4096,shuffle32_fused,3,11,100,70.5846881866,671744,9.51685156168,524288,2909,true,1,0.0705846881866 +fp16,4,64,16,64,4096,baseline_transform,3,12,100,22.7588582039,1048576,46.0733131076,524288,2909,true,1,0.0227588582039 +fp16,4,64,16,64,4096,control_optimized_transform,3,13,100,24.0902853012,1048576,43.5269232759,524288,2909,true,1,0.0240902853012 +fp16,4,64,16,64,4096,optimized_transform,3,14,100,24.0573716164,1048576,43.5864738975,524288,2909,true,1,0.0240573716164 +fp16,4,64,16,64,4096,shuffle32_transform,4,0,100,10.9570288658,1048576,95.6989356185,524288,2909,true,1,0.0109570288658 +fp16,4,64,16,64,4096,baseline_split,4,1,100,438.287086487,1720320,3.92509853254,524288,2909,true,1,0.438287086487 +fp16,4,64,16,64,4096,control_optimized_split,4,2,100,380.737800598,1720320,4.51838508627,524288,2909,true,1,0.380737800598 +fp16,4,64,16,64,4096,optimized_split,4,3,100,162.324790955,1720320,10.5980114922,524288,2909,true,1,0.162324790955 +fp16,4,64,16,64,4096,control_shuffle32_split,4,4,100,247.746963501,1720320,6.94385907173,524288,2909,true,1,0.247746963501 +fp16,4,64,16,64,4096,shuffle32_split,4,5,100,70.5702829361,1720320,24.377399784,524288,2909,true,1,0.0705702829361 +fp16,4,64,16,64,4096,baseline_fused,4,6,100,433.8697052,671744,1.54826205183,524288,2909,true,1,0.4338697052 +fp16,4,64,16,64,4096,control_optimized_fused,4,7,100,373.726196289,671744,1.79742283701,524288,2909,true,1,0.373726196289 +fp16,4,64,16,64,4096,optimized_fused,4,8,100,147.559776306,671744,4.55235171004,524288,2909,true,1,0.147559776306 +fp16,4,64,16,64,4096,control_shuffle32_fused,4,9,100,248.44707489,671744,2.70377101561,524288,2909,true,1,0.24844707489 +fp16,4,64,16,64,4096,shuffle32_fused,4,10,100,69.651427269,671744,9.64436805302,524288,2909,true,1,0.069651427269 +fp16,4,64,16,64,4096,baseline_transform,4,11,100,23.1677722931,1048576,45.2601133477,524288,2909,true,1,0.0231677722931 +fp16,4,64,16,64,4096,control_optimized_transform,4,12,100,23.9209127426,1048576,43.8351166313,524288,2909,true,1,0.0239209127426 +fp16,4,64,16,64,4096,optimized_transform,4,13,100,24.0141701698,1048576,43.6648858813,524288,2909,true,1,0.0240141701698 +fp16,4,64,16,64,4096,control_shuffle32_transform,4,14,100,11.3602280617,1048576,92.3023723034,524288,2909,true,1,0.0113602280617 +fp16,16,64,16,64,16384,baseline_transform,0,0,100,68.2253694534,4194304,61.4771899896,2097152,2909,true,1,0.0682253694534 +fp16,16,64,16,64,16384,control_optimized_transform,0,1,100,73.9346313477,4194304,56.7298967148,2097152,2909,true,1,0.0739346313477 +fp16,16,64,16,64,16384,optimized_transform,0,2,100,73.9755439758,4194304,56.6985218976,2097152,2909,true,1,0.0739755439758 +fp16,16,64,16,64,16384,control_shuffle32_transform,0,3,100,21.1666297913,4194304,198.156439705,2097152,2909,true,1,0.0211666297913 +fp16,16,64,16,64,16384,shuffle32_transform,0,4,100,21.0834288597,4194304,198.938418789,2097152,2909,true,1,0.0210834288597 +fp16,16,64,16,64,16384,baseline_split,0,5,100,1548.53668213,6881280,4.44373070358,2097152,2909,true,1,1.54853668213 +fp16,16,64,16,64,16384,control_optimized_split,0,6,100,1361.771698,6881280,5.05318182932,2097152,2909,true,1,1.361771698 +fp16,16,64,16,64,16384,optimized_split,0,7,100,576.756095886,6881280,11.9310052361,2097152,2909,true,1,0.576756095886 +fp16,16,64,16,64,16384,control_shuffle32_split,0,8,100,851.639480591,6881280,8.08003874506,2097152,2909,true,1,0.851639480591 +fp16,16,64,16,64,16384,shuffle32_split,0,9,100,228.424682617,6881280,30.1249406201,2097152,2909,true,1,0.228424682617 +fp16,16,64,16,64,16384,baseline_fused,0,10,100,1526.07208252,2686976,1.76071368501,2097152,2909,true,1,1.52607208252 +fp16,16,64,16,64,16384,control_optimized_fused,0,11,100,1333.47381592,2686976,2.01501969362,2097152,2909,true,1,1.33347381592 +fp16,16,64,16,64,16384,optimized_fused,0,12,100,544.50969696,2686976,4.93467061285,2097152,2909,true,1,0.54450969696 +fp16,16,64,16,64,16384,control_shuffle32_fused,0,13,100,866.521606445,2686976,3.10087593894,2097152,2909,true,1,0.866521606445 +fp16,16,64,16,64,16384,shuffle32_fused,0,14,100,211.608905792,2686976,12.6978398661,2097152,2909,true,1,0.211608905792 +fp16,16,64,16,64,16384,control_optimized_transform,1,0,100,73.9764547348,4194304,56.6978238554,2097152,2909,true,1,0.0739764547348 +fp16,16,64,16,64,16384,optimized_transform,1,1,100,74.0196561813,4194304,56.6647322669,2097152,2909,true,1,0.0740196561813 +fp16,16,64,16,64,16384,control_shuffle32_transform,1,2,100,21.2486863136,4194304,197.391214595,2097152,2909,true,1,0.0212486863136 +fp16,16,64,16,64,16384,shuffle32_transform,1,3,100,21.2210273743,4194304,197.64848921,2097152,2909,true,1,0.0212210273743 +fp16,16,64,16,64,16384,baseline_split,1,4,100,1548.38577271,6881280,4.44416380033,2097152,2909,true,1,1.54838577271 +fp16,16,64,16,64,16384,control_optimized_split,1,5,100,1361.37619019,6881280,5.05464988268,2097152,2909,true,1,1.36137619019 +fp16,16,64,16,64,16384,optimized_split,1,6,100,576.796798706,6881280,11.9301633009,2097152,2909,true,1,0.576796798706 +fp16,16,64,16,64,16384,control_shuffle32_split,1,7,100,851.68800354,6881280,8.07957840359,2097152,2909,true,1,0.85168800354 +fp16,16,64,16,64,16384,shuffle32_split,1,8,100,228.358631134,6881280,30.1336540941,2097152,2909,true,1,0.228358631134 +fp16,16,64,16,64,16384,baseline_fused,1,9,100,1526.11831665,2686976,1.76066034375,2097152,2909,true,1,1.52611831665 +fp16,16,64,16,64,16384,control_optimized_fused,1,10,100,1333.51333618,2686976,2.0149599761,2097152,2909,true,1,1.33351333618 +fp16,16,64,16,64,16384,optimized_fused,1,11,100,544.506034851,2686976,4.93470380128,2097152,2909,true,1,0.544506034851 +fp16,16,64,16,64,16384,control_shuffle32_fused,1,12,100,866.382675171,2686976,3.10137318878,2097152,2909,true,1,0.866382675171 +fp16,16,64,16,64,16384,shuffle32_fused,1,13,100,211.606616974,2686976,12.6979772109,2097152,2909,true,1,0.211606616974 +fp16,16,64,16,64,16384,baseline_transform,1,14,100,68.223772049,4194304,61.4786294283,2097152,2909,true,1,0.068223772049 +fp16,16,64,16,64,16384,optimized_transform,2,0,100,74.0361118317,4194304,56.6521376695,2097152,2909,true,1,0.0740361118317 +fp16,16,64,16,64,16384,control_shuffle32_transform,2,1,100,21.2338280678,4194304,197.529337932,2097152,2909,true,1,0.0212338280678 +fp16,16,64,16,64,16384,shuffle32_transform,2,2,100,21.1769127846,4194304,198.060219762,2097152,2909,true,1,0.0211769127846 +fp16,16,64,16,64,16384,baseline_split,2,3,100,1548.54751587,6881280,4.44369961495,2097152,2909,true,1,1.54854751587 +fp16,16,64,16,64,16384,control_optimized_split,2,4,100,1361.72363281,6881280,5.05336019306,2097152,2909,true,1,1.36172363281 +fp16,16,64,16,64,16384,optimized_split,2,5,100,576.742172241,6881280,11.9312932731,2097152,2909,true,1,0.576742172241 +fp16,16,64,16,64,16384,control_shuffle32_split,2,6,100,851.716079712,6881280,8.07931206644,2097152,2909,true,1,0.851716079712 +fp16,16,64,16,64,16384,shuffle32_split,2,7,100,228.408679962,6881280,30.1270512186,2097152,2909,true,1,0.228408679962 +fp16,16,64,16,64,16384,baseline_fused,2,8,100,1526.19537354,2686976,1.76057144884,2097152,2909,true,1,1.52619537354 +fp16,16,64,16,64,16384,control_optimized_fused,2,9,100,1333.60671997,2686976,2.01481888158,2097152,2909,true,1,1.33360671997 +fp16,16,64,16,64,16384,optimized_fused,2,10,100,544.504890442,2686976,4.93471417276,2097152,2909,true,1,0.544504890442 +fp16,16,64,16,64,16384,control_shuffle32_fused,2,11,100,866.449813843,2686976,3.10113287241,2097152,2909,true,1,0.866449813843 +fp16,16,64,16,64,16384,shuffle32_fused,2,12,100,211.472454071,2686976,12.7060330945,2097152,2909,true,1,0.211472454071 +fp16,16,64,16,64,16384,baseline_transform,2,13,100,68.2610273361,4194304,61.4450758168,2097152,2909,true,1,0.0682610273361 +fp16,16,64,16,64,16384,control_optimized_transform,2,14,100,73.9787435532,4194304,56.6960696891,2097152,2909,true,1,0.0739787435532 +fp16,16,64,16,64,16384,control_shuffle32_transform,3,0,100,21.2260580063,4194304,197.601645994,2097152,2909,true,1,0.0212260580063 +fp16,16,64,16,64,16384,shuffle32_transform,3,1,100,21.1979436874,4194304,197.863720267,2097152,2909,true,1,0.0211979436874 +fp16,16,64,16,64,16384,baseline_split,3,2,100,1548.46694946,6881280,4.4439308197,2097152,2909,true,1,1.54846694946 +fp16,16,64,16,64,16384,control_optimized_split,3,3,100,1361.77688599,6881280,5.05316257811,2097152,2909,true,1,1.36177688599 +fp16,16,64,16,64,16384,optimized_split,3,4,100,576.836357117,6881280,11.9293451515,2097152,2909,true,1,0.576836357117 +fp16,16,64,16,64,16384,control_shuffle32_split,3,5,100,851.702194214,6881280,8.07944378534,2097152,2909,true,1,0.851702194214 +fp16,16,64,16,64,16384,shuffle32_split,3,6,100,228.368015289,6881280,30.1324158345,2097152,2909,true,1,0.228368015289 +fp16,16,64,16,64,16384,baseline_fused,3,7,100,1526.17523193,2686976,1.76059468387,2097152,2909,true,1,1.52617523193 +fp16,16,64,16,64,16384,control_optimized_fused,3,8,100,1333.45306396,2686976,2.0150510525,2097152,2909,true,1,1.33345306396 +fp16,16,64,16,64,16384,optimized_fused,3,9,100,544.457588196,2686976,4.93514289865,2097152,2909,true,1,0.544457588196 +fp16,16,64,16,64,16384,control_shuffle32_fused,3,10,100,866.582641602,2686976,3.10065753802,2097152,2909,true,1,0.866582641602 +fp16,16,64,16,64,16384,shuffle32_fused,3,11,100,211.415081024,2686976,12.709481211,2097152,2909,true,1,0.211415081024 +fp16,16,64,16,64,16384,baseline_transform,3,12,100,68.2678842545,4194304,61.4389041905,2097152,2909,true,1,0.0682678842545 +fp16,16,64,16,64,16384,control_optimized_transform,3,13,100,73.9714288712,4194304,56.7016760932,2097152,2909,true,1,0.0739714288712 +fp16,16,64,16,64,16384,optimized_transform,3,14,100,74.0271997452,4194304,56.6589579835,2097152,2909,true,1,0.0740271997452 +fp16,16,64,16,64,16384,shuffle32_transform,4,0,100,21.2089133263,4194304,197.761381523,2097152,2909,true,1,0.0212089133263 +fp16,16,64,16,64,16384,baseline_split,4,1,100,1548.51821899,6881280,4.44378368662,2097152,2909,true,1,1.54851821899 +fp16,16,64,16,64,16384,control_optimized_split,4,2,100,1361.57577515,6881280,5.05390895285,2097152,2909,true,1,1.36157577515 +fp16,16,64,16,64,16384,optimized_split,4,3,100,576.618499756,6881280,11.9338522835,2097152,2909,true,1,0.576618499756 +fp16,16,64,16,64,16384,control_shuffle32_split,4,4,100,851.656951904,6881280,8.07987298714,2097152,2909,true,1,0.851656951904 +fp16,16,64,16,64,16384,shuffle32_split,4,5,100,228.479766846,6881280,30.1176777927,2097152,2909,true,1,0.228479766846 +fp16,16,64,16,64,16384,baseline_fused,4,6,100,1526.02416992,2686976,1.76076896615,2097152,2909,true,1,1.52602416992 +fp16,16,64,16,64,16384,control_optimized_fused,4,7,100,1333.5168457,2686976,2.01495467317,2097152,2909,true,1,1.3335168457 +fp16,16,64,16,64,16384,optimized_fused,4,8,100,544.49005127,2686976,4.9348486602,2097152,2909,true,1,0.54449005127 +fp16,16,64,16,64,16384,control_shuffle32_fused,4,9,100,866.471786499,2686976,3.1010542315,2097152,2909,true,1,0.866471786499 +fp16,16,64,16,64,16384,shuffle32_fused,4,10,100,211.674747467,2686976,12.6938901884,2097152,2909,true,1,0.211674747467 +fp16,16,64,16,64,16384,baseline_transform,4,11,100,68.2207965851,4194304,61.4813108312,2097152,2909,true,1,0.0682207965851 +fp16,16,64,16,64,16384,control_optimized_transform,4,12,100,73.9250278473,4194304,56.737266419,2097152,2909,true,1,0.0739250278473 +fp16,16,64,16,64,16384,optimized_transform,4,13,100,73.9970302582,4194304,56.6820585281,2097152,2909,true,1,0.0739970302582 +fp16,16,64,16,64,16384,control_shuffle32_transform,4,14,100,21.1332583427,4194304,198.469347792,2097152,2909,true,1,0.0211332583427 +fp16,1,1,1,128,1,baseline_transform,0,0,100,9.29188549519,512,0.0551018413072,256,2909,true,1,0.00929188549519 +fp16,1,1,1,128,1,control_optimized_transform,0,1,100,9.43040013313,512,0.0542925000819,256,2909,true,1,0.00943040013313 +fp16,1,1,1,128,1,optimized_transform,0,2,100,9.52617168427,512,0.053746669383,256,2909,true,1,0.00952617168427 +fp16,1,1,1,128,1,control_shuffle32_transform,0,3,100,8.995885849,512,0.0569149062798,256,2909,true,1,0.008995885849 +fp16,1,1,1,128,1,shuffle32_transform,0,4,100,9.17120039463,512,0.0558269340947,256,2909,true,1,0.00917120039463 +fp16,1,1,1,128,1,baseline_split,0,5,100,80.1353168488,836,0.010432354084,256,2909,true,1,0.0801353168488 +fp16,1,1,1,128,1,control_optimized_split,0,6,100,64.6671962738,836,0.012927729176,256,2909,true,1,0.0646671962738 +fp16,1,1,1,128,1,optimized_split,0,7,100,34.0617156029,836,0.0245436844623,256,2909,true,1,0.0340617156029 +fp16,1,1,1,128,1,control_shuffle32_split,0,8,100,165.305137634,836,0.00505731407967,256,2909,true,1,0.165305137634 +fp16,1,1,1,128,1,shuffle32_split,0,9,100,46.0518836975,836,0.0181534376637,256,2909,true,1,0.0460518836975 +fp16,1,1,1,128,1,baseline_fused,0,10,100,75.7604551315,324,0.00427663745469,256,2909,true,1,0.0757604551315 +fp16,1,1,1,128,1,control_optimized_fused,0,11,100,59.242515564,324,0.00546904527797,256,2909,true,1,0.059242515564 +fp16,1,1,1,128,1,optimized_fused,0,12,100,28.8578295708,324,0.0112274555924,256,2909,true,1,0.0288578295708 +fp16,1,1,1,128,1,control_shuffle32_fused,0,13,100,159.296226501,324,0.00203394648521,256,2909,true,1,0.159296226501 +fp16,1,1,1,128,1,shuffle32_fused,0,14,100,40.3638839722,324,0.00802697778597,256,2909,true,1,0.0403638839722 +fp16,1,1,1,128,1,control_optimized_transform,1,0,100,9.49417114258,512,0.0539278250108,256,2909,true,1,0.00949417114258 +fp16,1,1,1,128,1,optimized_transform,1,1,100,9.62742865086,512,0.053181386076,256,2909,true,1,0.00962742865086 +fp16,1,1,1,128,1,control_shuffle32_transform,1,2,100,9.0464001894,512,0.0565970982137,256,2909,true,1,0.0090464001894 +fp16,1,1,1,128,1,shuffle32_transform,1,3,100,9.1867429018,512,0.0557324838055,256,2909,true,1,0.0091867429018 +fp16,1,1,1,128,1,baseline_split,1,4,100,80.1693725586,836,0.0104279224512,256,2909,true,1,0.0801693725586 +fp16,1,1,1,128,1,control_optimized_split,1,5,100,64.6658277512,836,0.0129280027655,256,2909,true,1,0.0646658277512 +fp16,1,1,1,128,1,optimized_split,1,6,100,34.0786290169,836,0.0245315032945,256,2909,true,1,0.0340786290169 +fp16,1,1,1,128,1,control_shuffle32_split,1,7,100,165.314064026,836,0.00505704100208,256,2909,true,1,0.165314064026 +fp16,1,1,1,128,1,shuffle32_split,1,8,100,46.7028570175,836,0.0179004038165,256,2909,true,1,0.0467028570175 +fp16,1,1,1,128,1,baseline_fused,1,9,100,75.7556533813,324,0.00427690852812,256,2909,true,1,0.0757556533813 +fp16,1,1,1,128,1,control_optimized_fused,1,10,100,59.2349720001,324,0.00546974175997,256,2909,true,1,0.0592349720001 +fp16,1,1,1,128,1,optimized_fused,1,11,100,28.8886857033,324,0.0112154634976,256,2909,true,1,0.0288886857033 +fp16,1,1,1,128,1,control_shuffle32_fused,1,12,100,159.31977272,324,0.0020336458838,256,2909,true,1,0.15931977272 +fp16,1,1,1,128,1,shuffle32_fused,1,13,100,40.4020595551,324,0.00801939315887,256,2909,true,1,0.0404020595551 +fp16,1,1,1,128,1,baseline_transform,1,14,100,9.28640007973,512,0.0551343896025,256,2909,true,1,0.00928640007973 +fp16,1,1,1,128,1,optimized_transform,2,0,100,9.69622850418,512,0.0528040361032,256,2909,true,1,0.00969622850418 +fp16,1,1,1,128,1,control_shuffle32_transform,2,1,100,9.03405725956,512,0.0566744249333,256,2909,true,1,0.00903405725956 +fp16,1,1,1,128,1,shuffle32_transform,2,2,100,9.1053712368,512,0.0562305464197,256,2909,true,1,0.0091053712368 +fp16,1,1,1,128,1,baseline_split,2,3,100,80.1673126221,836,0.0104281904015,256,2909,true,1,0.0801673126221 +fp16,1,1,1,128,1,control_optimized_split,2,4,100,64.666967392,836,0.0129277749323,256,2909,true,1,0.064666967392 +fp16,1,1,1,128,1,optimized_split,2,5,100,34.0912008286,836,0.0245224568124,256,2909,true,1,0.0340912008286 +fp16,1,1,1,128,1,control_shuffle32_split,2,6,100,165.33531189,836,0.00505639110269,256,2909,true,1,0.16533531189 +fp16,1,1,1,128,1,shuffle32_split,2,7,100,46.0591983795,836,0.0181505547081,256,2909,true,1,0.0460591983795 +fp16,1,1,1,128,1,baseline_fused,2,8,100,75.7520008087,324,0.00427711474999,256,2909,true,1,0.0757520008087 +fp16,1,1,1,128,1,control_optimized_fused,2,9,100,59.2436552048,324,0.00546894007266,256,2909,true,1,0.0592436552048 +fp16,1,1,1,128,1,optimized_fused,2,10,100,28.8948583603,324,0.011213067597,256,2909,true,1,0.0288948583603 +fp16,1,1,1,128,1,control_shuffle32_fused,2,11,100,159.34170723,324,0.0020333659381,256,2909,true,1,0.15934170723 +fp16,1,1,1,128,1,shuffle32_fused,2,12,100,40.3481149673,324,0.00803011492017,256,2909,true,1,0.0403481149673 +fp16,1,1,1,128,1,baseline_transform,2,13,100,9.2299425602,512,0.0554716344832,256,2909,true,1,0.0092299425602 +fp16,1,1,1,128,1,control_optimized_transform,2,14,100,9.51885700226,512,0.0537879705387,256,2909,true,1,0.00951885700226 +fp16,1,1,1,128,1,control_shuffle32_transform,3,0,100,8.99954319,512,0.0568917765258,256,2909,true,1,0.00899954319 +fp16,1,1,1,128,1,shuffle32_transform,3,1,100,9.14994299412,512,0.0559566327713,256,2909,true,1,0.00914994299412 +fp16,1,1,1,128,1,baseline_split,3,2,100,80.2575969696,836,0.0104164593953,256,2909,true,1,0.0802575969696 +fp16,1,1,1,128,1,control_optimized_split,3,3,100,64.666056633,836,0.0129279570076,256,2909,true,1,0.064666056633 +fp16,1,1,1,128,1,optimized_split,3,4,100,34.1085720062,836,0.0245099677538,256,2909,true,1,0.0341085720062 +fp16,1,1,1,128,1,control_shuffle32_split,3,5,100,165.262413025,836,0.0050586215262,256,2909,true,1,0.165262413025 +fp16,1,1,1,128,1,shuffle32_split,3,6,100,46.0825157166,836,0.0181413706913,256,2909,true,1,0.0460825157166 +fp16,1,1,1,128,1,baseline_fused,3,7,100,75.7561159134,324,0.00427688241528,256,2909,true,1,0.0757561159134 +fp16,1,1,1,128,1,control_optimized_fused,3,8,100,59.2397689819,324,0.0054692988438,256,2909,true,1,0.0592397689819 +fp16,1,1,1,128,1,optimized_fused,3,9,100,28.8829708099,324,0.0112176826315,256,2909,true,1,0.0288829708099 +fp16,1,1,1,128,1,control_shuffle32_fused,3,10,100,159.335546494,324,0.00203344455854,256,2909,true,1,0.159335546494 +fp16,1,1,1,128,1,shuffle32_fused,3,11,100,40.348572731,324,0.0080300238167,256,2909,true,1,0.040348572731 +fp16,1,1,1,128,1,baseline_transform,3,12,100,9.28662836552,512,0.0551330342777,256,2909,true,1,0.00928662836552 +fp16,1,1,1,128,1,control_optimized_transform,3,13,100,9.49325740337,512,0.0539330156389,256,2909,true,1,0.00949325740337 +fp16,1,1,1,128,1,optimized_transform,3,14,100,9.62034285069,512,0.0532205564756,256,2909,true,1,0.00962034285069 +fp16,1,1,1,128,1,shuffle32_transform,4,0,100,9.18102860451,512,0.0557671718557,256,2909,true,1,0.00918102860451 +fp16,1,1,1,128,1,baseline_split,4,1,100,80.1885700226,836,0.0104254259649,256,2909,true,1,0.0801885700226 +fp16,1,1,1,128,1,control_optimized_split,4,2,100,64.6809148788,836,0.012924987248,256,2909,true,1,0.0646809148788 +fp16,1,1,1,128,1,optimized_split,4,3,100,34.1915440559,836,0.0244504898238,256,2909,true,1,0.0341915440559 +fp16,1,1,1,128,1,control_shuffle32_split,4,4,100,164.698505402,836,0.0050759416302,256,2909,true,1,0.164698505402 +fp16,1,1,1,128,1,shuffle32_split,4,5,100,46.1931419373,836,0.0180979246039,256,2909,true,1,0.0461931419373 +fp16,1,1,1,128,1,baseline_fused,4,6,100,75.7536029816,324,0.00427702428991,256,2909,true,1,0.0757536029816 +fp16,1,1,1,128,1,control_optimized_fused,4,7,100,59.2351961136,324,0.00546972106547,256,2909,true,1,0.0592351961136 +fp16,1,1,1,128,1,optimized_fused,4,8,100,28.8982844353,324,0.0112117382167,256,2909,true,1,0.0288982844353 +fp16,1,1,1,128,1,control_shuffle32_fused,4,9,100,159.298295975,324,0.00203392006184,256,2909,true,1,0.159298295975 +fp16,1,1,1,128,1,shuffle32_fused,4,10,100,40.3405714035,324,0.00803161652717,256,2909,true,1,0.0403405714035 +fp16,1,1,1,128,1,baseline_transform,4,11,100,9.2841142416,512,0.0551479642189,256,2909,true,1,0.0092841142416 +fp16,1,1,1,128,1,control_optimized_transform,4,12,100,9.47359979153,512,0.0540449260331,256,2909,true,1,0.00947359979153 +fp16,1,1,1,128,1,optimized_transform,4,13,100,9.71222877502,512,0.0527170448576,256,2909,true,1,0.00971222877502 +fp16,1,1,1,128,1,control_shuffle32_transform,4,14,100,9.09485757351,512,0.0562955489805,256,2909,true,1,0.00909485757351 +fp16,1,17,1,128,17,baseline_transform,0,0,100,9.46080029011,8704,0.920006736544,4352,2909,true,1,0.00946080029011 +fp16,1,17,1,128,17,control_optimized_transform,0,1,100,9.78765666485,8704,0.88928333901,4352,2909,true,1,0.00978765666485 +fp16,1,17,1,128,17,optimized_transform,0,2,100,9.82422828674,8704,0.885972897408,4352,2909,true,1,0.00982422828674 +fp16,1,17,1,128,17,control_shuffle32_transform,0,3,100,9.14811432362,8704,0.951452910632,4352,2909,true,1,0.00914811432362 +fp16,1,17,1,128,17,shuffle32_transform,0,4,100,9.16022837162,8704,0.950194650929,4352,2909,true,1,0.00916022837162 +fp16,1,17,1,128,17,baseline_split,0,5,100,80.3922271729,14212,0.176783260021,4352,2909,true,1,0.0803922271729 +fp16,1,17,1,128,17,control_optimized_split,0,6,100,64.4601154327,14212,0.220477420876,4352,2909,true,1,0.0644601154327 +fp16,1,17,1,128,17,optimized_split,0,7,100,34.6537137032,14212,0.41011477505,4352,2909,true,1,0.0346537137032 +fp16,1,17,1,128,17,control_shuffle32_split,0,8,100,163.314743042,14212,0.0870221495946,4352,2909,true,1,0.163314743042 +fp16,1,17,1,128,17,shuffle32_split,0,9,100,45.6233119965,14212,0.311507415356,4352,2909,true,1,0.0456233119965 +fp16,1,17,1,128,17,baseline_fused,0,10,100,75.7165718079,5508,0.0727449733722,4352,2909,true,1,0.0757165718079 +fp16,1,17,1,128,17,control_optimized_fused,0,11,100,59.3252563477,5508,0.092844099446,4352,2909,true,1,0.0593252563477 +fp16,1,17,1,128,17,optimized_fused,0,12,100,29.3842291832,5508,0.187447489797,4352,2909,true,1,0.0293842291832 +fp16,1,17,1,128,17,control_shuffle32_fused,0,13,100,157.979202271,5508,0.0348653488614,4352,2909,true,1,0.157979202271 +fp16,1,17,1,128,17,shuffle32_fused,0,14,100,39.8768019676,5508,0.138125419498,4352,2909,true,1,0.0398768019676 +fp16,1,17,1,128,17,control_optimized_transform,1,0,100,9.68388557434,8704,0.898812768199,4352,2909,true,1,0.00968388557434 +fp16,1,17,1,128,17,optimized_transform,1,1,100,9.91268575191,8704,0.878066774014,4352,2909,true,1,0.00991268575191 +fp16,1,17,1,128,17,control_shuffle32_transform,1,2,100,9.13531422615,8704,0.952786054702,4352,2909,true,1,0.00913531422615 +fp16,1,17,1,128,17,shuffle32_transform,1,3,100,9.26354289055,8704,0.939597312048,4352,2909,true,1,0.00926354289055 +fp16,1,17,1,128,17,baseline_split,1,4,100,80.3090286255,14212,0.176966403943,4352,2909,true,1,0.0803090286255 +fp16,1,17,1,128,17,control_optimized_split,1,5,100,64.4447994232,14212,0.22052981974,4352,2909,true,1,0.0644447994232 +fp16,1,17,1,128,17,optimized_split,1,6,100,34.7062849998,14212,0.409493554267,4352,2909,true,1,0.0347062849998 +fp16,1,17,1,128,17,control_shuffle32_split,1,7,100,163.319530487,14212,0.0870195986825,4352,2909,true,1,0.163319530487 +fp16,1,17,1,128,17,shuffle32_split,1,8,100,45.6025123596,14212,0.311649496149,4352,2909,true,1,0.0456025123596 +fp16,1,17,1,128,17,baseline_fused,1,9,100,75.7172584534,5508,0.072744313681,4352,2909,true,1,0.0757172584534 +fp16,1,17,1,128,17,control_optimized_fused,1,10,100,59.322514534,5508,0.0928483905861,4352,2909,true,1,0.059322514534 +fp16,1,17,1,128,17,optimized_fused,1,11,100,29.3702864647,5508,0.187536475227,4352,2909,true,1,0.0293702864647 +fp16,1,17,1,128,17,control_shuffle32_fused,1,12,100,157.97873497,5508,0.0348654519929,4352,2909,true,1,0.15797873497 +fp16,1,17,1,128,17,shuffle32_fused,1,13,100,39.9392008781,5508,0.137909619594,4352,2909,true,1,0.0399392008781 +fp16,1,17,1,128,17,baseline_transform,1,14,100,9.52457129955,8704,0.913846904627,4352,2909,true,1,0.00952457129955 +fp16,1,17,1,128,17,optimized_transform,2,0,100,9.83657181263,8704,0.884861124973,4352,2909,true,1,0.00983657181263 +fp16,1,17,1,128,17,control_shuffle32_transform,2,1,100,9.13782835007,8704,0.952523911213,4352,2909,true,1,0.00913782835007 +fp16,1,17,1,128,17,shuffle32_transform,2,2,100,9.16845679283,8704,0.949341879083,4352,2909,true,1,0.00916845679283 +fp16,1,17,1,128,17,baseline_split,2,3,100,80.3058338165,14212,0.176973444202,4352,2909,true,1,0.0803058338165 +fp16,1,17,1,128,17,control_optimized_split,2,4,100,64.4306278229,14212,0.220578325561,4352,2909,true,1,0.0644306278229 +fp16,1,17,1,128,17,optimized_split,2,5,100,34.650285244,14212,0.410155353698,4352,2909,true,1,0.034650285244 +fp16,1,17,1,128,17,control_shuffle32_split,2,6,100,163.936920166,14212,0.0866918811553,4352,2909,true,1,0.163936920166 +fp16,1,17,1,128,17,shuffle32_split,2,7,100,45.6294822693,14212,0.311465291588,4352,2909,true,1,0.0456294822693 +fp16,1,17,1,128,17,baseline_fused,2,8,100,75.7024002075,5508,0.0727585913379,4352,2909,true,1,0.0757024002075 +fp16,1,17,1,128,17,control_optimized_fused,2,9,100,59.3316555023,5508,0.0928340858412,4352,2909,true,1,0.0593316555023 +fp16,1,17,1,128,17,optimized_fused,2,10,100,29.3654847145,5508,0.187567140592,4352,2909,true,1,0.0293654847145 +fp16,1,17,1,128,17,control_shuffle32_fused,2,11,100,157.979431152,5508,0.0348652983482,4352,2909,true,1,0.157979431152 +fp16,1,17,1,128,17,shuffle32_fused,2,12,100,39.9232006073,5508,0.137964890495,4352,2909,true,1,0.0399232006073 +fp16,1,17,1,128,17,baseline_transform,2,13,100,9.49828565121,8704,0.916375893464,4352,2909,true,1,0.00949828565121 +fp16,1,17,1,128,17,control_optimized_transform,2,14,100,9.70217168331,8704,0.897118736311,4352,2909,true,1,0.00970217168331 +fp16,1,17,1,128,17,control_shuffle32_transform,3,0,100,9.18285667896,8704,0.947853190385,4352,2909,true,1,0.00918285667896 +fp16,1,17,1,128,17,shuffle32_transform,3,1,100,9.23634290695,8704,0.942364319697,4352,2909,true,1,0.00923634290695 +fp16,1,17,1,128,17,baseline_split,3,2,100,80.3389644623,14212,0.176900462872,4352,2909,true,1,0.0803389644623 +fp16,1,17,1,128,17,control_optimized_split,3,3,100,64.4530248642,14212,0.22050167591,4352,2909,true,1,0.0644530248642 +fp16,1,17,1,128,17,optimized_split,3,4,100,34.689142704,14212,0.409695913251,4352,2909,true,1,0.034689142704 +fp16,1,17,1,128,17,control_shuffle32_split,3,5,100,164.017829895,14212,0.0866491161912,4352,2909,true,1,0.164017829895 +fp16,1,17,1,128,17,shuffle32_split,3,6,100,46.26765728,14212,0.307169215722,4352,2909,true,1,0.04626765728 +fp16,1,17,1,128,17,baseline_fused,3,7,100,75.716342926,5508,0.072745193272,4352,2909,true,1,0.075716342926 +fp16,1,17,1,128,17,control_optimized_fused,3,8,100,59.2852592468,5508,0.0929067371886,4352,2909,true,1,0.0592852592468 +fp16,1,17,1,128,17,optimized_fused,3,9,100,29.3766856194,5508,0.187495623957,4352,2909,true,1,0.0293766856194 +fp16,1,17,1,128,17,control_shuffle32_fused,3,10,100,157.983989716,5508,0.0348642923243,4352,2909,true,1,0.157983989716 +fp16,1,17,1,128,17,shuffle32_fused,3,11,100,39.8761129379,5508,0.138127806203,4352,2909,true,1,0.0398761129379 +fp16,1,17,1,128,17,baseline_transform,3,12,100,9.65691387653,8704,0.90132314643,4352,2909,true,1,0.00965691387653 +fp16,1,17,1,128,17,control_optimized_transform,3,13,100,9.75017130375,8704,0.892702264283,4352,2909,true,1,0.00975017130375 +fp16,1,17,1,128,17,optimized_transform,3,14,100,9.84777092934,8704,0.883854840091,4352,2909,true,1,0.00984777092934 +fp16,1,17,1,128,17,shuffle32_transform,4,0,100,9.25394296646,8704,0.940572038486,4352,2909,true,1,0.00925394296646 +fp16,1,17,1,128,17,baseline_split,4,1,100,80.30834198,14212,0.176967917026,4352,2909,true,1,0.08030834198 +fp16,1,17,1,128,17,control_optimized_split,4,2,100,64.4317722321,14212,0.220574407744,4352,2909,true,1,0.0644317722321 +fp16,1,17,1,128,17,optimized_split,4,3,100,34.680685997,14212,0.409795815493,4352,2909,true,1,0.034680685997 +fp16,1,17,1,128,17,control_shuffle32_split,4,4,100,163.372573853,14212,0.0869913453945,4352,2909,true,1,0.163372573853 +fp16,1,17,1,128,17,shuffle32_split,4,5,100,45.6432008743,14212,0.311371677002,4352,2909,true,1,0.0456432008743 +fp16,1,17,1,128,17,baseline_fused,4,6,100,75.7177114487,5508,0.0727438784746,4352,2909,true,1,0.0757177114487 +fp16,1,17,1,128,17,control_optimized_fused,4,7,100,59.2761182785,5508,0.0929210643336,4352,2909,true,1,0.0592761182785 +fp16,1,17,1,128,17,optimized_fused,4,8,100,29.3460583687,5508,0.187691305279,4352,2909,true,1,0.0293460583687 +fp16,1,17,1,128,17,control_shuffle32_fused,4,9,100,157.969150543,5508,0.0348675673767,4352,2909,true,1,0.157969150543 +fp16,1,17,1,128,17,shuffle32_fused,4,10,100,39.9037718773,5508,0.13803206416,4352,2909,true,1,0.0399037718773 +fp16,1,17,1,128,17,baseline_transform,4,11,100,9.44571435452,8704,0.921476097341,4352,2909,true,1,0.00944571435452 +fp16,1,17,1,128,17,control_optimized_transform,4,12,100,9.70514237881,8704,0.896844132757,4352,2909,true,1,0.00970514237881 +fp16,1,17,1,128,17,optimized_transform,4,13,100,9.80662882328,8704,0.887562908401,4352,2909,true,1,0.00980662882328 +fp16,1,17,1,128,17,control_shuffle32_transform,4,14,100,9.27794277668,8704,0.938139004465,4352,2909,true,1,0.00927794277668 +fp16,1,257,1,128,257,baseline_transform,0,0,100,9.86308574677,131584,13.3410581007,65792,2909,true,1,0.00986308574677 +fp16,1,257,1,128,257,control_optimized_transform,0,1,100,9.92228627205,131584,13.2614597475,65792,2909,true,1,0.00992228627205 +fp16,1,257,1,128,257,optimized_transform,0,2,100,9.95405733585,131584,13.2191322152,65792,2909,true,1,0.00995405733585 +fp16,1,257,1,128,257,control_shuffle32_transform,0,3,100,10.4251432419,131584,12.6217930005,65792,2909,true,1,0.0104251432419 +fp16,1,257,1,128,257,shuffle32_transform,0,4,100,10.40548563,131584,12.6456375683,65792,2909,true,1,0.01040548563 +fp16,1,257,1,128,257,baseline_split,0,5,100,81.6013717651,214852,2.63294593403,65792,2909,true,1,0.0816013717651 +fp16,1,257,1,128,257,control_optimized_split,0,6,100,65.7120037079,214852,3.26960049727,65792,2909,true,1,0.0657120037079 +fp16,1,257,1,128,257,optimized_split,0,7,100,35.7266283035,214852,6.01377768354,65792,2909,true,1,0.0357266283035 +fp16,1,257,1,128,257,control_shuffle32_split,0,8,100,177.269477844,214852,1.21200785726,65792,2909,true,1,0.177269477844 +fp16,1,257,1,128,257,shuffle32_split,0,9,100,52.0303964615,214852,4.12935542705,65792,2909,true,1,0.0520303964615 +fp16,1,257,1,128,257,baseline_fused,0,10,100,76.6678857803,83268,1.08608707743,65792,2909,true,1,0.0766678857803 +fp16,1,257,1,128,257,control_optimized_fused,0,11,100,60.2502822876,83268,1.38203501857,65792,2909,true,1,0.0602502822876 +fp16,1,257,1,128,257,optimized_fused,0,12,100,29.657599926,83268,2.80764459052,65792,2909,true,1,0.029657599926 +fp16,1,257,1,128,257,control_shuffle32_fused,0,13,100,170.349254608,83268,0.488807539496,65792,2909,true,1,0.170349254608 +fp16,1,257,1,128,257,shuffle32_fused,0,14,100,44.9046850204,83268,1.8543276712,65792,2909,true,1,0.0449046850204 +fp16,1,257,1,128,257,control_optimized_transform,1,0,100,10.0006866455,131584,13.1574965464,65792,2909,true,1,0.0100006866455 +fp16,1,257,1,128,257,optimized_transform,1,1,100,9.96502816677,131584,13.2045788329,65792,2909,true,1,0.00996502816677 +fp16,1,257,1,128,257,control_shuffle32_transform,1,2,100,10.5995428562,131584,12.4141202866,65792,2909,true,1,0.0105995428562 +fp16,1,257,1,128,257,shuffle32_transform,1,3,100,10.4697144032,131584,12.5680601145,65792,2909,true,1,0.0104697144032 +fp16,1,257,1,128,257,baseline_split,1,4,100,81.664686203,214852,2.63090461728,65792,2909,true,1,0.081664686203 +fp16,1,257,1,128,257,control_optimized_split,1,5,100,65.6923437119,214852,3.27057900297,65792,2909,true,1,0.0656923437119 +fp16,1,257,1,128,257,optimized_split,1,6,100,35.2820563316,214852,6.08955436102,65792,2909,true,1,0.0352820563316 +fp16,1,257,1,128,257,control_shuffle32_split,1,7,100,177.276802063,214852,1.21195778297,65792,2909,true,1,0.177276802063 +fp16,1,257,1,128,257,shuffle32_split,1,8,100,52.102856636,214852,4.12361267446,65792,2909,true,1,0.052102856636 +fp16,1,257,1,128,257,baseline_fused,1,9,100,76.663312912,83268,1.08615186113,65792,2909,true,1,0.076663312912 +fp16,1,257,1,128,257,control_optimized_fused,1,10,100,60.2541732788,83268,1.38194577187,65792,2909,true,1,0.0602541732788 +fp16,1,257,1,128,257,optimized_fused,1,11,100,29.6532583237,83268,2.80805566428,65792,2909,true,1,0.0296532583237 +fp16,1,257,1,128,257,control_shuffle32_fused,1,12,100,170.345821381,83268,0.488817391147,65792,2909,true,1,0.170345821381 +fp16,1,257,1,128,257,shuffle32_fused,1,13,100,44.9026298523,83268,1.85441254274,65792,2909,true,1,0.0449026298523 +fp16,1,257,1,128,257,baseline_transform,1,14,100,9.95017111301,131584,13.2242951911,65792,2909,true,1,0.00995017111301 +fp16,1,257,1,128,257,optimized_transform,2,0,100,9.91931438446,131584,13.2654329624,65792,2909,true,1,0.00991931438446 +fp16,1,257,1,128,257,control_shuffle32_transform,2,1,100,10.5316579342,131584,12.4941391775,65792,2909,true,1,0.0105316579342 +fp16,1,257,1,128,257,shuffle32_transform,2,2,100,10.5476570129,131584,12.4751876022,65792,2909,true,1,0.0105476570129 +fp16,1,257,1,128,257,baseline_split,2,3,100,81.674747467,214852,2.63058052413,65792,2909,true,1,0.081674747467 +fp16,1,257,1,128,257,control_optimized_split,2,4,100,65.7005691528,214852,3.27016953994,65792,2909,true,1,0.0657005691528 +fp16,1,257,1,128,257,optimized_split,2,5,100,35.297601223,214852,6.08687255099,65792,2909,true,1,0.035297601223 +fp16,1,257,1,128,257,control_shuffle32_split,2,6,100,177.87771225,214852,1.20786352198,65792,2909,true,1,0.17787771225 +fp16,1,257,1,128,257,shuffle32_split,2,7,100,51.5058279037,214852,4.17141144496,65792,2909,true,1,0.0515058279037 +fp16,1,257,1,128,257,baseline_fused,2,8,100,76.6628551483,83268,1.08615834668,65792,2909,true,1,0.0766628551483 +fp16,1,257,1,128,257,control_optimized_fused,2,9,100,60.2475404739,83268,1.38209791379,65792,2909,true,1,0.0602475404739 +fp16,1,257,1,128,257,optimized_fused,2,10,100,29.6511983871,83268,2.80825074632,65792,2909,true,1,0.0296511983871 +fp16,1,257,1,128,257,control_shuffle32_fused,2,11,100,170.345134735,83268,0.488819361524,65792,2909,true,1,0.170345134735 +fp16,1,257,1,128,257,shuffle32_fused,2,12,100,44.9083423615,83268,1.85417665452,65792,2909,true,1,0.0449083423615 +fp16,1,257,1,128,257,baseline_transform,2,13,100,9.89211380482,131584,13.3019092376,65792,2909,true,1,0.00989211380482 +fp16,1,257,1,128,257,control_optimized_transform,2,14,100,9.9728000164,131584,13.1942884429,65792,2909,true,1,0.0099728000164 +fp16,1,257,1,128,257,control_shuffle32_transform,3,0,100,10.5620574951,131584,12.4581787271,65792,2909,true,1,0.0105620574951 +fp16,1,257,1,128,257,shuffle32_transform,3,1,100,10.5519998074,131584,12.4700532982,65792,2909,true,1,0.0105519998074 +fp16,1,257,1,128,257,baseline_split,3,2,100,81.6233158112,214852,2.63223807885,65792,2909,true,1,0.0816233158112 +fp16,1,257,1,128,257,control_optimized_split,3,3,100,65.7874298096,214852,3.26585185988,65792,2909,true,1,0.0657874298096 +fp16,1,257,1,128,257,optimized_split,3,4,100,35.1936006546,214852,6.10485986099,65792,2909,true,1,0.0351936006546 +fp16,1,257,1,128,257,control_shuffle32_split,3,5,100,177.879772186,214852,1.20784953432,65792,2909,true,1,0.177879772186 +fp16,1,257,1,128,257,shuffle32_split,3,6,100,52.0710897446,214852,4.12612835748,65792,2909,true,1,0.0520710897446 +fp16,1,257,1,128,257,baseline_fused,3,7,100,76.6690301895,83268,1.08607086583,65792,2909,true,1,0.0766690301895 +fp16,1,257,1,128,257,control_optimized_fused,3,8,100,60.2349710464,83268,1.38238632066,65792,2909,true,1,0.0602349710464 +fp16,1,257,1,128,257,optimized_fused,3,9,100,29.6242284775,83268,2.81080737894,65792,2909,true,1,0.0296242284775 +fp16,1,257,1,128,257,control_shuffle32_fused,3,10,100,170.307655334,83268,0.488926935413,65792,2909,true,1,0.170307655334 +fp16,1,257,1,128,257,shuffle32_fused,3,11,100,44.8895978928,83268,1.85495089974,65792,2909,true,1,0.0448895978928 +fp16,1,257,1,128,257,baseline_transform,3,12,100,9.93851482868,131584,13.2398051689,65792,2909,true,1,0.00993851482868 +fp16,1,257,1,128,257,control_optimized_transform,3,13,100,10.0422859192,131584,13.1029927906,65792,2909,true,1,0.0100422859192 +fp16,1,257,1,128,257,optimized_transform,3,14,100,10.0596570969,131584,13.0803663319,65792,2909,true,1,0.0100596570969 +fp16,1,257,1,128,257,shuffle32_transform,4,0,100,10.4500579834,131584,12.591700468,65792,2909,true,1,0.0104500579834 +fp16,1,257,1,128,257,baseline_split,4,1,100,81.6212558746,214852,2.6323045106,65792,2909,true,1,0.0816212558746 +fp16,1,257,1,128,257,control_optimized_split,4,2,100,65.7087993622,214852,3.26975994213,65792,2909,true,1,0.0657087993622 +fp16,1,257,1,128,257,optimized_split,4,3,100,35.2521157265,214852,6.09472638939,65792,2909,true,1,0.0352521157265 +fp16,1,257,1,128,257,control_shuffle32_split,4,4,100,177.267665863,214852,1.21202024607,65792,2909,true,1,0.177267665863 +fp16,1,257,1,128,257,shuffle32_split,4,5,100,51.4580583572,214852,4.17528384978,65792,2909,true,1,0.0514580583572 +fp16,1,257,1,128,257,baseline_fused,4,6,100,76.669716835,83268,1.0860611391,65792,2909,true,1,0.076669716835 +fp16,1,257,1,128,257,control_optimized_fused,4,7,100,60.2379417419,83268,1.38231814687,65792,2909,true,1,0.0602379417419 +fp16,1,257,1,128,257,optimized_fused,4,8,100,29.6349716187,83268,2.80978841726,65792,2909,true,1,0.0296349716187 +fp16,1,257,1,128,257,control_shuffle32_fused,4,9,100,170.333023071,83268,0.48885411941,65792,2909,true,1,0.170333023071 +fp16,1,257,1,128,257,shuffle32_fused,4,10,100,44.8578262329,83268,1.85626471438,65792,2909,true,1,0.0448578262329 +fp16,1,257,1,128,257,baseline_transform,4,11,100,9.96754288673,131584,13.2012474383,65792,2909,true,1,0.00996754288673 +fp16,1,257,1,128,257,control_optimized_transform,4,12,100,10.0100576878,131584,13.1451789894,65792,2909,true,1,0.0100100576878 +fp16,1,257,1,128,257,optimized_transform,4,13,100,9.96891379356,131584,13.1994320269,65792,2909,true,1,0.00996891379356 +fp16,1,257,1,128,257,control_shuffle32_transform,4,14,100,10.4326856136,131584,12.6126680007,65792,2909,true,1,0.0104326856136 +fp16,4,64,16,128,4096,baseline_transform,0,0,100,25.8546280861,2097152,81.1132147412,1048576,2909,true,1,0.0258546280861 +fp16,4,64,16,128,4096,control_optimized_transform,0,1,100,27.7497148514,2097152,75.5738216134,1048576,2909,true,1,0.0277497148514 +fp16,4,64,16,128,4096,optimized_transform,0,2,100,27.7997708321,2097152,75.4377441695,1048576,2909,true,1,0.0277997708321 +fp16,4,64,16,128,4096,control_shuffle32_transform,0,3,100,15.8299422264,2097152,132.480079207,1048576,2909,true,1,0.0158299422264 +fp16,4,64,16,128,4096,shuffle32_transform,0,4,100,15.8228576183,2097152,132.539396523,1048576,2909,true,1,0.0158228576183 +fp16,4,64,16,128,4096,baseline_split,0,5,100,526.434745789,3424256,6.50461624616,1048576,2909,true,1,0.526434745789 +fp16,4,64,16,128,4096,control_optimized_split,0,6,100,405.703773499,3424256,8.44028629675,1048576,2909,true,1,0.405703773499 +fp16,4,64,16,128,4096,optimized_split,0,7,100,168.882522583,3424256,20.2759643072,1048576,2909,true,1,0.168882522583 +fp16,4,64,16,128,4096,control_shuffle32_split,0,8,100,453.291664124,3424256,7.55420024461,1048576,2909,true,1,0.453291664124 +fp16,4,64,16,128,4096,shuffle32_split,0,9,100,105.874738693,3424256,32.342521382,1048576,2909,true,1,0.105874738693 +fp16,4,64,16,128,4096,baseline_fused,0,10,100,528.874969482,1327104,2.50929629228,1048576,2909,true,1,0.528874969482 +fp16,4,64,16,128,4096,control_optimized_fused,0,11,100,394.373931885,1327104,3.36509057193,1048576,2909,true,1,0.394373931885 +fp16,4,64,16,128,4096,optimized_fused,0,12,100,153.615083694,1327104,8.6391516255,1048576,2909,true,1,0.153615083694 +fp16,4,64,16,128,4096,control_shuffle32_fused,0,13,100,440.519294739,1327104,3.01258994975,1048576,2909,true,1,0.440519294739 +fp16,4,64,16,128,4096,shuffle32_fused,0,14,100,92.861032486,1327104,14.2912905928,1048576,2909,true,1,0.092861032486 +fp16,4,64,16,128,4096,control_optimized_transform,1,0,100,27.687997818,2097152,75.742276989,1048576,2909,true,1,0.027687997818 +fp16,4,64,16,128,4096,optimized_transform,1,1,100,27.7707433701,2097152,75.5165957229,1048576,2909,true,1,0.0277707433701 +fp16,4,64,16,128,4096,control_shuffle32_transform,1,2,100,15.6747436523,2097152,133.791789296,1048576,2909,true,1,0.0156747436523 +fp16,4,64,16,128,4096,shuffle32_transform,1,3,100,15.6534862518,2097152,133.973478257,1048576,2909,true,1,0.0156534862518 +fp16,4,64,16,128,4096,baseline_split,1,4,100,525.446166992,3424256,6.51685408536,1048576,2909,true,1,0.525446166992 +fp16,4,64,16,128,4096,control_optimized_split,1,5,100,404.191551208,3424256,8.47186436669,1048576,2909,true,1,0.404191551208 +fp16,4,64,16,128,4096,optimized_split,1,6,100,168.36183548,3424256,20.3386711142,1048576,2909,true,1,0.16836183548 +fp16,4,64,16,128,4096,control_shuffle32_split,1,7,100,453.369140625,3424256,7.55290930318,1048576,2909,true,1,0.453369140625 +fp16,4,64,16,128,4096,shuffle32_split,1,8,100,105.526857376,3424256,32.4491421913,1048576,2909,true,1,0.105526857376 +fp16,4,64,16,128,4096,baseline_fused,1,9,100,528.924560547,1327104,2.50906102494,1048576,2909,true,1,0.528924560547 +fp16,4,64,16,128,4096,control_optimized_fused,1,10,100,393.997459412,1327104,3.36830598345,1048576,2909,true,1,0.393997459412 +fp16,4,64,16,128,4096,optimized_fused,1,11,100,153.550624847,1327104,8.64277824541,1048576,2909,true,1,0.153550624847 +fp16,4,64,16,128,4096,control_shuffle32_fused,1,12,100,440.573234558,1327104,3.01222111536,1048576,2909,true,1,0.440573234558 +fp16,4,64,16,128,4096,shuffle32_fused,1,13,100,92.8482341766,1327104,14.293260521,1048576,2909,true,1,0.0928482341766 +fp16,4,64,16,128,4096,baseline_transform,1,14,100,25.8240008354,2097152,81.2094149689,1048576,2909,true,1,0.0258240008354 +fp16,4,64,16,128,4096,optimized_transform,2,0,100,27.8322291374,2097152,75.3497676972,1048576,2909,true,1,0.0278322291374 +fp16,4,64,16,128,4096,control_shuffle32_transform,2,1,100,15.85485816,2097152,132.271886562,1048576,2909,true,1,0.01585485816 +fp16,4,64,16,128,4096,shuffle32_transform,2,2,100,15.7675421238,2097152,133.00436958,1048576,2909,true,1,0.0157675421238 +fp16,4,64,16,128,4096,baseline_split,2,3,100,524.05166626,3424256,6.53419542474,1048576,2909,true,1,0.52405166626 +fp16,4,64,16,128,4096,control_optimized_split,2,4,100,402.904434204,3424256,8.49892855303,1048576,2909,true,1,0.402904434204 +fp16,4,64,16,128,4096,optimized_split,2,5,100,168.90789032,3424256,20.2729191248,1048576,2909,true,1,0.16890789032 +fp16,4,64,16,128,4096,control_shuffle32_split,2,6,100,454.52457428,3424256,7.53370927287,1048576,2909,true,1,0.45452457428 +fp16,4,64,16,128,4096,shuffle32_split,2,7,100,106.5487957,3424256,32.1379136902,1048576,2909,true,1,0.1065487957 +fp16,4,64,16,128,4096,baseline_fused,2,8,100,529.000892639,1327104,2.50869898041,1048576,2909,true,1,0.529000892639 +fp16,4,64,16,128,4096,control_optimized_fused,2,9,100,394.052772522,1327104,3.367833175,1048576,2909,true,1,0.394052772522 +fp16,4,64,16,128,4096,optimized_fused,2,10,100,153.365716934,1327104,8.65319855395,1048576,2909,true,1,0.153365716934 +fp16,4,64,16,128,4096,control_shuffle32_fused,2,11,100,441.243858337,1327104,3.00764299587,1048576,2909,true,1,0.441243858337 +fp16,4,64,16,128,4096,shuffle32_fused,2,12,100,92.9732513428,1327104,14.2740409831,1048576,2909,true,1,0.0929732513428 +fp16,4,64,16,128,4096,baseline_transform,2,13,100,25.8450269699,2097152,81.1433473233,1048576,2909,true,1,0.0258450269699 +fp16,4,64,16,128,4096,control_optimized_transform,2,14,100,27.7204585075,2097152,75.6535826934,1048576,2909,true,1,0.0277204585075 +fp16,4,64,16,128,4096,control_shuffle32_transform,3,0,100,15.6948578358,2097152,133.620324691,1048576,2909,true,1,0.0156948578358 +fp16,4,64,16,128,4096,shuffle32_transform,3,1,100,15.8226275444,2097152,132.541323754,1048576,2909,true,1,0.0158226275444 +fp16,4,64,16,128,4096,baseline_split,3,2,100,528.711090088,3424256,6.47661088295,1048576,2909,true,1,0.528711090088 +fp16,4,64,16,128,4096,control_optimized_split,3,3,100,405.521621704,3424256,8.44407749607,1048576,2909,true,1,0.405521621704 +fp16,4,64,16,128,4096,optimized_split,3,4,100,168.921833038,3424256,20.2712458088,1048576,2909,true,1,0.168921833038 +fp16,4,64,16,128,4096,control_shuffle32_split,3,5,100,451.413459778,3424256,7.58563114553,1048576,2909,true,1,0.451413459778 +fp16,4,64,16,128,4096,shuffle32_split,3,6,100,105.986051559,3424256,32.308553339,1048576,2909,true,1,0.105986051559 +fp16,4,64,16,128,4096,baseline_fused,3,7,100,528.886375427,1327104,2.50924217688,1048576,2909,true,1,0.528886375427 +fp16,4,64,16,128,4096,control_optimized_fused,3,8,100,393.691444397,1327104,3.37092415618,1048576,2909,true,1,0.393691444397 +fp16,4,64,16,128,4096,optimized_fused,3,9,100,154.040689468,1327104,8.61528213474,1048576,2909,true,1,0.154040689468 +fp16,4,64,16,128,4096,control_shuffle32_fused,3,10,100,440.946960449,1327104,3.00966809851,1048576,2909,true,1,0.440946960449 +fp16,4,64,16,128,4096,shuffle32_fused,3,11,100,92.8699398041,1327104,14.2899198901,1048576,2909,true,1,0.0928699398041 +fp16,4,64,16,128,4096,baseline_transform,3,12,100,25.8102846146,2097152,81.2525716519,1048576,2909,true,1,0.0258102846146 +fp16,4,64,16,128,4096,control_optimized_transform,3,13,100,27.7570295334,2097152,75.5539059926,1048576,2909,true,1,0.0277570295334 +fp16,4,64,16,128,4096,optimized_transform,3,14,100,27.7467417717,2097152,75.5819193928,1048576,2909,true,1,0.0277467417717 +fp16,4,64,16,128,4096,shuffle32_transform,4,0,100,15.6850290298,2097152,133.704056015,1048576,2909,true,1,0.0156850290298 +fp16,4,64,16,128,4096,baseline_split,4,1,100,527.552223206,3424256,6.49083796708,1048576,2909,true,1,0.527552223206 +fp16,4,64,16,128,4096,control_optimized_split,4,2,100,402.705612183,3424256,8.50312460619,1048576,2909,true,1,0.402705612183 +fp16,4,64,16,128,4096,optimized_split,4,3,100,168.855762482,3424256,20.2791776228,1048576,2909,true,1,0.168855762482 +fp16,4,64,16,128,4096,control_shuffle32_split,4,4,100,452.525253296,3424256,7.56699427283,1048576,2909,true,1,0.452525253296 +fp16,4,64,16,128,4096,shuffle32_split,4,5,100,105.994510651,3424256,32.3059748942,1048576,2909,true,1,0.105994510651 +fp16,4,64,16,128,4096,baseline_fused,4,6,100,528.866996765,1327104,2.50933412014,1048576,2909,true,1,0.528866996765 +fp16,4,64,16,128,4096,control_optimized_fused,4,7,100,393.097801208,1327104,3.37601481341,1048576,2909,true,1,0.393097801208 +fp16,4,64,16,128,4096,optimized_fused,4,8,100,154.000911713,1327104,8.61750742409,1048576,2909,true,1,0.154000911713 +fp16,4,64,16,128,4096,control_shuffle32_fused,4,9,100,442.571640015,1327104,2.99861961321,1048576,2909,true,1,0.442571640015 +fp16,4,64,16,128,4096,shuffle32_fused,4,10,100,92.9115390778,1327104,14.2835218658,1048576,2909,true,1,0.0929115390778 +fp16,4,64,16,128,4096,baseline_transform,4,11,100,25.7972574234,2097152,81.2936028656,1048576,2909,true,1,0.0257972574234 +fp16,4,64,16,128,4096,control_optimized_transform,4,12,100,27.6836562157,2097152,75.7541555805,1048576,2909,true,1,0.0276836562157 +fp16,4,64,16,128,4096,optimized_transform,4,13,100,27.7778291702,2097152,75.4973323203,1048576,2909,true,1,0.0277778291702 +fp16,4,64,16,128,4096,control_shuffle32_transform,4,14,100,15.6534862518,2097152,133.973478257,1048576,2909,true,1,0.0156534862518 +fp16,16,64,16,128,16384,baseline_transform,0,0,100,78.840918541,8388608,106.399166261,4194304,2909,true,1,0.078840918541 +fp16,16,64,16,128,16384,control_optimized_transform,0,1,100,87.0793151855,8388608,96.3329578572,4194304,2909,true,1,0.0870793151855 +fp16,16,64,16,128,16384,optimized_transform,0,2,100,87.1718883514,8388608,96.2306559906,4194304,2909,true,1,0.0871718883514 +fp16,16,64,16,128,16384,control_shuffle32_transform,0,3,100,26.720457077,8388608,313.939539875,4194304,2909,true,1,0.026720457077 +fp16,16,64,16,128,16384,shuffle32_transform,0,4,100,26.6870856285,8388608,314.332112422,4194304,2909,true,1,0.0266870856285 +fp16,16,64,16,128,16384,baseline_split,0,5,100,1884.31945801,13697024,7.26895004018,4194304,2909,true,1,1.88431945801 +fp16,16,64,16,128,16384,control_optimized_split,0,6,100,1453.67172241,13697024,9.42236392772,4194304,2909,true,1,1.45367172241 +fp16,16,64,16,128,16384,optimized_split,0,7,100,601.844100952,13697024,22.7584252771,4194304,2909,true,1,0.601844100952 +fp16,16,64,16,128,16384,control_shuffle32_split,0,8,100,1600.14282227,13697024,8.55987591196,4194304,2909,true,1,1.60014282227 +fp16,16,64,16,128,16384,shuffle32_split,0,9,100,288.818054199,13697024,47.4244037063,4194304,2909,true,1,0.288818054199 +fp16,16,64,16,128,16384,baseline_fused,0,10,100,1865.10177612,5308416,2.84618033609,4194304,2909,true,1,1.86510177612 +fp16,16,64,16,128,16384,control_optimized_fused,0,11,100,1421.29684448,5308416,3.73491014253,4194304,2909,true,1,1.42129684448 +fp16,16,64,16,128,16384,optimized_fused,0,12,100,566.042060852,5308416,9.37812994322,4194304,2909,true,1,0.566042060852 +fp16,16,64,16,128,16384,control_shuffle32_fused,0,13,100,1574.28039551,5308416,3.37196347941,4194304,2909,true,1,1.57428039551 +fp16,16,64,16,128,16384,shuffle32_fused,0,14,100,265.183086395,5308416,20.0179282629,4194304,2909,true,1,0.265183086395 +fp16,16,64,16,128,16384,control_optimized_transform,1,0,100,87.0966815948,8388608,96.3137498053,4194304,2909,true,1,0.0870966815948 +fp16,16,64,16,128,16384,optimized_transform,1,1,100,87.1981716156,8388608,96.2016501559,4194304,2909,true,1,0.0871981716156 +fp16,16,64,16,128,16384,control_shuffle32_transform,1,2,100,26.728913784,8388608,313.840213178,4194304,2909,true,1,0.026728913784 +fp16,16,64,16,128,16384,shuffle32_transform,1,3,100,26.6827440262,8388608,314.383258025,4194304,2909,true,1,0.0266827440262 +fp16,16,64,16,128,16384,baseline_split,1,4,100,1884.01885986,13697024,7.27010981249,4194304,2909,true,1,1.88401885986 +fp16,16,64,16,128,16384,control_optimized_split,1,5,100,1453.76235962,13697024,9.42177647493,4194304,2909,true,1,1.45376235962 +fp16,16,64,16,128,16384,optimized_split,1,6,100,601.806640625,13697024,22.7598419083,4194304,2909,true,1,0.601806640625 +fp16,16,64,16,128,16384,control_shuffle32_split,1,7,100,1600.65872192,13697024,8.55711702463,4194304,2909,true,1,1.60065872192 +fp16,16,64,16,128,16384,shuffle32_split,1,8,100,288.715877533,13697024,47.441187222,4194304,2909,true,1,0.288715877533 +fp16,16,64,16,128,16384,baseline_fused,1,9,100,1865.24902344,5308416,2.84595565166,4194304,2909,true,1,1.86524902344 +fp16,16,64,16,128,16384,control_optimized_fused,1,10,100,1421.34231567,5308416,3.73479065631,4194304,2909,true,1,1.42134231567 +fp16,16,64,16,128,16384,optimized_fused,1,11,100,566.189956665,5308416,9.3756802598,4194304,2909,true,1,0.566189956665 +fp16,16,64,16,128,16384,control_shuffle32_fused,1,12,100,1574.56924438,5308416,3.37134490524,4194304,2909,true,1,1.57456924438 +fp16,16,64,16,128,16384,shuffle32_fused,1,13,100,265.21074295,5308416,20.0158407648,4194304,2909,true,1,0.26521074295 +fp16,16,64,16,128,16384,baseline_transform,1,14,100,78.8633155823,8388608,106.368949087,4194304,2909,true,1,0.0788633155823 +fp16,16,64,16,128,16384,optimized_transform,2,0,100,87.1172523499,8388608,96.2910075069,4194304,2909,true,1,0.0871172523499 +fp16,16,64,16,128,16384,control_shuffle32_transform,2,1,100,26.7375993729,8388608,313.738263597,4194304,2909,true,1,0.0267375993729 +fp16,16,64,16,128,16384,shuffle32_transform,2,2,100,26.6102862358,8388608,315.239299783,4194304,2909,true,1,0.0266102862358 +fp16,16,64,16,128,16384,baseline_split,2,3,100,1884.1796875,13697024,7.26948925884,4194304,2909,true,1,1.8841796875 +fp16,16,64,16,128,16384,control_optimized_split,2,4,100,1453.83041382,13697024,9.421335439,4194304,2909,true,1,1.45383041382 +fp16,16,64,16,128,16384,optimized_split,2,5,100,601.754302979,13697024,22.7618214481,4194304,2909,true,1,0.601754302979 +fp16,16,64,16,128,16384,control_shuffle32_split,2,6,100,1600.25024414,13697024,8.55930130313,4194304,2909,true,1,1.60025024414 +fp16,16,64,16,128,16384,shuffle32_split,2,7,100,288.867206573,13697024,47.4163341782,4194304,2909,true,1,0.288867206573 +fp16,16,64,16,128,16384,baseline_fused,2,8,100,1865.19699097,5308416,2.84603504386,4194304,2909,true,1,1.86519699097 +fp16,16,64,16,128,16384,control_optimized_fused,2,9,100,1421.40350342,5308416,3.73462988324,4194304,2909,true,1,1.42140350342 +fp16,16,64,16,128,16384,optimized_fused,2,10,100,566.142883301,5308416,9.37645982415,4194304,2909,true,1,0.566142883301 +fp16,16,64,16,128,16384,control_shuffle32_fused,2,11,100,1574.09286499,5308416,3.37236520034,4194304,2909,true,1,1.57409286499 +fp16,16,64,16,128,16384,shuffle32_fused,2,12,100,265.231323242,5308416,20.0142876607,4194304,2909,true,1,0.265231323242 +fp16,16,64,16,128,16384,baseline_transform,2,13,100,78.8427400589,8388608,106.396708102,4194304,2909,true,1,0.0788427400589 +fp16,16,64,16,128,16384,control_optimized_transform,2,14,100,87.1046829224,8388608,96.3049025444,4194304,2909,true,1,0.0871046829224 +fp16,16,64,16,128,16384,control_shuffle32_transform,3,0,100,26.719315052,8388608,313.952958138,4194304,2909,true,1,0.026719315052 +fp16,16,64,16,128,16384,shuffle32_transform,3,1,100,26.624455452,8388608,315.071533205,4194304,2909,true,1,0.026624455452 +fp16,16,64,16,128,16384,baseline_split,3,2,100,1884.28634644,13697024,7.26907777361,4194304,2909,true,1,1.88428634644 +fp16,16,64,16,128,16384,control_optimized_split,3,3,100,1453.4954834,13697024,9.42350640676,4194304,2909,true,1,1.4534954834 +fp16,16,64,16,128,16384,optimized_split,3,4,100,601.792221069,13697024,22.7603872574,4194304,2909,true,1,0.601792221069 +fp16,16,64,16,128,16384,control_shuffle32_split,3,5,100,1600.79925537,13697024,8.55636579917,4194304,2909,true,1,1.60079925537 +fp16,16,64,16,128,16384,shuffle32_split,3,6,100,288.891201019,13697024,47.4123959182,4194304,2909,true,1,0.288891201019 +fp16,16,64,16,128,16384,baseline_fused,3,7,100,1865.25802612,5308416,2.84594191563,4194304,2909,true,1,1.86525802612 +fp16,16,64,16,128,16384,control_optimized_fused,3,8,100,1421.31408691,5308416,3.7348648331,4194304,2909,true,1,1.42131408691 +fp16,16,64,16,128,16384,optimized_fused,3,9,100,566.134643555,5308416,9.37659629283,4194304,2909,true,1,0.566134643555 +fp16,16,64,16,128,16384,control_shuffle32_fused,3,10,100,1574.43908691,5308416,3.3716236113,4194304,2909,true,1,1.57443908691 +fp16,16,64,16,128,16384,shuffle32_fused,3,11,100,265.224685669,5308416,20.0147885428,4194304,2909,true,1,0.265224685669 +fp16,16,64,16,128,16384,baseline_transform,3,12,100,78.8333749771,8388608,106.409347595,4194304,2909,true,1,0.0788333749771 +fp16,16,64,16,128,16384,control_optimized_transform,3,13,100,87.0955467224,8388608,96.3150047928,4194304,2909,true,1,0.0870955467224 +fp16,16,64,16,128,16384,optimized_transform,3,14,100,87.1311950684,8388608,96.2755990368,4194304,2909,true,1,0.0871311950684 +fp16,16,64,16,128,16384,shuffle32_transform,4,0,100,26.6612577438,8388608,314.636619195,4194304,2909,true,1,0.0266612577438 +fp16,16,64,16,128,16384,baseline_split,4,1,100,1884.17816162,13697024,7.26949514594,4194304,2909,true,1,1.88417816162 +fp16,16,64,16,128,16384,control_optimized_split,4,2,100,1453.75213623,13697024,9.42184273278,4194304,2909,true,1,1.45375213623 +fp16,16,64,16,128,16384,optimized_split,4,3,100,601.796112061,13697024,22.7602400971,4194304,2909,true,1,0.601796112061 +fp16,16,64,16,128,16384,control_shuffle32_split,4,4,100,1602.90328979,13697024,8.54513437411,4194304,2909,true,1,1.60290328979 +fp16,16,64,16,128,16384,shuffle32_split,4,5,100,288.682975769,13697024,47.446594187,4194304,2909,true,1,0.288682975769 +fp16,16,64,16,128,16384,baseline_fused,4,6,100,1865.18310547,5308416,2.84605623139,4194304,2909,true,1,1.86518310547 +fp16,16,64,16,128,16384,control_optimized_fused,4,7,100,1421.40258789,5308416,3.73463228872,4194304,2909,true,1,1.42140258789 +fp16,16,64,16,128,16384,optimized_fused,4,8,100,566.068572998,5308416,9.37769071313,4194304,2909,true,1,0.566068572998 +fp16,16,64,16,128,16384,control_shuffle32_fused,4,9,100,1574.29672241,5308416,3.37192850905,4194304,2909,true,1,1.57429672241 +fp16,16,64,16,128,16384,shuffle32_fused,4,10,100,265.193138123,5308416,20.0171695149,4194304,2909,true,1,0.265193138123 +fp16,16,64,16,128,16384,baseline_transform,4,11,100,78.7997722626,8388608,106.454723905,4194304,2909,true,1,0.0787997722626 +fp16,16,64,16,128,16384,control_optimized_transform,4,12,100,87.0868587494,8388608,96.3246133856,4194304,2909,true,1,0.0870868587494 +fp16,16,64,16,128,16384,optimized_transform,4,13,100,87.1533679962,8388608,96.2511052971,4194304,2909,true,1,0.0871533679962 +fp16,16,64,16,128,16384,control_shuffle32_transform,4,14,100,26.6713142395,8388608,314.517984553,4194304,2909,true,1,0.0266713142395 +fp16,1,1,1,256,1,baseline_transform,0,0,100,9.70377147198,1024,0.10552598059,512,2909,true,1,0.00970377147198 +fp16,1,1,1,256,1,control_optimized_transform,0,1,100,10.429713726,1024,0.0981810265265,512,2909,true,1,0.010429713726 +fp16,1,1,1,256,1,optimized_transform,0,2,100,10.4198861122,1024,0.0982736268873,512,2909,true,1,0.0104198861122 +fp16,1,1,1,256,1,control_shuffle32_transform,0,3,100,11.403657198,1024,0.0897957543115,512,2909,true,1,0.011403657198 +fp16,1,1,1,256,1,shuffle32_transform,0,4,100,11.5442287922,1024,0.0887023306999,512,2909,true,1,0.0115442287922 +fp16,1,1,1,256,1,baseline_split,0,5,100,100.562057495,1668,0.0165867728003,512,2909,true,1,0.100562057495 +fp16,1,1,1,256,1,control_optimized_split,0,6,100,66.8754291534,1668,0.0249418960164,512,2909,true,1,0.0668754291534 +fp16,1,1,1,256,1,optimized_split,0,7,100,35.4587435722,1668,0.0470405838436,512,2909,true,1,0.0354587435722 +fp16,1,1,1,256,1,control_shuffle32_split,0,8,100,303.226280212,1668,0.0055008424693,512,2909,true,1,0.303226280212 +fp16,1,1,1,256,1,shuffle32_split,0,9,100,68.1979465485,1668,0.0244582144246,512,2909,true,1,0.0681979465485 +fp16,1,1,1,256,1,baseline_fused,0,10,100,96.1254787445,644,0.00669957651615,512,2909,true,1,0.0961254787445 +fp16,1,1,1,256,1,control_optimized_fused,0,11,100,61.2612581253,644,0.0105123534793,512,2909,true,1,0.0612612581253 +fp16,1,1,1,256,1,optimized_fused,0,12,100,29.6831989288,644,0.0216957748235,512,2909,true,1,0.0296831989288 +fp16,1,1,1,256,1,control_shuffle32_fused,0,13,100,299.409370422,644,0.00215090128639,512,2909,true,1,0.299409370422 +fp16,1,1,1,256,1,shuffle32_fused,0,14,100,61.6582870483,644,0.0104446625235,512,2909,true,1,0.0616582870483 +fp16,1,1,1,256,1,control_optimized_transform,1,0,100,10.6093716621,1024,0.0965184397917,512,2909,true,1,0.0106093716621 +fp16,1,1,1,256,1,optimized_transform,1,1,100,10.533028841,1024,0.0972180002026,512,2909,true,1,0.010533028841 +fp16,1,1,1,256,1,control_shuffle32_transform,1,2,100,11.4098286629,1024,0.0897471846648,512,2909,true,1,0.0114098286629 +fp16,1,1,1,256,1,shuffle32_transform,1,3,100,11.5430855751,1024,0.088711115701,512,2909,true,1,0.0115430855751 +fp16,1,1,1,256,1,baseline_split,1,4,100,100.558853149,1668,0.016587301344,512,2909,true,1,0.100558853149 +fp16,1,1,1,256,1,control_optimized_split,1,5,100,67.3611402512,1668,0.0247620511437,512,2909,true,1,0.0673611402512 +fp16,1,1,1,256,1,optimized_split,1,6,100,35.4738283157,1668,0.047020580501,512,2909,true,1,0.0354738283157 +fp16,1,1,1,256,1,control_shuffle32_split,1,7,100,302.64913559,1668,0.00551133244359,512,2909,true,1,0.30264913559 +fp16,1,1,1,256,1,shuffle32_split,1,8,100,67.6719999313,1668,0.024648303607,512,2909,true,1,0.0676719999313 +fp16,1,1,1,256,1,baseline_fused,1,9,100,96.1469650269,644,0.00669807933948,512,2909,true,1,0.0961469650269 +fp16,1,1,1,256,1,control_optimized_fused,1,10,100,61.2674283981,644,0.010511294775,512,2909,true,1,0.0612674283981 +fp16,1,1,1,256,1,optimized_fused,1,11,100,29.6925711632,644,0.0216889267171,512,2909,true,1,0.0296925711632 +fp16,1,1,1,256,1,control_shuffle32_fused,1,12,100,299.434509277,644,0.00215072070869,512,2909,true,1,0.299434509277 +fp16,1,1,1,256,1,shuffle32_fused,1,13,100,61.6644573212,644,0.0104436174091,512,2909,true,1,0.0616644573212 +fp16,1,1,1,256,1,baseline_transform,1,14,100,9.74720001221,1024,0.105055810768,512,2909,true,1,0.00974720001221 +fp16,1,1,1,256,1,optimized_transform,2,0,100,10.6105136871,1024,0.0965080513719,512,2909,true,1,0.0106105136871 +fp16,1,1,1,256,1,control_shuffle32_transform,2,1,100,11.4793145657,1024,0.0892039323553,512,2909,true,1,0.0114793145657 +fp16,1,1,1,256,1,shuffle32_transform,2,2,100,11.6427433491,1024,0.0879517798596,512,2909,true,1,0.0116427433491 +fp16,1,1,1,256,1,baseline_split,2,3,100,100.546512604,1668,0.0165893371814,512,2909,true,1,0.100546512604 +fp16,1,1,1,256,1,control_optimized_split,2,4,100,66.9833135605,1668,0.0249017241957,512,2909,true,1,0.0669833135605 +fp16,1,1,1,256,1,optimized_split,2,5,100,35.5668568611,1668,0.0468975936365,512,2909,true,1,0.0355668568611 +fp16,1,1,1,256,1,control_shuffle32_split,2,6,100,303.306980133,1668,0.00549937887769,512,2909,true,1,0.303306980133 +fp16,1,1,1,256,1,shuffle32_split,2,7,100,67.7956581116,1668,0.0246033455012,512,2909,true,1,0.0677956581116 +fp16,1,1,1,256,1,baseline_fused,2,8,100,96.1312007904,644,0.00669917773527,512,2909,true,1,0.0961312007904 +fp16,1,1,1,256,1,control_optimized_fused,2,9,100,61.2726831436,644,0.0105103933263,512,2909,true,1,0.0612726831436 +fp16,1,1,1,256,1,optimized_fused,2,10,100,29.7462844849,644,0.0216497626898,512,2909,true,1,0.0297462844849 +fp16,1,1,1,256,1,control_shuffle32_fused,2,11,100,299.422168732,644,0.00215080934965,512,2909,true,1,0.299422168732 +fp16,1,1,1,256,1,shuffle32_fused,2,12,100,61.6594266891,644,0.0104444694766,512,2909,true,1,0.0616594266891 +fp16,1,1,1,256,1,baseline_transform,2,13,100,9.84800040722,1024,0.103980499356,512,2909,true,1,0.00984800040722 +fp16,1,1,1,256,1,control_optimized_transform,2,14,100,10.4879999161,1024,0.0976353936112,512,2909,true,1,0.0104879999161 +fp16,1,1,1,256,1,control_shuffle32_transform,3,0,100,11.3686859608,1024,0.0900719752075,512,2909,true,1,0.0113686859608 +fp16,1,1,1,256,1,shuffle32_transform,3,1,100,11.5120005608,1024,0.0889506558478,512,2909,true,1,0.0115120005608 +fp16,1,1,1,256,1,baseline_split,3,2,100,100.562286377,1668,0.0165867350484,512,2909,true,1,0.100562286377 +fp16,1,1,1,256,1,control_optimized_split,3,3,100,66.8825149536,1668,0.0249392535726,512,2909,true,1,0.0668825149536 +fp16,1,1,1,256,1,optimized_split,3,4,100,35.4436564445,1668,0.0470606073786,512,2909,true,1,0.0354436564445 +fp16,1,1,1,256,1,control_shuffle32_split,3,5,100,303.293704987,1668,0.00549961958516,512,2909,true,1,0.303293704987 +fp16,1,1,1,256,1,shuffle32_split,3,6,100,67.7008008957,1668,0.0246378178387,512,2909,true,1,0.0677008008957 +fp16,1,1,1,256,1,baseline_fused,3,7,100,96.1298274994,644,0.00669927343835,512,2909,true,1,0.0961298274994 +fp16,1,1,1,256,1,control_optimized_fused,3,8,100,61.2729120255,644,0.0105103540653,512,2909,true,1,0.0612729120255 +fp16,1,1,1,256,1,optimized_fused,3,9,100,29.7350859642,644,0.0216579161996,512,2909,true,1,0.0297350859642 +fp16,1,1,1,256,1,control_shuffle32_fused,3,10,100,299.391536713,644,0.00215102940808,512,2909,true,1,0.299391536713 +fp16,1,1,1,256,1,shuffle32_fused,3,11,100,61.6173744202,644,0.0104515975577,512,2909,true,1,0.0616173744202 +fp16,1,1,1,256,1,baseline_transform,3,12,100,9.78674292564,1024,0.104631337288,512,2909,true,1,0.00978674292564 +fp16,1,1,1,256,1,control_optimized_transform,3,13,100,10.5101716518,1024,0.0974294268373,512,2909,true,1,0.0105101716518 +fp16,1,1,1,256,1,optimized_transform,3,14,100,10.5787432194,1024,0.0967978878743,512,2909,true,1,0.0105787432194 +fp16,1,1,1,256,1,shuffle32_transform,4,0,100,11.5554285049,1024,0.0886163589314,512,2909,true,1,0.0115554285049 +fp16,1,1,1,256,1,baseline_split,4,1,100,100.561141968,1668,0.0165869238093,512,2909,true,1,0.100561141968 +fp16,1,1,1,256,1,control_optimized_split,4,2,100,66.9677686691,1668,0.0249075045077,512,2909,true,1,0.0669677686691 +fp16,1,1,1,256,1,optimized_split,4,3,100,35.4859423637,1668,0.0470045288047,512,2909,true,1,0.0354859423637 +fp16,1,1,1,256,1,control_shuffle32_split,4,4,100,302.72983551,1668,0.00550986326534,512,2909,true,1,0.30272983551 +fp16,1,1,1,256,1,shuffle32_split,4,5,100,67.723197937,1668,0.0246296697559,512,2909,true,1,0.067723197937 +fp16,1,1,1,256,1,baseline_fused,4,6,100,96.0825061798,644,0.0067025728783,512,2909,true,1,0.0960825061798 +fp16,1,1,1,256,1,control_optimized_fused,4,7,100,61.2752008438,644,0.0105099614711,512,2909,true,1,0.0612752008438 +fp16,1,1,1,256,1,optimized_fused,4,8,100,29.740114212,644,0.0216542544325,512,2909,true,1,0.029740114212 +fp16,1,1,1,256,1,control_shuffle32_fused,4,9,100,299.388332367,644,0.0021510524305,512,2909,true,1,0.299388332367 +fp16,1,1,1,256,1,shuffle32_fused,4,10,100,61.6191959381,644,0.0104512885992,512,2909,true,1,0.0616191959381 +fp16,1,1,1,256,1,baseline_transform,4,11,100,9.74605739117,1024,0.105068127439,512,2909,true,1,0.00974605739117 +fp16,1,1,1,256,1,control_optimized_transform,4,12,100,10.517257452,1024,0.0973637856326,512,2909,true,1,0.010517257452 +fp16,1,1,1,256,1,optimized_transform,4,13,100,10.5321145058,1024,0.0972264400885,512,2909,true,1,0.0105321145058 +fp16,1,1,1,256,1,control_shuffle32_transform,4,14,100,11.418056488,1024,0.0896825130505,512,2909,true,1,0.011418056488 +fp16,1,17,1,256,17,baseline_transform,0,0,100,9.95405733585,17408,1.74883461212,8704,2909,true,1,0.00995405733585 +fp16,1,17,1,256,17,control_optimized_transform,0,1,100,10.6379425526,17408,1.63640665608,8704,2909,true,1,0.0106379425526 +fp16,1,17,1,256,17,optimized_transform,0,2,100,10.7209134102,17408,1.62374224415,8704,2909,true,1,0.0107209134102 +fp16,1,17,1,256,17,control_shuffle32_transform,0,3,100,11.5266287327,17408,1.51024210146,8704,2909,true,1,0.0115266287327 +fp16,1,17,1,256,17,shuffle32_transform,0,4,100,11.6800010204,17408,1.49041082869,8704,2909,true,1,0.0116800010204 +fp16,1,17,1,256,17,baseline_split,0,5,100,101.142406464,28356,0.280357181438,8704,2909,true,1,0.101142406464 +fp16,1,17,1,256,17,control_optimized_split,0,6,100,67.2331428528,28356,0.421756276694,8704,2909,true,1,0.0672331428528 +fp16,1,17,1,256,17,optimized_split,0,7,100,36.2125706673,28356,0.783043000745,8704,2909,true,1,0.0362125706673 +fp16,1,17,1,256,17,control_shuffle32_split,0,8,100,303.226051331,28356,0.0935143925648,8704,2909,true,1,0.303226051331 +fp16,1,17,1,256,17,shuffle32_split,0,9,100,68.0018281937,28356,0.416988789173,8704,2909,true,1,0.0680018281937 +fp16,1,17,1,256,17,baseline_fused,0,10,100,96.5826320648,10948,0.113353713457,8704,2909,true,1,0.0965826320648 +fp16,1,17,1,256,17,control_optimized_fused,0,11,100,61.6287994385,10948,0.177644219906,8704,2909,true,1,0.0616287994385 +fp16,1,17,1,256,17,optimized_fused,0,12,100,30.4864001274,10948,0.359110946332,8704,2909,true,1,0.0304864001274 +fp16,1,17,1,256,17,control_shuffle32_fused,0,13,100,299.917259216,10948,0.0365034010667,8704,2909,true,1,0.299917259216 +fp16,1,17,1,256,17,shuffle32_fused,0,14,100,61.5913152695,10948,0.177752333297,8704,2909,true,1,0.0615913152695 +fp16,1,17,1,256,17,control_optimized_transform,1,0,100,10.6921136379,17408,1.62811587956,8704,2909,true,1,0.0106921136379 +fp16,1,17,1,256,17,optimized_transform,1,1,100,10.6832003593,17408,1.62947426,8704,2909,true,1,0.0106832003593 +fp16,1,17,1,256,17,control_shuffle32_transform,1,2,100,11.6041135788,17408,1.50015767097,8704,2909,true,1,0.0116041135788 +fp16,1,17,1,256,17,shuffle32_transform,1,3,100,11.7849135399,17408,1.47714278438,8704,2909,true,1,0.0117849135399 +fp16,1,17,1,256,17,baseline_split,1,4,100,101.25462532,28356,0.280046466127,8704,2909,true,1,0.10125462532 +fp16,1,17,1,256,17,control_optimized_split,1,5,100,67.2550868988,28356,0.421618665703,8704,2909,true,1,0.0672550868988 +fp16,1,17,1,256,17,optimized_split,1,6,100,36.2582850456,28356,0.782055741586,8704,2909,true,1,0.0362582850456 +fp16,1,17,1,256,17,control_shuffle32_split,1,7,100,303.209590912,28356,0.0935194692052,8704,2909,true,1,0.303209590912 +fp16,1,17,1,256,17,shuffle32_split,1,8,100,68.0155467987,28356,0.416904683335,8704,2909,true,1,0.0680155467987 +fp16,1,17,1,256,17,baseline_fused,1,9,100,96.5805721283,10948,0.113356131143,8704,2909,true,1,0.0965805721283 +fp16,1,17,1,256,17,control_optimized_fused,1,10,100,61.6774845123,10948,0.177503996581,8704,2909,true,1,0.0616774845123 +fp16,1,17,1,256,17,optimized_fused,1,11,100,30.4637718201,10948,0.359377691793,8704,2909,true,1,0.0304637718201 +fp16,1,17,1,256,17,control_shuffle32_fused,1,12,100,299.918861389,10948,0.0365032060648,8704,2909,true,1,0.299918861389 +fp16,1,17,1,256,17,shuffle32_fused,1,13,100,61.5867424011,10948,0.17776553156,8704,2909,true,1,0.0615867424011 +fp16,1,17,1,256,17,baseline_transform,1,14,100,10.0884580612,17408,1.72553624096,8704,2909,true,1,0.0100884580612 +fp16,1,17,1,256,17,optimized_transform,2,0,100,10.6726849079,17408,1.63107972832,8704,2909,true,1,0.0106726849079 +fp16,1,17,1,256,17,control_shuffle32_transform,2,1,100,11.757029295,17408,1.4806461363,8704,2909,true,1,0.011757029295 +fp16,1,17,1,256,17,shuffle32_transform,2,2,100,11.681599617,17408,1.49020686984,8704,2909,true,1,0.011681599617 +fp16,1,17,1,256,17,baseline_split,2,3,100,101.171197891,28356,0.280277397036,8704,2909,true,1,0.101171197891 +fp16,1,17,1,256,17,control_optimized_split,2,4,100,67.2546291351,28356,0.421621535419,8704,2909,true,1,0.0672546291351 +fp16,1,17,1,256,17,optimized_split,2,5,100,36.2715435028,28356,0.781769874166,8704,2909,true,1,0.0362715435028 +fp16,1,17,1,256,17,control_shuffle32_split,2,6,100,303.825359344,28356,0.0933299315804,8704,2909,true,1,0.303825359344 +fp16,1,17,1,256,17,shuffle32_split,2,7,100,68.0290317535,28356,0.416822043017,8704,2909,true,1,0.0680290317535 +fp16,1,17,1,256,17,baseline_fused,2,8,100,96.5849113464,10948,0.113351038453,8704,2909,true,1,0.0965849113464 +fp16,1,17,1,256,17,control_optimized_fused,2,9,100,61.669254303,10948,0.177527685777,8704,2909,true,1,0.061669254303 +fp16,1,17,1,256,17,optimized_fused,2,10,100,30.4770278931,10948,0.359221379408,8704,2909,true,1,0.0304770278931 +fp16,1,17,1,256,17,control_shuffle32_fused,2,11,100,299.919986725,10948,0.0365030691004,8704,2909,true,1,0.299919986725 +fp16,1,17,1,256,17,shuffle32_fused,2,12,100,61.5844535828,10948,0.177772138309,8704,2909,true,1,0.0615844535828 +fp16,1,17,1,256,17,baseline_transform,2,13,100,10.0900566578,17408,1.72526285931,8704,2909,true,1,0.0100900566578 +fp16,1,17,1,256,17,control_optimized_transform,2,14,100,10.6813716888,17408,1.629753229,8704,2909,true,1,0.0106813716888 +fp16,1,17,1,256,17,control_shuffle32_transform,3,0,100,11.6130292416,17408,1.49900595597,8704,2909,true,1,0.0116130292416 +fp16,1,17,1,256,17,shuffle32_transform,3,1,100,11.7430853844,17408,1.48240427709,8704,2909,true,1,0.0117430853844 +fp16,1,17,1,256,17,baseline_split,3,2,100,101.166858673,28356,0.280289418609,8704,2909,true,1,0.101166858673 +fp16,1,17,1,256,17,control_optimized_split,3,3,100,67.3337125778,28356,0.421126340943,8704,2909,true,1,0.0673337125778 +fp16,1,17,1,256,17,optimized_split,3,4,100,36.313598156,28356,0.780864509163,8704,2909,true,1,0.036313598156 +fp16,1,17,1,256,17,control_shuffle32_split,3,5,100,303.162746429,28356,0.0935339197641,8704,2909,true,1,0.303162746429 +fp16,1,17,1,256,17,shuffle32_split,3,6,100,68.0921125412,28356,0.416435897518,8704,2909,true,1,0.0680921125412 +fp16,1,17,1,256,17,baseline_fused,3,7,100,96.5835475922,10948,0.113352638963,8704,2909,true,1,0.0965835475922 +fp16,1,17,1,256,17,control_optimized_fused,3,8,100,61.6726827621,10948,0.177517816798,8704,2909,true,1,0.0616726827621 +fp16,1,17,1,256,17,optimized_fused,3,9,100,30.4546284676,10948,0.359485587278,8704,2909,true,1,0.0304546284676 +fp16,1,17,1,256,17,control_shuffle32_fused,3,10,100,299.90858078,10948,0.0365044573634,8704,2909,true,1,0.29990858078 +fp16,1,17,1,256,17,shuffle32_fused,3,11,100,61.5972566605,10948,0.177735188116,8704,2909,true,1,0.0615972566605 +fp16,1,17,1,256,17,baseline_transform,3,12,100,10.0271999836,17408,1.73607787104,8704,2909,true,1,0.0100271999836 +fp16,1,17,1,256,17,control_optimized_transform,3,13,100,10.6717717648,17408,1.63121929364,8704,2909,true,1,0.0106717717648 +fp16,1,17,1,256,17,optimized_transform,3,14,100,10.8628571033,17408,1.60252499268,8704,2909,true,1,0.0108628571033 +fp16,1,17,1,256,17,shuffle32_transform,4,0,100,11.6975998878,17408,1.48816852747,8704,2909,true,1,0.0116975998878 +fp16,1,17,1,256,17,baseline_split,4,1,100,101.228351593,28356,0.280119151935,8704,2909,true,1,0.101228351593 +fp16,1,17,1,256,17,control_optimized_split,4,2,100,67.2438812256,28356,0.421688925196,8704,2909,true,1,0.0672438812256 +fp16,1,17,1,256,17,optimized_split,4,3,100,36.2125706673,28356,0.783043000745,8704,2909,true,1,0.0362125706673 +fp16,1,17,1,256,17,control_shuffle32_split,4,4,100,303.180103302,28356,0.0935285650053,8704,2909,true,1,0.303180103302 +fp16,1,17,1,256,17,shuffle32_split,4,5,100,67.9970312119,28356,0.417018206452,8704,2909,true,1,0.0679970312119 +fp16,1,17,1,256,17,baseline_fused,4,6,100,96.5913105011,10948,0.11334352897,8704,2909,true,1,0.0965913105011 +fp16,1,17,1,256,17,control_optimized_fused,4,7,100,61.6322278976,10948,0.177634337967,8704,2909,true,1,0.0616322278976 +fp16,1,17,1,256,17,optimized_fused,4,8,100,30.4834294319,10948,0.359145942698,8704,2909,true,1,0.0304834294319 +fp16,1,17,1,256,17,control_shuffle32_fused,4,9,100,299.909934998,10948,0.0365042925307,8704,2909,true,1,0.299909934998 +fp16,1,17,1,256,17,shuffle32_fused,4,10,100,61.5856027603,10948,0.177768821109,8704,2909,true,1,0.0615856027603 +fp16,1,17,1,256,17,baseline_transform,4,11,100,9.92777168751,17408,1.75346498166,8704,2909,true,1,0.00992777168751 +fp16,1,17,1,256,17,control_optimized_transform,4,12,100,10.6335997581,17408,1.63707496952,8704,2909,true,1,0.0106335997581 +fp16,1,17,1,256,17,optimized_transform,4,13,100,10.6212568283,17408,1.6389774093,8704,2909,true,1,0.0106212568283 +fp16,1,17,1,256,17,control_shuffle32_transform,4,14,100,11.6388571262,17408,1.49567949939,8704,2909,true,1,0.0116388571262 +fp16,1,257,1,256,257,baseline_transform,0,0,100,10.4475426674,263168,25.1894640087,131584,2909,true,1,0.0104475426674 +fp16,1,257,1,256,257,control_optimized_transform,0,1,100,11.0000002384,263168,23.9243631178,131584,2909,true,1,0.0110000002384 +fp16,1,257,1,256,257,optimized_transform,0,2,100,10.9949719906,263168,23.9353042668,131584,2909,true,1,0.0109949719906 +fp16,1,257,1,256,257,control_shuffle32_transform,0,3,100,13.6205720901,263168,19.3213617063,131584,2909,true,1,0.0136205720901 +fp16,1,257,1,256,257,shuffle32_transform,0,4,100,13.7714290619,263168,19.109708863,131584,2909,true,1,0.0137714290619 +fp16,1,257,1,256,257,baseline_split,0,5,100,103.864908218,428676,4.12724574019,131584,2909,true,1,0.103864908218 +fp16,1,257,1,256,257,control_optimized_split,0,6,100,69.6850299835,428676,6.15162252354,131584,2909,true,1,0.0696850299835 +fp16,1,257,1,256,257,optimized_split,0,7,100,36.9250273705,428676,11.6093617399,131584,2909,true,1,0.0369250273705 +fp16,1,257,1,256,257,control_shuffle32_split,0,8,100,326.234970093,428676,1.31400995999,131584,2909,true,1,0.326234970093 +fp16,1,257,1,256,257,shuffle32_split,0,9,100,76.7753124237,428676,5.58351358617,131584,2909,true,1,0.0767753124237 +fp16,1,257,1,256,257,baseline_fused,0,10,100,98.9330291748,165508,1.67292967152,131584,2909,true,1,0.0989330291748 +fp16,1,257,1,256,257,control_optimized_fused,0,11,100,63.9682245255,165508,2.58734709659,131584,2909,true,1,0.0639682245255 +fp16,1,257,1,256,257,optimized_fused,0,12,100,30.8486866951,165508,5.36515546467,131584,2909,true,1,0.0308486866951 +fp16,1,257,1,256,257,control_shuffle32_fused,0,13,100,320.812568665,165508,0.515902480657,131584,2909,true,1,0.320812568665 +fp16,1,257,1,256,257,shuffle32_fused,0,14,100,68.0262851715,165508,2.43300070822,131584,2909,true,1,0.0680262851715 +fp16,1,257,1,256,257,control_optimized_transform,1,0,100,11.0726857185,263168,23.7673141539,131584,2909,true,1,0.0110726857185 +fp16,1,257,1,256,257,optimized_transform,1,1,100,11.096457243,263168,23.7163983277,131584,2909,true,1,0.011096457243 +fp16,1,257,1,256,257,control_shuffle32_transform,1,2,100,13.739657402,263168,19.1538982596,131584,2909,true,1,0.013739657402 +fp16,1,257,1,256,257,shuffle32_transform,1,3,100,13.7462866306,263168,19.1446611781,131584,2909,true,1,0.0137462866306 +fp16,1,257,1,256,257,baseline_split,1,4,100,103.992004395,428676,4.12220153363,131584,2909,true,1,0.103992004395 +fp16,1,257,1,256,257,control_optimized_split,1,5,100,70.1616001129,428676,6.10983785019,131584,2909,true,1,0.0701616001129 +fp16,1,257,1,256,257,optimized_split,1,6,100,37.0342874527,428676,11.5751113221,131584,2909,true,1,0.0370342874527 +fp16,1,257,1,256,257,control_shuffle32_split,1,7,100,326.234512329,428676,1.31401180378,131584,2909,true,1,0.326234512329 +fp16,1,257,1,256,257,shuffle32_split,1,8,100,76.6203403473,428676,5.59480678443,131584,2909,true,1,0.0766203403473 +fp16,1,257,1,256,257,baseline_fused,1,9,100,98.9469718933,165508,1.6726939373,131584,2909,true,1,0.0989469718933 +fp16,1,257,1,256,257,control_optimized_fused,1,10,100,63.977599144,165508,2.58696797339,131584,2909,true,1,0.063977599144 +fp16,1,257,1,256,257,optimized_fused,1,11,100,30.8678865433,165508,5.36181833401,131584,2909,true,1,0.0308678865433 +fp16,1,257,1,256,257,control_shuffle32_fused,1,12,100,320.805015564,165508,0.515914627173,131584,2909,true,1,0.320805015564 +fp16,1,257,1,256,257,shuffle32_fused,1,13,100,68.0228567123,165508,2.43312333529,131584,2909,true,1,0.0680228567123 +fp16,1,257,1,256,257,baseline_transform,1,14,100,10.4527997971,263168,25.1767952232,131584,2909,true,1,0.0104527997971 +fp16,1,257,1,256,257,optimized_transform,2,0,100,11.1723423004,263168,23.5553112251,131584,2909,true,1,0.0111723423004 +fp16,1,257,1,256,257,control_shuffle32_transform,2,1,100,13.6009132862,263168,19.3492888648,131584,2909,true,1,0.0136009132862 +fp16,1,257,1,256,257,shuffle32_transform,2,2,100,13.8228571415,263168,19.0386109982,131584,2909,true,1,0.0138228571415 +fp16,1,257,1,256,257,baseline_split,2,3,100,103.874511719,428676,4.12686416434,131584,2909,true,1,0.103874511719 +fp16,1,257,1,256,257,control_optimized_split,2,4,100,69.6656036377,428676,6.15333791162,131584,2909,true,1,0.0696656036377 +fp16,1,257,1,256,257,optimized_split,2,5,100,36.9627428055,428676,11.5975159705,131584,2909,true,1,0.0369627428055 +fp16,1,257,1,256,257,control_shuffle32_split,2,6,100,325.706520081,428676,1.31614190558,131584,2909,true,1,0.325706520081 +fp16,1,257,1,256,257,shuffle32_split,2,7,100,76.6048002243,428676,5.59594175228,131584,2909,true,1,0.0766048002243 +fp16,1,257,1,256,257,baseline_fused,2,8,100,98.8925743103,165508,1.67361403173,131584,2909,true,1,0.0988925743103 +fp16,1,257,1,256,257,control_optimized_fused,2,9,100,63.9734840393,165508,2.58713438052,131584,2909,true,1,0.0639734840393 +fp16,1,257,1,256,257,optimized_fused,2,10,100,30.8457136154,165508,5.36567258789,131584,2909,true,1,0.0308457136154 +fp16,1,257,1,256,257,control_shuffle32_fused,2,11,100,320.757713318,165508,0.515990709274,131584,2909,true,1,0.320757713318 +fp16,1,257,1,256,257,shuffle32_fused,2,12,100,68.0276584625,165508,2.43295159264,131584,2909,true,1,0.0680276584625 +fp16,1,257,1,256,257,baseline_transform,2,13,100,10.5289149284,263168,24.9947883318,131584,2909,true,1,0.0105289149284 +fp16,1,257,1,256,257,control_optimized_transform,2,14,100,10.9122288227,263168,24.1167963278,131584,2909,true,1,0.0109122288227 +fp16,1,257,1,256,257,control_shuffle32_transform,3,0,100,13.6178290844,263168,19.3252535605,131584,2909,true,1,0.0136178290844 +fp16,1,257,1,256,257,shuffle32_transform,3,1,100,13.669257164,263168,19.2525458291,131584,2909,true,1,0.013669257164 +fp16,1,257,1,256,257,baseline_split,3,2,100,103.924570084,428676,4.12487633728,131584,2909,true,1,0.103924570084 +fp16,1,257,1,256,257,control_optimized_split,3,3,100,69.7156572342,428676,6.14892001319,131584,2909,true,1,0.0697156572342 +fp16,1,257,1,256,257,optimized_split,3,4,100,36.9236564636,428676,11.609792774,131584,2909,true,1,0.0369236564636 +fp16,1,257,1,256,257,control_shuffle32_split,3,5,100,325.693702698,428676,1.31619370117,131584,2909,true,1,0.325693702698 +fp16,1,257,1,256,257,shuffle32_split,3,6,100,76.65143013,428676,5.59253753352,131584,2909,true,1,0.07665143013 +fp16,1,257,1,256,257,baseline_fused,3,7,100,98.938293457,165508,1.67284065873,131584,2909,true,1,0.098938293457 +fp16,1,257,1,256,257,control_optimized_fused,3,8,100,63.9787387848,165508,2.58692189224,131584,2909,true,1,0.0639787387848 +fp16,1,257,1,256,257,optimized_fused,3,9,100,30.8457136154,165508,5.36567258789,131584,2909,true,1,0.0308457136154 +fp16,1,257,1,256,257,control_shuffle32_fused,3,10,100,320.817604065,165508,0.515894383297,131584,2909,true,1,0.320817604065 +fp16,1,257,1,256,257,shuffle32_fused,3,11,100,68.0203390121,165508,2.43321339475,131584,2909,true,1,0.0680203390121 +fp16,1,257,1,256,257,baseline_transform,3,12,100,10.5739426613,263168,24.8883513397,131584,2909,true,1,0.0105739426613 +fp16,1,257,1,256,257,control_optimized_transform,3,13,100,10.9348571301,263168,24.0668896603,131584,2909,true,1,0.0109348571301 +fp16,1,257,1,256,257,optimized_transform,3,14,100,11.0793149471,263168,23.753093152,131584,2909,true,1,0.0110793149471 +fp16,1,257,1,256,257,shuffle32_transform,4,0,100,13.9209139347,263168,18.9045059279,131584,2909,true,1,0.0139209139347 +fp16,1,257,1,256,257,baseline_split,4,1,100,103.99269104,428676,4.12217431545,131584,2909,true,1,0.10399269104 +fp16,1,257,1,256,257,control_optimized_split,4,2,100,69.714512825,428676,6.14902095172,131584,2909,true,1,0.069714512825 +fp16,1,257,1,256,257,optimized_split,4,3,100,36.9232010841,428676,11.6099359593,131584,2909,true,1,0.0369232010841 +fp16,1,257,1,256,257,control_shuffle32_split,4,4,100,325.741233826,428676,1.31600164635,131584,2909,true,1,0.325741233826 +fp16,1,257,1,256,257,shuffle32_split,4,5,100,76.6594266891,428676,5.59195416029,131584,2909,true,1,0.0766594266891 +fp16,1,257,1,256,257,baseline_fused,4,6,100,98.9369106293,165508,1.67286403979,131584,2909,true,1,0.0989369106293 +fp16,1,257,1,256,257,control_optimized_fused,4,7,100,63.929142952,165508,2.5889288102,131584,2909,true,1,0.063929142952 +fp16,1,257,1,256,257,optimized_fused,4,8,100,30.8587431908,165508,5.36340702461,131584,2909,true,1,0.0308587431908 +fp16,1,257,1,256,257,control_shuffle32_fused,4,9,100,320.768928528,165508,0.515972668424,131584,2909,true,1,0.320768928528 +fp16,1,257,1,256,257,shuffle32_fused,4,10,100,68.038854599,165508,2.43255123819,131584,2909,true,1,0.068038854599 +fp16,1,257,1,256,257,baseline_transform,4,11,100,10.361828804,263168,25.3978332375,131584,2909,true,1,0.010361828804 +fp16,1,257,1,256,257,control_optimized_transform,4,12,100,10.9364569187,263168,24.0633691474,131584,2909,true,1,0.0109364569187 +fp16,1,257,1,256,257,optimized_transform,4,13,100,11.0009133816,263168,23.9223772492,131584,2909,true,1,0.0110009133816 +fp16,1,257,1,256,257,control_shuffle32_transform,4,14,100,13.6482286453,263168,19.2822092038,131584,2909,true,1,0.0136482286453 +fp16,4,64,16,256,4096,baseline_transform,0,0,100,49.1997671127,4194304,85.2504848324,2097152,2909,true,1,0.0491997671127 +fp16,4,64,16,256,4096,control_optimized_transform,0,1,100,37.3666286469,4194304,112.247322059,2097152,2909,true,1,0.0373666286469 +fp16,4,64,16,256,4096,optimized_transform,0,2,100,37.5190877914,4194304,111.791204075,2097152,2909,true,1,0.0375190877914 +fp16,4,64,16,256,4096,control_shuffle32_transform,0,3,100,22.646856308,4194304,185.204689912,2097152,2909,true,1,0.022646856308 +fp16,4,64,16,256,4096,shuffle32_transform,0,4,100,22.7167987823,4194304,184.634465454,2097152,2909,true,1,0.0227167987823 +fp16,4,64,16,256,4096,baseline_split,0,5,100,1142.04246521,6832128,5.98237649486,2097152,2909,true,1,1.14204246521 +fp16,4,64,16,256,4096,control_optimized_split,0,6,100,501.390609741,6832128,13.6263581074,2097152,2909,true,1,0.501390609741 +fp16,4,64,16,256,4096,optimized_split,0,7,100,188.817596436,6832128,36.1837462661,2097152,2909,true,1,0.188817596436 +fp16,4,64,16,256,4096,control_shuffle32_split,0,8,100,844.619216919,6832128,8.08900373463,2097152,2909,true,1,0.844619216919 +fp16,4,64,16,256,4096,shuffle32_split,0,9,100,149.543542862,6832128,45.6865463346,2097152,2909,true,1,0.149543542862 +fp16,4,64,16,256,4096,baseline_fused,0,10,100,1130.12916565,2637824,2.33409072182,2097152,2909,true,1,1.13012916565 +fp16,4,64,16,256,4096,control_optimized_fused,0,11,100,494.296226501,2637824,5.33652465581,2097152,2909,true,1,0.494296226501 +fp16,4,64,16,256,4096,optimized_fused,0,12,100,171.216220856,2637824,15.4063907427,2097152,2909,true,1,0.171216220856 +fp16,4,64,16,256,4096,control_shuffle32_fused,0,13,100,1018.01506042,2637824,2.59114437747,2097152,2909,true,1,1.01801506042 +fp16,4,64,16,256,4096,shuffle32_fused,0,14,100,131.713600159,2637824,20.0269675783,2097152,2909,true,1,0.131713600159 +fp16,4,64,16,256,4096,control_optimized_transform,1,0,100,37.4013710022,4194304,112.143054856,2097152,2909,true,1,0.0374013710022 +fp16,4,64,16,256,4096,optimized_transform,1,1,100,37.4692559242,4194304,111.939879684,2097152,2909,true,1,0.0374692559242 +fp16,4,64,16,256,4096,control_shuffle32_transform,1,2,100,22.6902866364,4194304,184.850198996,2097152,2909,true,1,0.0226902866364 +fp16,4,64,16,256,4096,shuffle32_transform,1,3,100,22.7414870262,4194304,184.434025584,2097152,2909,true,1,0.0227414870262 +fp16,4,64,16,256,4096,baseline_split,1,4,100,1142.25395203,6832128,5.98126886572,2097152,2909,true,1,1.14225395203 +fp16,4,64,16,256,4096,control_optimized_split,1,5,100,502.626304626,6832128,13.5928580281,2097152,2909,true,1,0.502626304626 +fp16,4,64,16,256,4096,optimized_split,1,6,100,188.892326355,6832128,36.1694311878,2097152,2909,true,1,0.188892326355 +fp16,4,64,16,256,4096,control_shuffle32_split,1,7,100,846.064910889,6832128,8.07518183543,2097152,2909,true,1,0.846064910889 +fp16,4,64,16,256,4096,shuffle32_split,1,8,100,149.427194595,6832128,45.7221191799,2097152,2909,true,1,0.149427194595 +fp16,4,64,16,256,4096,baseline_fused,1,9,100,1129.41047668,2637824,2.33557599691,2097152,2909,true,1,1.12941047668 +fp16,4,64,16,256,4096,control_optimized_fused,1,10,100,493.88710022,2637824,5.34094532703,2097152,2909,true,1,0.49388710022 +fp16,4,64,16,256,4096,optimized_fused,1,11,100,171.214618683,2637824,15.4065349109,2097152,2909,true,1,0.171214618683 +fp16,4,64,16,256,4096,control_shuffle32_fused,1,12,100,969.022369385,2637824,2.72214974942,2097152,2909,true,1,0.969022369385 +fp16,4,64,16,256,4096,shuffle32_fused,1,13,100,131.953144073,2637824,19.9906112016,2097152,2909,true,1,0.131953144073 +fp16,4,64,16,256,4096,baseline_transform,1,14,100,49.2002296448,4194304,85.2496833914,2097152,2909,true,1,0.0492002296448 +fp16,4,64,16,256,4096,optimized_transform,2,0,100,37.5067424774,4194304,111.82800006,2097152,2909,true,1,0.0375067424774 +fp16,4,64,16,256,4096,control_shuffle32_transform,2,1,100,22.6082277298,4194304,185.521131958,2097152,2909,true,1,0.0226082277298 +fp16,4,64,16,256,4096,shuffle32_transform,2,2,100,22.9119992256,4194304,183.061458701,2097152,2909,true,1,0.0229119992256 +fp16,4,64,16,256,4096,baseline_split,2,3,100,1142.17597961,6832128,5.9816771863,2097152,2909,true,1,1.14217597961 +fp16,4,64,16,256,4096,control_optimized_split,2,4,100,504.122314453,6832128,13.5525204978,2097152,2909,true,1,0.504122314453 +fp16,4,64,16,256,4096,optimized_split,2,5,100,188.802967072,6832128,36.1865499572,2097152,2909,true,1,0.188802967072 +fp16,4,64,16,256,4096,control_shuffle32_split,2,6,100,843.254623413,6832128,8.10209373338,2097152,2909,true,1,0.843254623413 +fp16,4,64,16,256,4096,shuffle32_split,2,7,100,149.795656204,6832128,45.609653665,2097152,2909,true,1,0.149795656204 +fp16,4,64,16,256,4096,baseline_fused,2,8,100,1130.53573608,2637824,2.3332513213,2097152,2909,true,1,1.13053573608 +fp16,4,64,16,256,4096,control_optimized_fused,2,9,100,496.401824951,2637824,5.31388860277,2097152,2909,true,1,0.496401824951 +fp16,4,64,16,256,4096,optimized_fused,2,10,100,171.203212738,2637824,15.4075613291,2097152,2909,true,1,0.171203212738 +fp16,4,64,16,256,4096,control_shuffle32_fused,2,11,100,969.187927246,2637824,2.72168474848,2097152,2909,true,1,0.969187927246 +fp16,4,64,16,256,4096,shuffle32_fused,2,12,100,131.672449112,2637824,20.0332265238,2097152,2909,true,1,0.131672449112 +fp16,4,64,16,256,4096,baseline_transform,2,13,100,49.2144012451,4194304,85.225135202,2097152,2909,true,1,0.0492144012451 +fp16,4,64,16,256,4096,control_optimized_transform,2,14,100,37.4260568619,4194304,112.069086398,2097152,2909,true,1,0.0374260568619 +fp16,4,64,16,256,4096,control_shuffle32_transform,3,0,100,22.2642278671,4194304,188.38757962,2097152,2909,true,1,0.0222642278671 +fp16,4,64,16,256,4096,shuffle32_transform,3,1,100,22.691886425,4194304,184.83716697,2097152,2909,true,1,0.022691886425 +fp16,4,64,16,256,4096,baseline_split,3,2,100,1141.98623657,6832128,5.98267105259,2097152,2909,true,1,1.14198623657 +fp16,4,64,16,256,4096,control_optimized_split,3,3,100,503.975791931,6832128,13.5564606661,2097152,2909,true,1,0.503975791931 +fp16,4,64,16,256,4096,optimized_split,3,4,100,188.909492493,6832128,36.1661444846,2097152,2909,true,1,0.188909492493 +fp16,4,64,16,256,4096,control_shuffle32_split,3,5,100,843.979873657,6832128,8.09513142819,2097152,2909,true,1,0.843979873657 +fp16,4,64,16,256,4096,shuffle32_split,3,6,100,149.915428162,6832128,45.573214737,2097152,2909,true,1,0.149915428162 +fp16,4,64,16,256,4096,baseline_fused,3,7,100,1130.18630981,2637824,2.33397270618,2097152,2909,true,1,1.13018630981 +fp16,4,64,16,256,4096,control_optimized_fused,3,8,100,495.21232605,2637824,5.32665255132,2097152,2909,true,1,0.49521232605 +fp16,4,64,16,256,4096,optimized_fused,3,9,100,171.342182159,2637824,15.3950648157,2097152,2909,true,1,0.171342182159 +fp16,4,64,16,256,4096,control_shuffle32_fused,3,10,100,983.836135864,2637824,2.68116193728,2097152,2909,true,1,0.983836135864 +fp16,4,64,16,256,4096,shuffle32_fused,3,11,100,131.753377914,2637824,20.0209212223,2097152,2909,true,1,0.131753377914 +fp16,4,64,16,256,4096,baseline_transform,3,12,100,49.2029714584,4194304,85.2449328908,2097152,2909,true,1,0.0492029714584 +fp16,4,64,16,256,4096,control_optimized_transform,3,13,100,37.3250293732,4194304,112.372423289,2097152,2909,true,1,0.0373250293732 +fp16,4,64,16,256,4096,optimized_transform,3,14,100,37.4438858032,4194304,112.015724598,2097152,2909,true,1,0.0374438858032 +fp16,4,64,16,256,4096,shuffle32_transform,4,0,100,22.8018283844,4194304,183.945950706,2097152,2909,true,1,0.0228018283844 +fp16,4,64,16,256,4096,baseline_split,4,1,100,1141.2688446,6832128,5.98643170915,2097152,2909,true,1,1.1412688446 +fp16,4,64,16,256,4096,control_optimized_split,4,2,100,503.415756226,6832128,13.5715418429,2097152,2909,true,1,0.503415756226 +fp16,4,64,16,256,4096,optimized_split,4,3,100,188.76411438,6832128,36.1939981148,2097152,2909,true,1,0.18876411438 +fp16,4,64,16,256,4096,control_shuffle32_split,4,4,100,839.587860107,6832128,8.13747830885,2097152,2909,true,1,0.839587860107 +fp16,4,64,16,256,4096,shuffle32_split,4,5,100,149.149713516,6832128,45.8071815153,2097152,2909,true,1,0.149149713516 +fp16,4,64,16,256,4096,baseline_fused,4,6,100,1131.09947205,2637824,2.33208843713,2097152,2909,true,1,1.13109947205 +fp16,4,64,16,256,4096,control_optimized_fused,4,7,100,494.378738403,2637824,5.33563398887,2097152,2909,true,1,0.494378738403 +fp16,4,64,16,256,4096,optimized_fused,4,8,100,171.08959198,2637824,15.417793505,2097152,2909,true,1,0.17108959198 +fp16,4,64,16,256,4096,control_shuffle32_fused,4,9,100,993.493270874,2637824,2.65510001661,2097152,2909,true,1,0.993493270874 +fp16,4,64,16,256,4096,shuffle32_fused,4,10,100,131.782627106,2637824,20.0164775732,2097152,2909,true,1,0.131782627106 +fp16,4,64,16,256,4096,baseline_transform,4,11,100,49.1718864441,4194304,85.2988222197,2097152,2909,true,1,0.0491718864441 +fp16,4,64,16,256,4096,control_optimized_transform,4,12,100,37.355427742,4194304,112.280979058,2097152,2909,true,1,0.037355427742 +fp16,4,64,16,256,4096,optimized_transform,4,13,100,37.4411416054,4194304,112.023934639,2097152,2909,true,1,0.0374411416054 +fp16,4,64,16,256,4096,control_shuffle32_transform,4,14,100,23.1156563759,4194304,181.448622172,2097152,2909,true,1,0.0231156563759 +fp16,16,64,16,256,16384,baseline_transform,0,0,100,170.492572784,16777216,98.4043804724,8388608,2909,true,1,0.170492572784 +fp16,16,64,16,256,16384,control_optimized_transform,0,1,100,120.751771927,16777216,138.939708563,8388608,2909,true,1,0.120751771927 +fp16,16,64,16,256,16384,optimized_transform,0,2,100,120.854177475,16777216,138.821978276,8388608,2909,true,1,0.120854177475 +fp16,16,64,16,256,16384,control_shuffle32_transform,0,3,100,43.3341693878,16777216,387.159053399,8388608,2909,true,1,0.0433341693878 +fp16,16,64,16,256,16384,shuffle32_transform,0,4,100,43.5010290146,16777216,385.674003122,8388608,2909,true,1,0.0435010290146 +fp16,16,64,16,256,16384,baseline_split,0,5,100,4267.23022461,27328512,6.40427409855,8388608,2909,true,1,4.26723022461 +fp16,16,64,16,256,16384,control_optimized_split,0,6,100,1870.59020996,27328512,14.6095664644,8388608,2909,true,1,1.87059020996 +fp16,16,64,16,256,16384,optimized_split,0,7,100,677.973480225,27328512,40.3091165025,8388608,2909,true,1,0.677973480225 +fp16,16,64,16,256,16384,control_shuffle32_split,0,8,100,3204.70672607,27328512,8.52761713815,8388608,2909,true,1,3.20470672607 +fp16,16,64,16,256,16384,shuffle32_split,0,9,100,436.312446594,27328512,62.6351877269,8388608,2909,true,1,0.436312446594 +fp16,16,64,16,256,16384,baseline_fused,0,10,100,4260.78735352,10551296,2.476372352,8388608,2909,true,1,4.26078735352 +fp16,16,64,16,256,16384,control_optimized_fused,0,11,100,1824.2149353,10551296,5.78402018085,8388608,2909,true,1,1.8242149353 +fp16,16,64,16,256,16384,optimized_fused,0,12,100,631.245231628,10551296,16.7150506195,8388608,2909,true,1,0.631245231628 +fp16,16,64,16,256,16384,control_shuffle32_fused,0,13,100,2906.27716064,10551296,3.63051953299,8388608,2909,true,1,2.90627716064 +fp16,16,64,16,256,16384,shuffle32_fused,0,14,100,391.83681488,10551296,26.9277811561,8388608,2909,true,1,0.39183681488 +fp16,16,64,16,256,16384,control_optimized_transform,1,0,100,120.776233673,16777216,138.911568028,8388608,2909,true,1,0.120776233673 +fp16,16,64,16,256,16384,optimized_transform,1,1,100,120.904455185,16777216,138.764249625,8388608,2909,true,1,0.120904455185 +fp16,16,64,16,256,16384,control_shuffle32_transform,1,2,100,43.3901691437,16777216,386.659382323,8388608,2909,true,1,0.0433901691437 +fp16,16,64,16,256,16384,shuffle32_transform,1,3,100,43.4893703461,16777216,385.777394947,8388608,2909,true,1,0.0434893703461 +fp16,16,64,16,256,16384,baseline_split,1,4,100,4264.66766357,27328512,6.40812231007,8388608,2909,true,1,4.26466766357 +fp16,16,64,16,256,16384,control_optimized_split,1,5,100,1870.70648193,27328512,14.6086584207,8388608,2909,true,1,1.87070648193 +fp16,16,64,16,256,16384,optimized_split,1,6,100,678.017807007,27328512,40.3064812127,8388608,2909,true,1,0.678017807007 +fp16,16,64,16,256,16384,control_shuffle32_split,1,7,100,3168.60656738,27328512,8.62477288323,8388608,2909,true,1,3.16860656738 +fp16,16,64,16,256,16384,shuffle32_split,1,8,100,436.336708069,27328512,62.6317050448,8388608,2909,true,1,0.436336708069 +fp16,16,64,16,256,16384,baseline_fused,1,9,100,4263.22265625,10551296,2.47495776101,8388608,2909,true,1,4.26322265625 +fp16,16,64,16,256,16384,control_optimized_fused,1,10,100,1823.99337769,10551296,5.78472275672,8388608,2909,true,1,1.82399337769 +fp16,16,64,16,256,16384,optimized_fused,1,11,100,631.402740479,10551296,16.7108809062,8388608,2909,true,1,0.631402740479 +fp16,16,64,16,256,16384,control_shuffle32_fused,1,12,100,2897.51525879,10551296,3.64149799315,8388608,2909,true,1,2.89751525879 +fp16,16,64,16,256,16384,shuffle32_fused,1,13,100,392.311096191,10551296,26.8952270339,8388608,2909,true,1,0.392311096191 +fp16,16,64,16,256,16384,baseline_transform,1,14,100,170.588340759,16777216,98.3491364376,8388608,2909,true,1,0.170588340759 +fp16,16,64,16,256,16384,optimized_transform,2,0,100,120.872459412,16777216,138.800981478,8388608,2909,true,1,0.120872459412 +fp16,16,64,16,256,16384,control_shuffle32_transform,2,1,100,43.3254861832,16777216,387.236647018,8388608,2909,true,1,0.0433254861832 +fp16,16,64,16,256,16384,shuffle32_transform,2,2,100,43.4836578369,16777216,385.828075065,8388608,2909,true,1,0.0434836578369 +fp16,16,64,16,256,16384,baseline_split,2,3,100,4266.66717529,27328512,6.40511923645,8388608,2909,true,1,4.26666717529 +fp16,16,64,16,256,16384,control_optimized_split,2,4,100,1870.55160522,27328512,14.6098679789,8388608,2909,true,1,1.87055160522 +fp16,16,64,16,256,16384,optimized_split,2,5,100,678.028564453,27328512,40.3058417193,8388608,2909,true,1,0.678028564453 +fp16,16,64,16,256,16384,control_shuffle32_split,2,6,100,3171.16271973,27328512,8.61782078542,8388608,2909,true,1,3.17116271973 +fp16,16,64,16,256,16384,shuffle32_split,2,7,100,436.425857544,27328512,62.6189111566,8388608,2909,true,1,0.436425857544 +fp16,16,64,16,256,16384,baseline_fused,2,8,100,4263.83850098,10551296,2.4746002921,8388608,2909,true,1,4.26383850098 +fp16,16,64,16,256,16384,control_optimized_fused,2,9,100,1824.08981323,10551296,5.78441693137,8388608,2909,true,1,1.82408981323 +fp16,16,64,16,256,16384,optimized_fused,2,10,100,631.245231628,10551296,16.7150506195,8388608,2909,true,1,0.631245231628 +fp16,16,64,16,256,16384,control_shuffle32_fused,2,11,100,2902.64373779,10551296,3.63506408403,8388608,2909,true,1,2.90264373779 +fp16,16,64,16,256,16384,shuffle32_fused,2,12,100,391.790161133,10551296,26.9309876733,8388608,2909,true,1,0.391790161133 +fp16,16,64,16,256,16384,baseline_transform,2,13,100,170.521144867,16777216,98.3878920886,8388608,2909,true,1,0.170521144867 +fp16,16,64,16,256,16384,control_optimized_transform,2,14,100,120.764570236,16777216,138.924984101,8388608,2909,true,1,0.120764570236 +fp16,16,64,16,256,16384,control_shuffle32_transform,3,0,100,43.3789682388,16777216,386.759221834,8388608,2909,true,1,0.0433789682388 +fp16,16,64,16,256,16384,shuffle32_transform,3,1,100,43.4738302231,16777216,385.915294648,8388608,2909,true,1,0.0434738302231 +fp16,16,64,16,256,16384,baseline_split,3,2,100,4266.6809082,27328512,6.40509862068,8388608,2909,true,1,4.2666809082 +fp16,16,64,16,256,16384,control_optimized_split,3,3,100,1870.69854736,27328512,14.6087203834,8388608,2909,true,1,1.87069854736 +fp16,16,64,16,256,16384,optimized_split,3,4,100,678.041610718,27328512,40.3050661907,8388608,2909,true,1,0.678041610718 +fp16,16,64,16,256,16384,control_shuffle32_split,3,5,100,3201.39312744,27328512,8.53644363941,8388608,2909,true,1,3.20139312744 +fp16,16,64,16,256,16384,shuffle32_split,3,6,100,436.478424072,27328512,62.6113697558,8388608,2909,true,1,0.436478424072 +fp16,16,64,16,256,16384,baseline_fused,3,7,100,4263.04962158,10551296,2.47505821808,8388608,2909,true,1,4.26304962158 +fp16,16,64,16,256,16384,control_optimized_fused,3,8,100,1823.881073,10551296,5.78507894852,8388608,2909,true,1,1.823881073 +fp16,16,64,16,256,16384,optimized_fused,3,9,100,631.267204285,10551296,16.7144688151,8388608,2909,true,1,0.631267204285 +fp16,16,64,16,256,16384,control_shuffle32_fused,3,10,100,2901.87133789,10551296,3.63603164008,8388608,2909,true,1,2.90187133789 +fp16,16,64,16,256,16384,shuffle32_fused,3,11,100,392.14012146,10551296,26.9069534653,8388608,2909,true,1,0.39214012146 +fp16,16,64,16,256,16384,baseline_transform,3,12,100,170.591087341,16777216,98.3475529787,8388608,2909,true,1,0.170591087341 +fp16,16,64,16,256,16384,control_optimized_transform,3,13,100,120.770511627,16777216,138.918149588,8388608,2909,true,1,0.120770511627 +fp16,16,64,16,256,16384,optimized_transform,3,14,100,120.878171921,16777216,138.794421966,8388608,2909,true,1,0.120878171921 +fp16,16,64,16,256,16384,shuffle32_transform,4,0,100,43.5181713104,16777216,385.522081806,8388608,2909,true,1,0.0435181713104 +fp16,16,64,16,256,16384,baseline_split,4,1,100,4266.64367676,27328512,6.40515451264,8388608,2909,true,1,4.26664367676 +fp16,16,64,16,256,16384,control_optimized_split,4,2,100,1870.47531128,27328512,14.610463894,8388608,2909,true,1,1.87047531128 +fp16,16,64,16,256,16384,optimized_split,4,3,100,677.900085449,27328512,40.3134806834,8388608,2909,true,1,0.677900085449 +fp16,16,64,16,256,16384,control_shuffle32_split,4,4,100,3171.36627197,27328512,8.61726765575,8388608,2909,true,1,3.17136627197 +fp16,16,64,16,256,16384,shuffle32_split,4,5,100,436.401596069,27328512,62.6223924159,8388608,2909,true,1,0.436401596069 +fp16,16,64,16,256,16384,baseline_fused,4,6,100,4263.97979736,10551296,2.47451829076,8388608,2909,true,1,4.26397979736 +fp16,16,64,16,256,16384,control_optimized_fused,4,7,100,1824.05258179,10551296,5.78453499935,8388608,2909,true,1,1.82405258179 +fp16,16,64,16,256,16384,optimized_fused,4,8,100,631.262435913,10551296,16.7145950713,8388608,2909,true,1,0.631262435913 +fp16,16,64,16,256,16384,control_shuffle32_fused,4,9,100,2914.01763916,10551296,3.62087581702,8388608,2909,true,1,2.91401763916 +fp16,16,64,16,256,16384,shuffle32_fused,4,10,100,391.924362183,10551296,26.9217660807,8388608,2909,true,1,0.391924362183 +fp16,16,64,16,256,16384,baseline_transform,4,11,100,170.486621857,16777216,98.4078153305,8388608,2909,true,1,0.170486621857 +fp16,16,64,16,256,16384,control_optimized_transform,4,12,100,120.731658936,16777216,138.962854879,8388608,2909,true,1,0.120731658936 +fp16,16,64,16,256,16384,optimized_transform,4,13,100,120.820798874,16777216,138.860329979,8388608,2909,true,1,0.120820798874 +fp16,16,64,16,256,16384,control_shuffle32_transform,4,14,100,43.3746290207,16777216,386.797913407,8388608,2909,true,1,0.0433746290207 +bf16,1,1,1,64,1,baseline_transform,0,0,100,8.926628232,256,0.0286782414756,128,2909,true,1,0.008926628232 +bf16,1,1,1,64,1,control_optimized_transform,0,1,100,9.19817149639,256,0.0278316185016,128,2909,true,1,0.00919817149639 +bf16,1,1,1,64,1,optimized_transform,0,2,100,9.15954291821,256,0.0279489929013,128,2909,true,1,0.00915954291821 +bf16,1,1,1,64,1,control_shuffle32_transform,0,3,100,7.8018283844,256,0.0328128212243,128,2909,true,1,0.0078018283844 +bf16,1,1,1,64,1,shuffle32_transform,0,4,100,8.1227427721,256,0.0315164479761,128,2909,true,1,0.0081227427721 +bf16,1,1,1,64,1,baseline_split,0,5,100,69.6027421951,420,0.00603424501326,128,2909,true,1,0.0696027421951 +bf16,1,1,1,64,1,control_optimized_split,0,6,100,64.0564537048,420,0.00655671639169,128,2909,true,1,0.0640564537048 +bf16,1,1,1,64,1,optimized_split,0,7,100,33.6646842957,420,0.0124759821394,128,2909,true,1,0.0336646842957 +bf16,1,1,1,64,1,control_shuffle32_split,0,8,100,94.5163440704,420,0.00444367589681,128,2909,true,1,0.0945163440704 +bf16,1,1,1,64,1,shuffle32_split,0,9,100,35.2964568138,420,0.0118992113632,128,2909,true,1,0.0352964568138 +bf16,1,1,1,64,1,baseline_fused,0,10,100,65.1325702667,164,0.00251794147426,128,2909,true,1,0.0651325702667 +bf16,1,1,1,64,1,control_optimized_fused,0,11,100,58.5513162613,164,0.0028009617968,128,2909,true,1,0.0585513162613 +bf16,1,1,1,64,1,optimized_fused,0,12,100,28.7702870369,164,0.00570032547085,128,2909,true,1,0.0287702870369 +bf16,1,1,1,64,1,control_shuffle32_fused,0,13,100,89.3583965302,164,0.00183530598543,128,2909,true,1,0.0893583965302 +bf16,1,1,1,64,1,shuffle32_fused,0,14,100,29.6921133995,164,0.00552335220445,128,2909,true,1,0.0296921133995 +bf16,1,1,1,64,1,control_optimized_transform,1,0,100,9.30057168007,256,0.0275251897202,128,2909,true,1,0.00930057168007 +bf16,1,1,1,64,1,optimized_transform,1,1,100,9.21691417694,256,0.027775022647,128,2909,true,1,0.00921691417694 +bf16,1,1,1,64,1,control_shuffle32_transform,1,2,100,7.84548580647,256,0.0326302291936,128,2909,true,1,0.00784548580647 +bf16,1,1,1,64,1,shuffle32_transform,1,3,100,7.84777164459,256,0.0326207249132,128,2909,true,1,0.00784777164459 +bf16,1,1,1,64,1,baseline_split,1,4,100,69.6320009232,420,0.00603170947886,128,2909,true,1,0.0696320009232 +bf16,1,1,1,64,1,control_optimized_split,1,5,100,64.1298294067,420,0.00654921436538,128,2909,true,1,0.0641298294067 +bf16,1,1,1,64,1,optimized_split,1,6,100,33.7353134155,420,0.0124498621023,128,2909,true,1,0.0337353134155 +bf16,1,1,1,64,1,control_shuffle32_split,1,7,100,94.4802284241,420,0.00444537451915,128,2909,true,1,0.0944802284241 +bf16,1,1,1,64,1,shuffle32_split,1,8,100,34.708571434,420,0.0121007573244,128,2909,true,1,0.034708571434 +bf16,1,1,1,64,1,baseline_fused,1,9,100,65.1261711121,164,0.00251818888167,128,2909,true,1,0.0651261711121 +bf16,1,1,1,64,1,control_optimized_fused,1,10,100,58.5487985611,164,0.00280108224303,128,2909,true,1,0.0585487985611 +bf16,1,1,1,64,1,optimized_fused,1,11,100,28.760228157,164,0.00570231915771,128,2909,true,1,0.028760228157 +bf16,1,1,1,64,1,control_shuffle32_fused,1,12,100,89.3583965302,164,0.00183530598543,128,2909,true,1,0.0893583965302 +bf16,1,1,1,64,1,shuffle32_fused,1,13,100,29.7028565407,164,0.005521354479,128,2909,true,1,0.0297028565407 +bf16,1,1,1,64,1,baseline_transform,1,14,100,8.94148528576,256,0.0286305900886,128,2909,true,1,0.00894148528576 +bf16,1,1,1,64,1,optimized_transform,2,0,100,9.18948531151,256,0.0278579258056,128,2909,true,1,0.00918948531151 +bf16,1,1,1,64,1,control_shuffle32_transform,2,1,100,7.87268579006,256,0.0325174923561,128,2909,true,1,0.00787268579006 +bf16,1,1,1,64,1,shuffle32_transform,2,2,100,7.86605715752,256,0.032544894459,128,2909,true,1,0.00786605715752 +bf16,1,1,1,64,1,baseline_split,2,3,100,69.6372556686,420,0.00603125433315,128,2909,true,1,0.0696372556686 +bf16,1,1,1,64,1,control_optimized_split,2,4,100,64.1686820984,420,0.0065452489636,128,2909,true,1,0.0641686820984 +bf16,1,1,1,64,1,optimized_split,2,5,100,33.7499427795,420,0.0124444655431,128,2909,true,1,0.0337499427795 +bf16,1,1,1,64,1,control_shuffle32_split,2,6,100,94.4258308411,420,0.00444793544583,128,2909,true,1,0.0944258308411 +bf16,1,1,1,64,1,shuffle32_split,2,7,100,35.3412556648,420,0.0118841278302,128,2909,true,1,0.0353412556648 +bf16,1,1,1,64,1,baseline_fused,2,8,100,65.1156568527,164,0.00251859549495,128,2909,true,1,0.0651156568527 +bf16,1,1,1,64,1,control_optimized_fused,2,9,100,58.5508584976,164,0.00280098369534,128,2909,true,1,0.0585508584976 +bf16,1,1,1,64,1,optimized_fused,2,10,100,28.737142086,164,0.00570690013325,128,2909,true,1,0.028737142086 +bf16,1,1,1,64,1,control_shuffle32_fused,2,11,100,89.3142795563,164,0.00183621253863,128,2909,true,1,0.0893142795563 +bf16,1,1,1,64,1,shuffle32_fused,2,12,100,29.6603417397,164,0.00552926872655,128,2909,true,1,0.0296603417397 +bf16,1,1,1,64,1,baseline_transform,2,13,100,8.90422821045,256,0.0287503862153,128,2909,true,1,0.00890422821045 +bf16,1,1,1,64,1,control_optimized_transform,2,14,100,9.38948571682,256,0.0272645390515,128,2909,true,1,0.00938948571682 +bf16,1,1,1,64,1,control_shuffle32_transform,3,0,100,7.86925733089,256,0.0325316594992,128,2909,true,1,0.00786925733089 +bf16,1,1,1,64,1,shuffle32_transform,3,1,100,7.88685679436,256,0.0324590653381,128,2909,true,1,0.00788685679436 +bf16,1,1,1,64,1,baseline_split,3,2,100,69.5561170578,420,0.00603828991275,128,2909,true,1,0.0695561170578 +bf16,1,1,1,64,1,control_optimized_split,3,3,100,64.0630865097,420,0.00655603753866,128,2909,true,1,0.0640630865097 +bf16,1,1,1,64,1,optimized_split,3,4,100,33.7028574944,420,0.0124618513451,128,2909,true,1,0.0337028574944 +bf16,1,1,1,64,1,control_shuffle32_split,3,5,100,94.5371437073,420,0.00444269821924,128,2909,true,1,0.0945371437073 +bf16,1,1,1,64,1,shuffle32_split,3,6,100,34.7456002235,420,0.0120878614069,128,2909,true,1,0.0347456002235 +bf16,1,1,1,64,1,baseline_fused,3,7,100,65.1220560074,164,0.00251834800765,128,2909,true,1,0.0651220560074 +bf16,1,1,1,64,1,control_optimized_fused,3,8,100,58.5508584976,164,0.00280098369534,128,2909,true,1,0.0585508584976 +bf16,1,1,1,64,1,optimized_fused,3,9,100,28.7330293655,164,0.00570771699404,128,2909,true,1,0.0287330293655 +bf16,1,1,1,64,1,control_shuffle32_fused,3,10,100,89.359998703,164,0.00183527307946,128,2909,true,1,0.089359998703 +bf16,1,1,1,64,1,shuffle32_fused,3,11,100,29.6980571747,164,0.00552224675962,128,2909,true,1,0.0296980571747 +bf16,1,1,1,64,1,baseline_transform,3,12,100,8.9771425724,256,0.0285168691413,128,2909,true,1,0.0089771425724 +bf16,1,1,1,64,1,control_optimized_transform,3,13,100,9.25051450729,256,0.0276741363735,128,2909,true,1,0.00925051450729 +bf16,1,1,1,64,1,optimized_transform,3,14,100,9.2278856039,256,0.0277419997374,128,2909,true,1,0.0092278856039 +bf16,1,1,1,64,1,shuffle32_transform,4,0,100,7.88548529148,256,0.0324647108627,128,2909,true,1,0.00788548529148 +bf16,1,1,1,64,1,baseline_split,4,1,100,69.645485878,420,0.00603054160231,128,2909,true,1,0.069645485878 +bf16,1,1,1,64,1,control_optimized_split,4,2,100,64.0822839737,420,0.00655407351231,128,2909,true,1,0.0640822839737 +bf16,1,1,1,64,1,optimized_split,4,3,100,33.7104010582,420,0.0124590626874,128,2909,true,1,0.0337104010582 +bf16,1,1,1,64,1,control_shuffle32_split,4,4,100,93.9716625214,420,0.00446943247284,128,2909,true,1,0.0939716625214 +bf16,1,1,1,64,1,shuffle32_split,4,5,100,34.8854851723,420,0.0120393911085,128,2909,true,1,0.0348854851723 +bf16,1,1,1,64,1,baseline_fused,4,6,100,65.1259422302,164,0.00251819773172,128,2909,true,1,0.0651259422302 +bf16,1,1,1,64,1,control_optimized_fused,4,7,100,58.5579395294,164,0.00280064499055,128,2909,true,1,0.0585579395294 +bf16,1,1,1,64,1,optimized_fused,4,8,100,28.7042284012,164,0.00571344394658,128,2909,true,1,0.0287042284012 +bf16,1,1,1,64,1,control_shuffle32_fused,4,9,100,89.3158817291,164,0.00183617960015,128,2909,true,1,0.0893158817291 +bf16,1,1,1,64,1,shuffle32_fused,4,10,100,29.645485878,164,0.00553203953799,128,2909,true,1,0.029645485878 +bf16,1,1,1,64,1,baseline_transform,4,11,100,8.88457119465,256,0.0288139961278,128,2909,true,1,0.00888457119465 +bf16,1,1,1,64,1,control_optimized_transform,4,12,100,9.35588538647,256,0.0273624557618,128,2909,true,1,0.00935588538647 +bf16,1,1,1,64,1,optimized_transform,4,13,100,9.18194293976,256,0.0278808092884,128,2909,true,1,0.00918194293976 +bf16,1,1,1,64,1,control_shuffle32_transform,4,14,100,7.8598856926,256,0.0325704482243,128,2909,true,1,0.0078598856926 +bf16,1,17,1,64,17,baseline_transform,0,0,100,9.19657111168,4352,0.473219849784,2176,2909,true,1,0.00919657111168 +bf16,1,17,1,64,17,control_optimized_transform,0,1,100,9.3542855978,4352,0.465241300845,2176,2909,true,1,0.0093542855978 +bf16,1,17,1,64,17,optimized_transform,0,2,100,9.46742832661,4352,0.459681325262,2176,2909,true,1,0.00946742832661 +bf16,1,17,1,64,17,control_shuffle32_transform,0,3,100,7.92411446571,4352,0.549209633307,2176,2909,true,1,0.00792411446571 +bf16,1,17,1,64,17,shuffle32_transform,0,4,100,7.94262886047,4352,0.547929416878,2176,2909,true,1,0.00794262886047 +bf16,1,17,1,64,17,baseline_split,0,5,100,70.1334857941,7140,0.101805862338,2176,2909,true,1,0.0701334857941 +bf16,1,17,1,64,17,control_optimized_split,0,6,100,64.4525718689,7140,0.110779132515,2176,2909,true,1,0.0644525718689 +bf16,1,17,1,64,17,optimized_split,0,7,100,34.2891430855,7140,0.208229175696,2176,2909,true,1,0.0342891430855 +bf16,1,17,1,64,17,control_shuffle32_split,0,8,100,94.5147418976,7140,0.0755437708092,2176,2909,true,1,0.0945147418976 +bf16,1,17,1,64,17,shuffle32_split,0,9,100,34.6591997147,7140,0.206005910661,2176,2909,true,1,0.0346591997147 +bf16,1,17,1,64,17,baseline_fused,0,10,100,65.4461717606,2788,0.0425998943101,2176,2909,true,1,0.0654461717606 +bf16,1,17,1,64,17,control_optimized_fused,0,11,100,58.8210296631,2788,0.0473980142131,2176,2909,true,1,0.0588210296631 +bf16,1,17,1,64,17,optimized_fused,0,12,100,28.9190864563,2788,0.0964069181166,2176,2909,true,1,0.0289190864563 +bf16,1,17,1,64,17,control_shuffle32_fused,0,13,100,89.0525722504,2788,0.0313073494628,2176,2909,true,1,0.0890525722504 +bf16,1,17,1,64,17,shuffle32_fused,0,14,100,29.682970047,2788,0.093925910904,2176,2909,true,1,0.029682970047 +bf16,1,17,1,64,17,control_optimized_transform,1,0,100,9.46171402931,4352,0.459958944703,2176,2909,true,1,0.00946171402931 +bf16,1,17,1,64,17,optimized_transform,1,1,100,9.4767999649,4352,0.459226744905,2176,2909,true,1,0.0094767999649 +bf16,1,17,1,64,17,control_shuffle32_transform,1,2,100,7.88982868195,4352,0.551596260887,2176,2909,true,1,0.00788982868195 +bf16,1,17,1,64,17,shuffle32_transform,1,3,100,8.01851451397,4352,0.542743920014,2176,2909,true,1,0.00801851451397 +bf16,1,17,1,64,17,baseline_split,1,4,100,70.1755428314,7140,0.101744848874,2176,2909,true,1,0.0701755428314 +bf16,1,17,1,64,17,control_optimized_split,1,5,100,64.4637727737,7140,0.110759884083,2176,2909,true,1,0.0644637727737 +bf16,1,17,1,64,17,optimized_split,1,6,100,34.2765712738,7140,0.208305549087,2176,2909,true,1,0.0342765712738 +bf16,1,17,1,64,17,control_shuffle32_split,1,7,100,93.8777160645,7140,0.0760563880261,2176,2909,true,1,0.0938777160645 +bf16,1,17,1,64,17,shuffle32_split,1,8,100,34.5284557343,7140,0.206785963871,2176,2909,true,1,0.0345284557343 +bf16,1,17,1,64,17,baseline_fused,1,9,100,65.4454851151,2788,0.0426003412626,2176,2909,true,1,0.0654454851151 +bf16,1,17,1,64,17,control_optimized_fused,1,10,100,58.8125753403,2788,0.0474048276898,2176,2909,true,1,0.0588125753403 +bf16,1,17,1,64,17,optimized_fused,1,11,100,28.9133691788,2788,0.0964259814469,2176,2909,true,1,0.0289133691788 +bf16,1,17,1,64,17,control_shuffle32_fused,1,12,100,89.1037750244,2788,0.031289358944,2176,2909,true,1,0.0891037750244 +bf16,1,17,1,64,17,shuffle32_fused,1,13,100,29.7117733955,2788,0.0938348567379,2176,2909,true,1,0.0297117733955 +bf16,1,17,1,64,17,baseline_transform,1,14,100,9.22811448574,4352,0.471602298251,2176,2909,true,1,0.00922811448574 +bf16,1,17,1,64,17,optimized_transform,2,0,100,9.48022902012,4352,0.459060639861,2176,2909,true,1,0.00948022902012 +bf16,1,17,1,64,17,control_shuffle32_transform,2,1,100,7.93485701084,4352,0.54846609007,2176,2909,true,1,0.00793485701084 +bf16,1,17,1,64,17,shuffle32_transform,2,2,100,7.97325670719,4352,0.545824643533,2176,2909,true,1,0.00797325670719 +bf16,1,17,1,64,17,baseline_split,2,3,100,70.1830863953,7140,0.101733912923,2176,2909,true,1,0.0701830863953 +bf16,1,17,1,64,17,control_optimized_split,2,4,100,64.4630861282,7140,0.110761063871,2176,2909,true,1,0.0644630861282 +bf16,1,17,1,64,17,optimized_split,2,5,100,34.2852568626,7140,0.208252778406,2176,2909,true,1,0.0342852568626 +bf16,1,17,1,64,17,control_shuffle32_split,2,6,100,93.9140605927,7140,0.0760269543766,2176,2909,true,1,0.0939140605927 +bf16,1,17,1,64,17,shuffle32_split,2,7,100,34.5664000511,7140,0.206558970256,2176,2909,true,1,0.0345664000511 +bf16,1,17,1,64,17,baseline_fused,2,8,100,65.4313135147,2788,0.0426095679613,2176,2909,true,1,0.0654313135147 +bf16,1,17,1,64,17,control_optimized_fused,2,9,100,58.8390827179,2788,0.0473834715162,2176,2909,true,1,0.0588390827179 +bf16,1,17,1,64,17,optimized_fused,2,10,100,28.9108586311,2788,0.0964343548413,2176,2909,true,1,0.0289108586311 +bf16,1,17,1,64,17,control_shuffle32_fused,2,11,100,89.0982913971,2788,0.0312912846732,2176,2909,true,1,0.0890982913971 +bf16,1,17,1,64,17,shuffle32_fused,2,12,100,29.7056007385,2788,0.0938543550942,2176,2909,true,1,0.0297056007385 +bf16,1,17,1,64,17,baseline_transform,2,13,100,9.28640007973,4352,0.468642311621,2176,2909,true,1,0.00928640007973 +bf16,1,17,1,64,17,control_optimized_transform,2,14,100,9.39337134361,4352,0.463305435376,2176,2909,true,1,0.00939337134361 +bf16,1,17,1,64,17,control_shuffle32_transform,3,0,100,7.97577142715,4352,0.545652547813,2176,2909,true,1,0.00797577142715 +bf16,1,17,1,64,17,shuffle32_transform,3,1,100,8.00639986992,4352,0.543565156713,2176,2909,true,1,0.00800639986992 +bf16,1,17,1,64,17,baseline_split,3,2,100,70.1954317093,7140,0.101716020917,2176,2909,true,1,0.0701954317093 +bf16,1,17,1,64,17,control_optimized_split,3,3,100,64.4591999054,7140,0.110767741618,2176,2909,true,1,0.0644591999054 +bf16,1,17,1,64,17,optimized_split,3,4,100,34.2875432968,7140,0.208238891255,2176,2909,true,1,0.0342875432968 +bf16,1,17,1,64,17,control_shuffle32_split,3,5,100,93.8719940186,7140,0.0760610241068,2176,2909,true,1,0.0938719940186 +bf16,1,17,1,64,17,shuffle32_split,3,6,100,34.5789718628,7140,0.206483872,2176,2909,true,1,0.0345789718628 +bf16,1,17,1,64,17,baseline_fused,3,7,100,65.4349708557,2788,0.042607186395,2176,2909,true,1,0.0654349708557 +bf16,1,17,1,64,17,control_optimized_fused,3,8,100,58.8121128082,2788,0.0474052005085,2176,2909,true,1,0.0588121128082 +bf16,1,17,1,64,17,optimized_fused,3,9,100,28.8761162758,2788,0.0965503800225,2176,2909,true,1,0.0288761162758 +bf16,1,17,1,64,17,control_shuffle32_fused,3,10,100,89.0969085693,2788,0.0312917703293,2176,2909,true,1,0.0890969085693 +bf16,1,17,1,64,17,shuffle32_fused,3,11,100,29.7424006462,2788,0.0937382302513,2176,2909,true,1,0.0297424006462 +bf16,1,17,1,64,17,baseline_transform,3,12,100,9.20205712318,4352,0.472937729221,2176,2909,true,1,0.00920205712318 +bf16,1,17,1,64,17,control_optimized_transform,3,13,100,9.48320031166,4352,0.458916806244,2176,2909,true,1,0.00948320031166 +bf16,1,17,1,64,17,optimized_transform,3,14,100,9.49440002441,4352,0.458375462252,2176,2909,true,1,0.00949440002441 +bf16,1,17,1,64,17,shuffle32_transform,4,0,100,7.98468589783,4352,0.545043356206,2176,2909,true,1,0.00798468589783 +bf16,1,17,1,64,17,baseline_split,4,1,100,70.1679992676,7140,0.101755787175,2176,2909,true,1,0.0701679992676 +bf16,1,17,1,64,17,control_optimized_split,4,2,100,64.4358873367,7140,0.11080781681,2176,2909,true,1,0.0644358873367 +bf16,1,17,1,64,17,optimized_split,4,3,100,34.2651438713,7140,0.208375018848,2176,2909,true,1,0.0342651438713 +bf16,1,17,1,64,17,control_shuffle32_split,4,4,100,93.8363456726,7140,0.0760899196236,2176,2909,true,1,0.0938363456726 +bf16,1,17,1,64,17,shuffle32_split,4,5,100,34.5300579071,7140,0.206776369133,2176,2909,true,1,0.0345300579071 +bf16,1,17,1,64,17,baseline_fused,4,6,100,65.4416036606,2788,0.0426028679624,2176,2909,true,1,0.0654416036606 +bf16,1,17,1,64,17,control_optimized_fused,4,7,100,58.8041114807,2788,0.0474116508148,2176,2909,true,1,0.0588041114807 +bf16,1,17,1,64,17,optimized_fused,4,8,100,28.9419436455,2788,0.0963307797898,2176,2909,true,1,0.0289419436455 +bf16,1,17,1,64,17,control_shuffle32_fused,4,9,100,89.0994262695,2788,0.0312908861115,2176,2909,true,1,0.0890994262695 +bf16,1,17,1,64,17,shuffle32_fused,4,10,100,29.6912002563,2788,0.0938998752468,2176,2909,true,1,0.0296912002563 +bf16,1,17,1,64,17,baseline_transform,4,11,100,9.19588565826,4352,0.473255123185,2176,2909,true,1,0.00919588565826 +bf16,1,17,1,64,17,control_optimized_transform,4,12,100,9.33577120304,4352,0.46616395211,2176,2909,true,1,0.00933577120304 +bf16,1,17,1,64,17,optimized_transform,4,13,100,9.50788557529,4352,0.457725323421,2176,2909,true,1,0.00950788557529 +bf16,1,17,1,64,17,control_shuffle32_transform,4,14,100,7.94628560543,4352,0.547677269116,2176,2909,true,1,0.00794628560543 +bf16,1,257,1,64,257,baseline_transform,0,0,100,9.47702825069,65792,6.94226061795,32896,2909,true,1,0.00947702825069 +bf16,1,257,1,64,257,control_optimized_transform,0,1,100,9.56045687199,65792,6.88167949304,32896,2909,true,1,0.00956045687199 +bf16,1,257,1,64,257,optimized_transform,0,2,100,9.58057105541,65792,6.86723156892,32896,2909,true,1,0.00958057105541 +bf16,1,257,1,64,257,control_shuffle32_transform,0,3,100,8.92297148705,65792,7.37332850335,32896,2909,true,1,0.00892297148705 +bf16,1,257,1,64,257,shuffle32_transform,0,4,100,8.79245698452,65792,7.48277758035,32896,2909,true,1,0.00879245698452 +bf16,1,257,1,64,257,baseline_split,0,5,100,70.8566856384,107940,1.52335660393,32896,2909,true,1,0.0708566856384 +bf16,1,257,1,64,257,control_optimized_split,0,6,100,64.5094871521,107940,1.67324225886,32896,2909,true,1,0.0645094871521 +bf16,1,257,1,64,257,optimized_split,0,7,100,35.2004575729,107940,3.0664374114,32896,2909,true,1,0.0352004575729 +bf16,1,257,1,64,257,control_shuffle32_split,0,8,100,102.971429825,107940,1.04825192953,32896,2909,true,1,0.102971429825 +bf16,1,257,1,64,257,shuffle32_split,0,9,100,39.9942827225,107940,2.69888575697,32896,2909,true,1,0.0399942827225 +bf16,1,257,1,64,257,baseline_fused,0,10,100,65.9030866623,42148,0.639545158423,32896,2909,true,1,0.0659030866623 +bf16,1,257,1,64,257,control_optimized_fused,0,11,100,59.1714286804,42148,0.71230323384,32896,2909,true,1,0.0591714286804 +bf16,1,257,1,64,257,optimized_fused,0,12,100,29.1060566902,42148,1.44808348477,32896,2909,true,1,0.0291060566902 +bf16,1,257,1,64,257,control_shuffle32_fused,0,13,100,95.9652614594,42148,0.439200595706,32896,2909,true,1,0.0959652614594 +bf16,1,257,1,64,257,shuffle32_fused,0,14,100,33.2690286636,42148,1.26688399671,32896,2909,true,1,0.0332690286636 +bf16,1,257,1,64,257,control_optimized_transform,1,0,100,9.53257143497,65792,6.90181032986,32896,2909,true,1,0.00953257143497 +bf16,1,257,1,64,257,optimized_transform,1,1,100,9.66742873192,65792,6.80553245588,32896,2909,true,1,0.00966742873192 +bf16,1,257,1,64,257,control_shuffle32_transform,1,2,100,8.98514270782,65792,7.32230996651,32896,2909,true,1,0.00898514270782 +bf16,1,257,1,64,257,shuffle32_transform,1,3,100,8.77120018005,65792,7.50091192191,32896,2909,true,1,0.00877120018005 +bf16,1,257,1,64,257,baseline_split,1,4,100,70.8720016479,107940,1.52302739432,32896,2909,true,1,0.0708720016479 +bf16,1,257,1,64,257,control_optimized_split,1,5,100,64.5506286621,107940,1.67217581358,32896,2909,true,1,0.0645506286621 +bf16,1,257,1,64,257,optimized_split,1,6,100,34.6496009827,107940,3.11518738856,32896,2909,true,1,0.0346496009827 +bf16,1,257,1,64,257,control_shuffle32_split,1,7,100,102.975091934,107940,1.04821465048,32896,2909,true,1,0.102975091934 +bf16,1,257,1,64,257,shuffle32_split,1,8,100,39.4144010544,107940,2.73859292828,32896,2909,true,1,0.0394144010544 +bf16,1,257,1,64,257,baseline_fused,1,9,100,65.9179401398,42148,0.639401047888,32896,2909,true,1,0.0659179401398 +bf16,1,257,1,64,257,control_optimized_fused,1,10,100,59.1771459579,42148,0.712234416137,32896,2909,true,1,0.0591771459579 +bf16,1,257,1,64,257,optimized_fused,1,11,100,29.1197729111,42148,1.44740139728,32896,2909,true,1,0.0291197729111 +bf16,1,257,1,64,257,control_shuffle32_fused,1,12,100,95.9680080414,42148,0.439188025887,32896,2909,true,1,0.0959680080414 +bf16,1,257,1,64,257,shuffle32_fused,1,13,100,33.290514946,42148,1.26606632755,32896,2909,true,1,0.033290514946 +bf16,1,257,1,64,257,baseline_transform,1,14,100,9.49257194996,65792,6.93089294944,32896,2909,true,1,0.00949257194996 +bf16,1,257,1,64,257,optimized_transform,2,0,100,9.64617133141,65792,6.82052990141,32896,2909,true,1,0.00964617133141 +bf16,1,257,1,64,257,control_shuffle32_transform,2,1,100,8.91428530216,65792,7.38051316173,32896,2909,true,1,0.00891428530216 +bf16,1,257,1,64,257,shuffle32_transform,2,2,100,8.83017122746,65792,7.45081814444,32896,2909,true,1,0.00883017122746 +bf16,1,257,1,64,257,baseline_split,2,3,100,70.7986307144,107940,1.52460575735,32896,2909,true,1,0.0707986307144 +bf16,1,257,1,64,257,control_optimized_split,2,4,100,64.5968008041,107940,1.67098058505,32896,2909,true,1,0.0645968008041 +bf16,1,257,1,64,257,optimized_split,2,5,100,34.5810294151,107940,3.1213645697,32896,2909,true,1,0.0345810294151 +bf16,1,257,1,64,257,control_shuffle32_split,2,6,100,103.041133881,107940,1.04754282038,32896,2909,true,1,0.103041133881 +bf16,1,257,1,64,257,shuffle32_split,2,7,100,39.9334859848,107940,2.70299467572,32896,2909,true,1,0.0399334859848 +bf16,1,257,1,64,257,baseline_fused,2,8,100,65.9069681168,42148,0.639507493735,32896,2909,true,1,0.0659069681168 +bf16,1,257,1,64,257,control_optimized_fused,2,9,100,59.1286897659,42148,0.71281809502,32896,2909,true,1,0.0591286897659 +bf16,1,257,1,64,257,optimized_fused,2,10,100,29.1444563866,42148,1.44617554162,32896,2909,true,1,0.0291444563866 +bf16,1,257,1,64,257,control_shuffle32_fused,2,11,100,95.9673118591,42148,0.439191211919,32896,2909,true,1,0.0959673118591 +bf16,1,257,1,64,257,shuffle32_fused,2,12,100,33.2815980911,42148,1.26640553391,32896,2909,true,1,0.0332815980911 +bf16,1,257,1,64,257,baseline_transform,2,13,100,9.48159992695,65792,6.93891331705,32896,2909,true,1,0.00948159992695 +bf16,1,257,1,64,257,control_optimized_transform,2,14,100,9.55817103386,65792,6.88332524778,32896,2909,true,1,0.00955817103386 +bf16,1,257,1,64,257,control_shuffle32_transform,3,0,100,8.9163428545,65792,7.37881002039,32896,2909,true,1,0.0089163428545 +bf16,1,257,1,64,257,shuffle32_transform,3,1,100,8.92297148705,65792,7.37332850335,32896,2909,true,1,0.00892297148705 +bf16,1,257,1,64,257,baseline_split,3,2,100,70.8493709564,107940,1.52351387942,32896,2909,true,1,0.0708493709564 +bf16,1,257,1,64,257,control_optimized_split,3,3,100,65.3028583527,107940,1.65291386507,32896,2909,true,1,0.0653028583527 +bf16,1,257,1,64,257,optimized_split,3,4,100,34.6265149117,107940,3.11726433559,32896,2909,true,1,0.0346265149117 +bf16,1,257,1,64,257,control_shuffle32_split,3,5,100,102.357940674,107940,1.05453469745,32896,2909,true,1,0.102357940674 +bf16,1,257,1,64,257,shuffle32_split,3,6,100,39.3366861343,107940,2.74400338736,32896,2909,true,1,0.0393366861343 +bf16,1,257,1,64,257,baseline_fused,3,7,100,65.910859108,42148,0.639469740957,32896,2909,true,1,0.065910859108 +bf16,1,257,1,64,257,control_optimized_fused,3,8,100,59.1842269897,42148,0.712149201633,32896,2909,true,1,0.0591842269897 +bf16,1,257,1,64,257,optimized_fused,3,9,100,29.1312003136,42148,1.44683361984,32896,2909,true,1,0.0291312003136 +bf16,1,257,1,64,257,control_shuffle32_fused,3,10,100,95.9855937958,42148,0.439107561179,32896,2909,true,1,0.0959855937958 +bf16,1,257,1,64,257,shuffle32_fused,3,11,100,33.2671999931,42148,1.26695363628,32896,2909,true,1,0.0332671999931 +bf16,1,257,1,64,257,baseline_transform,3,12,100,9.60365712643,65792,6.85072354561,32896,2909,true,1,0.00960365712643 +bf16,1,257,1,64,257,control_optimized_transform,3,13,100,9.51268613338,65792,6.91623786147,32896,2909,true,1,0.00951268613338 +bf16,1,257,1,64,257,optimized_transform,3,14,100,9.58971440792,65792,6.86068397883,32896,2909,true,1,0.00958971440792 +bf16,1,257,1,64,257,shuffle32_transform,4,0,100,8.90468597412,65792,7.38846941837,32896,2909,true,1,0.00890468597412 +bf16,1,257,1,64,257,baseline_split,4,1,100,70.8550882339,107940,1.5233909475,32896,2909,true,1,0.0708550882339 +bf16,1,257,1,64,257,control_optimized_split,4,2,100,64.5693731308,107940,1.67169038147,32896,2909,true,1,0.0645693731308 +bf16,1,257,1,64,257,optimized_split,4,3,100,34.5789718628,107940,3.12155030023,32896,2909,true,1,0.0345789718628 +bf16,1,257,1,64,257,control_shuffle32_split,4,4,100,102.344913483,107940,1.05466892615,32896,2909,true,1,0.102344913483 +bf16,1,257,1,64,257,shuffle32_split,4,5,100,39.3483424187,107940,2.74319052253,32896,2909,true,1,0.0393483424187 +bf16,1,257,1,64,257,baseline_fused,4,6,100,65.9156608582,42148,0.639423157582,32896,2909,true,1,0.0659156608582 +bf16,1,257,1,64,257,control_optimized_fused,4,7,100,59.1689157486,42148,0.712333485695,32896,2909,true,1,0.0591689157486 +bf16,1,257,1,64,257,optimized_fused,4,8,100,29.0975999832,42148,1.44850434484,32896,2909,true,1,0.0290975999832 +bf16,1,257,1,64,257,control_shuffle32_fused,4,9,100,95.9725666046,42148,0.439167165067,32896,2909,true,1,0.0959725666046 +bf16,1,257,1,64,257,shuffle32_fused,4,10,100,33.2201123238,42148,1.26874947289,32896,2909,true,1,0.0332201123238 +bf16,1,257,1,64,257,baseline_transform,4,11,100,9.47154283524,65792,6.94628120724,32896,2909,true,1,0.00947154283524 +bf16,1,257,1,64,257,control_optimized_transform,4,12,100,9.5759999752,65792,6.87050962514,32896,2909,true,1,0.0095759999752 +bf16,1,257,1,64,257,optimized_transform,4,13,100,9.55062866211,65792,6.88876118292,32896,2909,true,1,0.00955062866211 +bf16,1,257,1,64,257,control_shuffle32_transform,4,14,100,8.8909715414,65792,7.39986622312,32896,2909,true,1,0.0088909715414 +bf16,4,64,16,64,4096,baseline_transform,0,0,100,23.5396575928,1048576,44.5450829464,524288,2909,true,1,0.0235396575928 +bf16,4,64,16,64,4096,control_optimized_transform,0,1,100,24.3366861343,1048576,43.0862276898,524288,2909,true,1,0.0243366861343 +bf16,4,64,16,64,4096,optimized_transform,0,2,100,23.9983987808,1048576,43.6935817917,524288,2909,true,1,0.0239983987808 +bf16,4,64,16,64,4096,control_shuffle32_transform,0,3,100,11.1371433735,1048576,94.1512526898,524288,2909,true,1,0.0111371433735 +bf16,4,64,16,64,4096,shuffle32_transform,0,4,100,11.1481142044,1048576,94.0585986808,524288,2909,true,1,0.0111481142044 +bf16,4,64,16,64,4096,baseline_split,0,5,100,437.863082886,1720320,3.92889939171,524288,2909,true,1,0.437863082886 +bf16,4,64,16,64,4096,control_optimized_split,0,6,100,380.808677673,1720320,4.51754411299,524288,2909,true,1,0.380808677673 +bf16,4,64,16,64,4096,optimized_split,0,7,100,162.023544312,1720320,10.6177161308,524288,2909,true,1,0.162023544312 +bf16,4,64,16,64,4096,control_shuffle32_split,0,8,100,249.860115051,1720320,6.88513250563,524288,2909,true,1,0.249860115051 +bf16,4,64,16,64,4096,shuffle32_split,0,9,100,72.7462863922,1720320,23.6482174599,524288,2909,true,1,0.0727462863922 +bf16,4,64,16,64,4096,baseline_fused,0,10,100,429.791107178,671744,1.56295462791,524288,2909,true,1,0.429791107178 +bf16,4,64,16,64,4096,control_optimized_fused,0,11,100,371.791534424,671744,1.80677594244,524288,2909,true,1,0.371791534424 +bf16,4,64,16,64,4096,optimized_fused,0,12,100,147.956113815,671744,4.54015709576,524288,2909,true,1,0.147956113815 +bf16,4,64,16,64,4096,control_shuffle32_fused,0,13,100,247.708568573,671744,2.71183190743,524288,2909,true,1,0.247708568573 +bf16,4,64,16,64,4096,shuffle32_fused,0,14,100,68.3926820755,671744,9.82186952777,524288,2909,true,1,0.0683926820755 +bf16,4,64,16,64,4096,control_optimized_transform,1,0,100,23.9440011978,1048576,43.7928477925,524288,2909,true,1,0.0239440011978 +bf16,4,64,16,64,4096,optimized_transform,1,1,100,23.9369130135,1048576,43.8058157044,524288,2909,true,1,0.0239369130135 +bf16,4,64,16,64,4096,control_shuffle32_transform,1,2,100,11.0459434986,1048576,94.9286043453,524288,2909,true,1,0.0110459434986 +bf16,4,64,16,64,4096,shuffle32_transform,1,3,100,11.3819420338,1048576,92.1262818673,524288,2909,true,1,0.0113819420338 +bf16,4,64,16,64,4096,baseline_split,1,4,100,438.947181702,1720320,3.91919591175,524288,2909,true,1,0.438947181702 +bf16,4,64,16,64,4096,control_optimized_split,1,5,100,380.758171082,1720320,4.5181433536,524288,2909,true,1,0.380758171082 +bf16,4,64,16,64,4096,optimized_split,1,6,100,162.301025391,1720320,10.5995633475,524288,2909,true,1,0.162301025391 +bf16,4,64,16,64,4096,control_shuffle32_split,1,7,100,250.698299408,1720320,6.86211276288,524288,2909,true,1,0.250698299408 +bf16,4,64,16,64,4096,shuffle32_split,1,8,100,73.0879974365,1720320,23.5376540655,524288,2909,true,1,0.0730879974365 +bf16,4,64,16,64,4096,baseline_fused,1,9,100,429.839096069,671744,1.56278013364,524288,2909,true,1,0.429839096069 +bf16,4,64,16,64,4096,control_optimized_fused,1,10,100,370.933494568,671744,1.81095535948,524288,2909,true,1,0.370933494568 +bf16,4,64,16,64,4096,optimized_fused,1,11,100,148.429946899,671744,4.52566354723,524288,2909,true,1,0.148429946899 +bf16,4,64,16,64,4096,control_shuffle32_fused,1,12,100,247.780799866,671744,2.71104137352,524288,2909,true,1,0.247780799866 +bf16,4,64,16,64,4096,shuffle32_fused,1,13,100,66.9177150726,671744,10.0383582923,524288,2909,true,1,0.0669177150726 +bf16,4,64,16,64,4096,baseline_transform,1,14,100,23.6326861382,1048576,44.3697340992,524288,2909,true,1,0.0236326861382 +bf16,4,64,16,64,4096,optimized_transform,2,0,100,23.9933729172,1048576,43.702734235,524288,2909,true,1,0.0239933729172 +bf16,4,64,16,64,4096,control_shuffle32_transform,2,1,100,11.2532567978,1048576,93.1797806486,524288,2909,true,1,0.0112532567978 +bf16,4,64,16,64,4096,shuffle32_transform,2,2,100,10.8235430717,1048576,96.8791820801,524288,2909,true,1,0.0108235430717 +bf16,4,64,16,64,4096,baseline_split,2,3,100,437.551994324,1720320,3.93169274125,524288,2909,true,1,0.437551994324 +bf16,4,64,16,64,4096,control_optimized_split,2,4,100,381.251869202,1720320,4.51229263112,524288,2909,true,1,0.381251869202 +bf16,4,64,16,64,4096,optimized_split,2,5,100,161.620807648,1720320,10.6441740085,524288,2909,true,1,0.161620807648 +bf16,4,64,16,64,4096,control_shuffle32_split,2,6,100,249.581489563,1720320,6.89281886654,524288,2909,true,1,0.249581489563 +bf16,4,64,16,64,4096,shuffle32_split,2,7,100,72.5305128098,1720320,23.7185693766,524288,2909,true,1,0.0725305128098 +bf16,4,64,16,64,4096,baseline_fused,2,8,100,428.928222656,671744,1.56609885878,524288,2909,true,1,0.428928222656 +bf16,4,64,16,64,4096,control_optimized_fused,2,9,100,371.883430481,671744,1.80632947032,524288,2909,true,1,0.371883430481 +bf16,4,64,16,64,4096,optimized_fused,2,10,100,148.495311737,671744,4.52367143543,524288,2909,true,1,0.148495311737 +bf16,4,64,16,64,4096,control_shuffle32_fused,2,11,100,249.86787796,671744,2.68839678587,524288,2909,true,1,0.24986787796 +bf16,4,64,16,64,4096,shuffle32_fused,2,12,100,64.4285678864,671744,10.4261823914,524288,2909,true,1,0.0644285678864 +bf16,4,64,16,64,4096,baseline_transform,2,13,100,22.7339434624,1048576,46.1238060935,524288,2909,true,1,0.0227339434624 +bf16,4,64,16,64,4096,control_optimized_transform,2,14,100,23.8836574554,1048576,43.9034935062,524288,2909,true,1,0.0238836574554 +bf16,4,64,16,64,4096,control_shuffle32_transform,3,0,100,11.2815999985,1048576,92.9456814762,524288,2909,true,1,0.0112815999985 +bf16,4,64,16,64,4096,shuffle32_transform,3,1,100,11.1344003677,1048576,94.1744472418,524288,2909,true,1,0.0111344003677 +bf16,4,64,16,64,4096,baseline_split,3,2,100,437.664451599,1720320,3.93068249823,524288,2909,true,1,0.437664451599 +bf16,4,64,16,64,4096,control_optimized_split,3,3,100,381.291885376,1720320,4.51181907085,524288,2909,true,1,0.381291885376 +bf16,4,64,16,64,4096,optimized_split,3,4,100,162.194976807,1720320,10.606493702,524288,2909,true,1,0.162194976807 +bf16,4,64,16,64,4096,control_shuffle32_split,3,5,100,250.280227661,1720320,6.87357533624,524288,2909,true,1,0.250280227661 +bf16,4,64,16,64,4096,shuffle32_split,3,6,100,71.2029743195,1720320,24.1607884564,524288,2909,true,1,0.0712029743195 +bf16,4,64,16,64,4096,baseline_fused,3,7,100,428.924331665,671744,1.56611306566,524288,2909,true,1,0.428924331665 +bf16,4,64,16,64,4096,control_optimized_fused,3,8,100,372.080001831,671744,1.80537517925,524288,2909,true,1,0.372080001831 +bf16,4,64,16,64,4096,optimized_fused,3,9,100,148.0742836,671744,4.53653385091,524288,2909,true,1,0.1480742836 +bf16,4,64,16,64,4096,control_shuffle32_fused,3,10,100,246.818294525,671744,2.72161349017,524288,2909,true,1,0.246818294525 +bf16,4,64,16,64,4096,shuffle32_fused,3,11,100,67.2203445435,671744,9.99316508361,524288,2909,true,1,0.0672203445435 +bf16,4,64,16,64,4096,baseline_transform,3,12,100,22.756114006,1048576,46.0788691655,524288,2909,true,1,0.022756114006 +bf16,4,64,16,64,4096,control_optimized_transform,3,13,100,23.915886879,1048576,43.8443284711,524288,2909,true,1,0.023915886879 +bf16,4,64,16,64,4096,optimized_transform,3,14,100,23.9579415321,1048576,43.7673661818,524288,2909,true,1,0.0239579415321 +bf16,4,64,16,64,4096,shuffle32_transform,4,0,100,10.9915423393,1048576,95.3984406946,524288,2909,true,1,0.0109915423393 +bf16,4,64,16,64,4096,baseline_split,4,1,100,437.78263092,1720320,3.92962141139,524288,2909,true,1,0.43778263092 +bf16,4,64,16,64,4096,control_optimized_split,4,2,100,381.566619873,1720320,4.5085704839,524288,2909,true,1,0.381566619873 +bf16,4,64,16,64,4096,optimized_split,4,3,100,161.959075928,1720320,10.6219425503,524288,2909,true,1,0.161959075928 +bf16,4,64,16,64,4096,control_shuffle32_split,4,4,100,250.228118896,1720320,6.87500672421,524288,2909,true,1,0.250228118896 +bf16,4,64,16,64,4096,shuffle32_split,4,5,100,72.3378276825,1720320,23.7817481547,524288,2909,true,1,0.0723378276825 +bf16,4,64,16,64,4096,baseline_fused,4,6,100,430.800476074,671744,1.55929261295,524288,2909,true,1,0.430800476074 +bf16,4,64,16,64,4096,control_optimized_fused,4,7,100,373.206634521,671744,1.79992512958,524288,2909,true,1,0.373206634521 +bf16,4,64,16,64,4096,optimized_fused,4,8,100,148.553142548,671744,4.52191039839,524288,2909,true,1,0.148553142548 +bf16,4,64,16,64,4096,control_shuffle32_fused,4,9,100,247.020568848,671744,2.71938488011,524288,2909,true,1,0.247020568848 +bf16,4,64,16,64,4096,shuffle32_fused,4,10,100,66.9092607498,671744,10.0396266895,524288,2909,true,1,0.0669092607498 +bf16,4,64,16,64,4096,baseline_transform,4,11,100,22.7417135239,1048576,46.1080471751,524288,2909,true,1,0.0227417135239 +bf16,4,64,16,64,4096,control_optimized_transform,4,12,100,23.9515423775,1048576,43.7790595476,524288,2909,true,1,0.0239515423775 +bf16,4,64,16,64,4096,optimized_transform,4,13,100,24.0016007423,1048576,43.6877527985,524288,2909,true,1,0.0240016007423 +bf16,4,64,16,64,4096,control_shuffle32_transform,4,14,100,11.3885712624,1048576,92.0726556338,524288,2909,true,1,0.0113885712624 +bf16,16,64,16,64,16384,baseline_transform,0,0,100,68.1213712692,4194304,61.5710447669,2097152,2909,true,1,0.0681213712692 +bf16,16,64,16,64,16384,control_optimized_transform,0,1,100,73.9131450653,4194304,56.7463878894,2097152,2909,true,1,0.0739131450653 +bf16,16,64,16,64,16384,optimized_transform,0,2,100,73.9503955841,4194304,56.7178034258,2097152,2909,true,1,0.0739503955841 +bf16,16,64,16,64,16384,control_shuffle32_transform,0,3,100,21.2514281273,4194304,197.365747604,2097152,2909,true,1,0.0212514281273 +bf16,16,64,16,64,16384,shuffle32_transform,0,4,100,21.3069725037,4194304,196.851241972,2097152,2909,true,1,0.0213069725037 +bf16,16,64,16,64,16384,baseline_split,0,5,100,1548.34655762,6881280,4.44427635799,2097152,2909,true,1,1.54834655762 +bf16,16,64,16,64,16384,control_optimized_split,0,6,100,1361.06948853,6881280,5.05578889103,2097152,2909,true,1,1.36106948853 +bf16,16,64,16,64,16384,optimized_split,0,7,100,576.563873291,6881280,11.9349829547,2097152,2909,true,1,0.576563873291 +bf16,16,64,16,64,16384,control_shuffle32_split,0,8,100,851.752471924,6881280,8.07896686752,2097152,2909,true,1,0.851752471924 +bf16,16,64,16,64,16384,shuffle32_split,0,9,100,228.768234253,6881280,30.0797006301,2097152,2909,true,1,0.228768234253 +bf16,16,64,16,64,16384,baseline_fused,0,10,100,1525.98358154,2686976,1.7608157994,2097152,2909,true,1,1.52598358154 +bf16,16,64,16,64,16384,control_optimized_fused,0,11,100,1333.91326904,2686976,2.01435585233,2097152,2909,true,1,1.33391326904 +bf16,16,64,16,64,16384,optimized_fused,0,12,100,544.632911682,2686976,4.9335542204,2097152,2909,true,1,0.544632911682 +bf16,16,64,16,64,16384,control_shuffle32_fused,0,13,100,867.045974731,2686976,3.09900060471,2097152,2909,true,1,0.867045974731 +bf16,16,64,16,64,16384,shuffle32_fused,0,14,100,211.951541901,2686976,12.677312823,2097152,2909,true,1,0.211951541901 +bf16,16,64,16,64,16384,control_optimized_transform,1,0,100,73.891658783,4194304,56.7628886546,2097152,2909,true,1,0.073891658783 +bf16,16,64,16,64,16384,optimized_transform,1,1,100,73.9444541931,4194304,56.7223606661,2097152,2909,true,1,0.0739444541931 +bf16,16,64,16,64,16384,control_shuffle32_transform,1,2,100,21.1915421486,4194304,197.923490919,2097152,2909,true,1,0.0211915421486 +bf16,16,64,16,64,16384,shuffle32_transform,1,3,100,21.2950873375,4194304,196.961108143,2097152,2909,true,1,0.0212950873375 +bf16,16,64,16,64,16384,baseline_split,1,4,100,1548.49212646,6881280,4.44385856563,2097152,2909,true,1,1.54849212646 +bf16,16,64,16,64,16384,control_optimized_split,1,5,100,1360.98144531,6881280,5.05611595492,2097152,2909,true,1,1.36098144531 +bf16,16,64,16,64,16384,optimized_split,1,6,100,576.503982544,6881280,11.9362228334,2097152,2909,true,1,0.576503982544 +bf16,16,64,16,64,16384,control_shuffle32_split,1,7,100,851.808242798,6881280,8.07843790922,2097152,2909,true,1,0.851808242798 +bf16,16,64,16,64,16384,shuffle32_split,1,8,100,228.708114624,6881280,30.0876075661,2097152,2909,true,1,0.228708114624 +bf16,16,64,16,64,16384,baseline_fused,1,9,100,1525.91842651,2686976,1.76089098428,2097152,2909,true,1,1.52591842651 +bf16,16,64,16,64,16384,control_optimized_fused,1,10,100,1333.61495972,2686976,2.01480643301,2097152,2909,true,1,1.33361495972 +bf16,16,64,16,64,16384,optimized_fused,1,11,100,544.602508545,2686976,4.93382964243,2097152,2909,true,1,0.544602508545 +bf16,16,64,16,64,16384,control_shuffle32_fused,1,12,100,867.024688721,2686976,3.09907668715,2097152,2909,true,1,0.867024688721 +bf16,16,64,16,64,16384,shuffle32_fused,1,13,100,211.835899353,2686976,12.6842334477,2097152,2909,true,1,0.211835899353 +bf16,16,64,16,64,16384,baseline_transform,1,14,100,68.1888008118,4194304,61.510159294,2097152,2909,true,1,0.0681888008118 +bf16,16,64,16,64,16384,optimized_transform,2,0,100,73.8925743103,4194304,56.7621853637,2097152,2909,true,1,0.0738925743103 +bf16,16,64,16,64,16384,control_shuffle32_transform,2,1,100,21.379430294,4194304,196.184086401,2097152,2909,true,1,0.021379430294 +bf16,16,64,16,64,16384,shuffle32_transform,2,2,100,21.2461709976,4194304,197.414583572,2097152,2909,true,1,0.0212461709976 +bf16,16,64,16,64,16384,baseline_split,2,3,100,1548.48449707,6881280,4.44388046055,2097152,2909,true,1,1.54848449707 +bf16,16,64,16,64,16384,control_optimized_split,2,4,100,1361.06445312,6881280,5.05580759545,2097152,2909,true,1,1.36106445313 +bf16,16,64,16,64,16384,optimized_split,2,5,100,576.496009827,6881280,11.9363879068,2097152,2909,true,1,0.576496009827 +bf16,16,64,16,64,16384,control_shuffle32_split,2,6,100,851.789016724,6881280,8.0786202509,2097152,2909,true,1,0.851789016724 +bf16,16,64,16,64,16384,shuffle32_split,2,7,100,228.734169006,6881280,30.0841803824,2097152,2909,true,1,0.228734169006 +bf16,16,64,16,64,16384,baseline_fused,2,8,100,1525.99411011,2686976,1.76080365068,2097152,2909,true,1,1.52599411011 +bf16,16,64,16,64,16384,control_optimized_fused,2,9,100,1333.49258423,2686976,2.01499133312,2097152,2909,true,1,1.33349258423 +bf16,16,64,16,64,16384,optimized_fused,2,10,100,544.559783936,2686976,4.93421673665,2097152,2909,true,1,0.544559783936 +bf16,16,64,16,64,16384,control_shuffle32_fused,2,11,100,867.111740112,2686976,3.09876556354,2097152,2909,true,1,0.867111740112 +bf16,16,64,16,64,16384,shuffle32_fused,2,12,100,211.838169098,2686976,12.6840975422,2097152,2909,true,1,0.211838169098 +bf16,16,64,16,64,16384,baseline_transform,2,13,100,68.1499433517,4194304,61.5452309087,2097152,2909,true,1,0.0681499433517 +bf16,16,64,16,64,16384,control_optimized_transform,2,14,100,73.8918876648,4194304,56.7627128302,2097152,2909,true,1,0.0738918876648 +bf16,16,64,16,64,16384,control_shuffle32_transform,3,0,100,21.4246845245,4194304,195.769697108,2097152,2909,true,1,0.0214246845245 +bf16,16,64,16,64,16384,shuffle32_transform,3,1,100,21.1757731438,4194304,198.070878996,2097152,2909,true,1,0.0211757731438 +bf16,16,64,16,64,16384,baseline_split,3,2,100,1548.43566895,6881280,4.44402059318,2097152,2909,true,1,1.54843566895 +bf16,16,64,16,64,16384,control_optimized_split,3,3,100,1361.18560791,6881280,5.05535759415,2097152,2909,true,1,1.36118560791 +bf16,16,64,16,64,16384,optimized_split,3,4,100,576.622200012,6881280,11.9337757025,2097152,2909,true,1,0.576622200012 +bf16,16,64,16,64,16384,control_shuffle32_split,3,5,100,851.894607544,6881280,8.0776189203,2097152,2909,true,1,0.851894607544 +bf16,16,64,16,64,16384,shuffle32_split,3,6,100,228.669719696,6881280,30.0926594441,2097152,2909,true,1,0.228669719696 +bf16,16,64,16,64,16384,baseline_fused,3,7,100,1525.77056885,2686976,1.76106162674,2097152,2909,true,1,1.52577056885 +bf16,16,64,16,64,16384,control_optimized_fused,3,8,100,1333.47229004,2686976,2.01502199939,2097152,2909,true,1,1.33347229004 +bf16,16,64,16,64,16384,optimized_fused,3,9,100,544.533233643,2686976,4.93445731866,2097152,2909,true,1,0.544533233643 +bf16,16,64,16,64,16384,control_shuffle32_fused,3,10,100,867.301712036,2686976,3.09808681651,2097152,2909,true,1,0.867301712036 +bf16,16,64,16,64,16384,shuffle32_fused,3,11,100,211.86170578,2686976,12.682688408,2097152,2909,true,1,0.21186170578 +bf16,16,64,16,64,16384,baseline_transform,3,12,100,68.1860542297,4194304,61.5126369663,2097152,2909,true,1,0.0681860542297 +bf16,16,64,16,64,16384,control_optimized_transform,3,13,100,73.902630806,4194304,56.7544612994,2097152,2909,true,1,0.073902630806 +bf16,16,64,16,64,16384,optimized_transform,3,14,100,73.9766836166,4194304,56.6976484339,2097152,2909,true,1,0.0739766836166 +bf16,16,64,16,64,16384,shuffle32_transform,4,0,100,21.3312005997,4194304,196.627657239,2097152,2909,true,1,0.0213312005997 +bf16,16,64,16,64,16384,baseline_split,4,1,100,1548.27545166,6881280,4.44448046542,2097152,2909,true,1,1.54827545166 +bf16,16,64,16,64,16384,control_optimized_split,4,2,100,1360.94940186,6881280,5.05623500082,2097152,2909,true,1,1.36094940186 +bf16,16,64,16,64,16384,optimized_split,4,3,100,576.46812439,6881280,11.9369653045,2097152,2909,true,1,0.57646812439 +bf16,16,64,16,64,16384,control_shuffle32_split,4,4,100,851.851196289,6881280,8.07803056447,2097152,2909,true,1,0.851851196289 +bf16,16,64,16,64,16384,shuffle32_split,4,5,100,228.679542542,6881280,30.091366825,2097152,2909,true,1,0.228679542542 +bf16,16,64,16,64,16384,baseline_fused,4,6,100,1525.73913574,2686976,1.76109790793,2097152,2909,true,1,1.52573913574 +bf16,16,64,16,64,16384,control_optimized_fused,4,7,100,1333.67797852,2686976,2.01471122961,2097152,2909,true,1,1.33367797852 +bf16,16,64,16,64,16384,optimized_fused,4,8,100,544.498977661,2686976,4.93476775942,2097152,2909,true,1,0.544498977661 +bf16,16,64,16,64,16384,control_shuffle32_fused,4,9,100,867.021942139,2686976,3.09908650451,2097152,2909,true,1,0.867021942139 +bf16,16,64,16,64,16384,shuffle32_fused,4,10,100,212.11063385,2686976,12.6678043021,2097152,2909,true,1,0.21211063385 +bf16,16,64,16,64,16384,baseline_transform,4,11,100,68.1375980377,4194304,61.5563818037,2097152,2909,true,1,0.0681375980377 +bf16,16,64,16,64,16384,control_optimized_transform,4,12,100,73.8484573364,4194304,56.7960950205,2097152,2909,true,1,0.0738484573364 +bf16,16,64,16,64,16384,optimized_transform,4,13,100,73.8957691193,4194304,56.7597313079,2097152,2909,true,1,0.0738957691193 +bf16,16,64,16,64,16384,control_shuffle32_transform,4,14,100,21.2758851051,4194304,197.13887245,2097152,2909,true,1,0.0212758851051 +bf16,1,1,1,128,1,baseline_transform,0,0,100,9.34857130051,512,0.0547677269116,256,2909,true,1,0.00934857130051 +bf16,1,1,1,128,1,control_optimized_transform,0,1,100,9.60937142372,512,0.0532813206425,256,2909,true,1,0.00960937142372 +bf16,1,1,1,128,1,optimized_transform,0,2,100,9.68777179718,512,0.0528501301144,256,2909,true,1,0.00968777179718 +bf16,1,1,1,128,1,control_shuffle32_transform,0,3,100,9.06811416149,512,0.0564615741357,256,2909,true,1,0.00906811416149 +bf16,1,1,1,128,1,shuffle32_transform,0,4,100,9.18605685234,512,0.0557366461181,256,2909,true,1,0.00918605685234 +bf16,1,1,1,128,1,baseline_split,0,5,100,79.0443468094,836,0.0105763414304,256,2909,true,1,0.0790443468094 +bf16,1,1,1,128,1,control_optimized_split,0,6,100,63.4658288956,836,0.0131724427861,256,2909,true,1,0.0634658288956 +bf16,1,1,1,128,1,optimized_split,0,7,100,33.8159990311,836,0.0247220257852,256,2909,true,1,0.0338159990311 +bf16,1,1,1,128,1,control_shuffle32_split,0,8,100,161.234073639,836,0.0051850082376,256,2909,true,1,0.161234073639 +bf16,1,1,1,128,1,shuffle32_split,0,9,100,45.341258049,836,0.0184379533337,256,2909,true,1,0.045341258049 +bf16,1,1,1,128,1,baseline_fused,0,10,100,74.5748567581,324,0.00434462785562,256,2909,true,1,0.0745748567581 +bf16,1,1,1,128,1,control_optimized_fused,0,11,100,58.1826257706,324,0.00556867270442,256,2909,true,1,0.0581826257706 +bf16,1,1,1,128,1,optimized_fused,0,12,100,28.9252567291,324,0.0112012834677,256,2909,true,1,0.0289252567291 +bf16,1,1,1,128,1,control_shuffle32_fused,0,13,100,157.18536377,324,0.00206126061759,256,2909,true,1,0.15718536377 +bf16,1,1,1,128,1,shuffle32_fused,0,14,100,40.8601140976,324,0.00792949327616,256,2909,true,1,0.0408601140976 +bf16,1,1,1,128,1,control_optimized_transform,1,0,100,9.71017122269,512,0.0527282154205,256,2909,true,1,0.00971017122269 +bf16,1,1,1,128,1,optimized_transform,1,1,100,9.63154315948,512,0.0531586674661,256,2909,true,1,0.00963154315948 +bf16,1,1,1,128,1,control_shuffle32_transform,1,2,100,9.09874260426,512,0.0562715115999,256,2909,true,1,0.00909874260426 +bf16,1,1,1,128,1,shuffle32_transform,1,3,100,9.13325667381,512,0.0560588646838,256,2909,true,1,0.00913325667381 +bf16,1,1,1,128,1,baseline_split,1,4,100,79.0507459641,836,0.010575485276,256,2909,true,1,0.0790507459641 +bf16,1,1,1,128,1,control_optimized_split,1,5,100,63.5462856293,836,0.0131557649943,256,2909,true,1,0.0635462856293 +bf16,1,1,1,128,1,optimized_split,1,6,100,33.952229023,836,0.0246228310793,256,2909,true,1,0.033952229023 +bf16,1,1,1,128,1,control_shuffle32_split,1,7,100,161.270637512,836,0.00518383267343,256,2909,true,1,0.161270637512 +bf16,1,1,1,128,1,shuffle32_split,1,8,100,46.1954307556,836,0.0180970279165,256,2909,true,1,0.0461954307556 +bf16,1,1,1,128,1,baseline_fused,1,9,100,74.5277690887,324,0.00434737285124,256,2909,true,1,0.0745277690887 +bf16,1,1,1,128,1,control_optimized_fused,1,10,100,58.1775999069,324,0.00556915377256,256,2909,true,1,0.0581775999069 +bf16,1,1,1,128,1,optimized_fused,1,11,100,28.8473129272,324,0.0112315486998,256,2909,true,1,0.0288473129272 +bf16,1,1,1,128,1,control_shuffle32_fused,1,12,100,157.18331337,324,0.002061287506,256,2909,true,1,0.15718331337 +bf16,1,1,1,128,1,shuffle32_fused,1,13,100,40.8717727661,324,0.00792723138911,256,2909,true,1,0.0408717727661 +bf16,1,1,1,128,1,baseline_transform,1,14,100,9.28502857685,512,0.0551425335703,256,2909,true,1,0.00928502857685 +bf16,1,1,1,128,1,optimized_transform,2,0,100,9.6509718895,512,0.0530516517779,256,2909,true,1,0.0096509718895 +bf16,1,1,1,128,1,control_shuffle32_transform,2,1,100,8.98788571358,512,0.0569655663541,256,2909,true,1,0.00898788571358 +bf16,1,1,1,128,1,shuffle32_transform,2,2,100,9.06080007553,512,0.0565071512153,256,2909,true,1,0.00906080007553 +bf16,1,1,1,128,1,baseline_split,2,3,100,79.0500545502,836,0.0105755777748,256,2909,true,1,0.0790500545502 +bf16,1,1,1,128,1,control_optimized_split,2,4,100,63.4475421906,836,0.0131762393173,256,2909,true,1,0.0634475421906 +bf16,1,1,1,128,1,optimized_split,2,5,100,34.007999897,836,0.0245824512624,256,2909,true,1,0.034007999897 +bf16,1,1,1,128,1,control_shuffle32_split,2,6,100,161.287765503,836,0.00518328217514,256,2909,true,1,0.161287765503 +bf16,1,1,1,128,1,shuffle32_split,2,7,100,45.46833992,836,0.0183864201216,256,2909,true,1,0.04546833992 +bf16,1,1,1,128,1,baseline_fused,2,8,100,74.5796537399,324,0.00434434840808,256,2909,true,1,0.0745796537399 +bf16,1,1,1,128,1,control_optimized_fused,2,9,100,58.1789731979,324,0.00556902231495,256,2909,true,1,0.0581789731979 +bf16,1,1,1,128,1,optimized_fused,2,10,100,28.9119982719,324,0.0112064201496,256,2909,true,1,0.0289119982719 +bf16,1,1,1,128,1,control_shuffle32_fused,2,11,100,157.181940079,324,0.00206130551536,256,2909,true,1,0.157181940079 +bf16,1,1,1,128,1,shuffle32_fused,2,12,100,40.8857107162,324,0.00792452899373,256,2909,true,1,0.0408857107162 +bf16,1,1,1,128,1,baseline_transform,2,13,100,9.28937137127,512,0.0551167543569,256,2909,true,1,0.00928937137127 +bf16,1,1,1,128,1,control_optimized_transform,2,14,100,9.66742873192,512,0.0529613420691,256,2909,true,1,0.00966742873192 +bf16,1,1,1,128,1,control_shuffle32_transform,3,0,100,8.98079991341,512,0.0570105118627,256,2909,true,1,0.00898079991341 +bf16,1,1,1,128,1,shuffle32_transform,3,1,100,9.10788595676,512,0.0562150209643,256,2909,true,1,0.00910788595676 +bf16,1,1,1,128,1,baseline_split,3,2,100,79.12296772,836,0.0105658321988,256,2909,true,1,0.07912296772 +bf16,1,1,1,128,1,control_optimized_split,3,3,100,63.4338283539,836,0.0131790879046,256,2909,true,1,0.0634338283539 +bf16,1,1,1,128,1,optimized_split,3,4,100,33.9017128944,836,0.0246595209688,256,2909,true,1,0.0339017128944 +bf16,1,1,1,128,1,control_shuffle32_split,3,5,100,161.94858551,836,0.00516213215056,256,2909,true,1,0.16194858551 +bf16,1,1,1,128,1,shuffle32_split,3,6,100,45.4374885559,836,0.0183989042214,256,2909,true,1,0.0454374885559 +bf16,1,1,1,128,1,baseline_fused,3,7,100,74.5755434036,324,0.00434458785297,256,2909,true,1,0.0745755434036 +bf16,1,1,1,128,1,control_optimized_fused,3,8,100,58.1837701797,324,0.0055685631749,256,2909,true,1,0.0581837701797 +bf16,1,1,1,128,1,optimized_fused,3,9,100,28.918170929,324,0.0112040281108,256,2909,true,1,0.028918170929 +bf16,1,1,1,128,1,control_shuffle32_fused,3,10,100,157.181940079,324,0.00206130551536,256,2909,true,1,0.157181940079 +bf16,1,1,1,128,1,shuffle32_fused,3,11,100,40.8889102936,324,0.00792390889544,256,2909,true,1,0.0408889102936 +bf16,1,1,1,128,1,baseline_transform,3,12,100,9.38399970531,512,0.0545609565301,256,2909,true,1,0.00938399970531 +bf16,1,1,1,128,1,control_optimized_transform,3,13,100,9.61531400681,512,0.0532483910185,256,2909,true,1,0.00961531400681 +bf16,1,1,1,128,1,optimized_transform,3,14,100,9.64914262295,512,0.0530617092116,256,2909,true,1,0.00964914262295 +bf16,1,1,1,128,1,shuffle32_transform,4,0,100,9.05120015144,512,0.0565670840809,256,2909,true,1,0.00905120015144 +bf16,1,1,1,128,1,baseline_split,4,1,100,79.0530252457,836,0.0105751803603,256,2909,true,1,0.0790530252457 +bf16,1,1,1,128,1,control_optimized_split,4,2,100,63.5010290146,836,0.0131651409902,256,2909,true,1,0.0635010290146 +bf16,1,1,1,128,1,optimized_split,4,3,100,33.9520001411,836,0.0246229970701,256,2909,true,1,0.0339520001411 +bf16,1,1,1,128,1,control_shuffle32_split,4,4,100,161.295776367,836,0.00518302474391,256,2909,true,1,0.161295776367 +bf16,1,1,1,128,1,shuffle32_split,4,5,100,45.5586290359,836,0.0183499815005,256,2909,true,1,0.0455586290359 +bf16,1,1,1,128,1,baseline_fused,4,6,100,74.5794248581,324,0.00434436174074,256,2909,true,1,0.0745794248581 +bf16,1,1,1,128,1,control_optimized_fused,4,7,100,58.1675434113,324,0.00557011661485,256,2909,true,1,0.0581675434113 +bf16,1,1,1,128,1,optimized_fused,4,8,100,28.8720011711,324,0.011221944682,256,2909,true,1,0.0288720011711 +bf16,1,1,1,128,1,control_shuffle32_fused,4,9,100,157.184915543,324,0.00206126649546,256,2909,true,1,0.157184915543 +bf16,1,1,1,128,1,shuffle32_fused,4,10,100,40.8537149429,324,0.00793073531875,256,2909,true,1,0.0408537149429 +bf16,1,1,1,128,1,baseline_transform,4,11,100,9.20022845268,512,0.055650792003,256,2909,true,1,0.00920022845268 +bf16,1,1,1,128,1,control_optimized_transform,4,12,100,9.57279980183,512,0.0534848749164,256,2909,true,1,0.00957279980183 +bf16,1,1,1,128,1,optimized_transform,4,13,100,9.58788573742,512,0.0534007198273,256,2909,true,1,0.00958788573742 +bf16,1,1,1,128,1,control_shuffle32_transform,4,14,100,9.10422861576,512,0.0562376036025,256,2909,true,1,0.00910422861576 +bf16,1,17,1,128,17,baseline_transform,0,0,100,9.53782856464,8704,0.912576687766,4352,2909,true,1,0.00953782856464 +bf16,1,17,1,128,17,control_optimized_transform,0,1,100,9.69120025635,8704,0.898134366205,4352,2909,true,1,0.00969120025635 +bf16,1,17,1,128,17,optimized_transform,0,2,100,9.77965772152,8704,0.890010698518,4352,2909,true,1,0.00977965772152 +bf16,1,17,1,128,17,control_shuffle32_transform,0,3,100,9.14102911949,8704,0.952190380998,4352,2909,true,1,0.00914102911949 +bf16,1,17,1,128,17,shuffle32_transform,0,4,100,9.27862882614,8704,0.938069639716,4352,2909,true,1,0.00927862882614 +bf16,1,17,1,128,17,baseline_split,0,5,100,80.4128074646,14212,0.17673801535,4352,2909,true,1,0.0804128074646 +bf16,1,17,1,128,17,control_optimized_split,0,6,100,64.6267414093,14212,0.219908967868,4352,2909,true,1,0.0646267414093 +bf16,1,17,1,128,17,optimized_split,0,7,100,34.7380566597,14212,0.409119028713,4352,2909,true,1,0.0347380566597 +bf16,1,17,1,128,17,control_shuffle32_split,0,8,100,163.260116577,14212,0.0870512670085,4352,2909,true,1,0.163260116577 +bf16,1,17,1,128,17,shuffle32_split,0,9,100,45.4578256607,14212,0.312641438376,4352,2909,true,1,0.0454578256607 +bf16,1,17,1,128,17,baseline_fused,0,10,100,75.647315979,5508,0.0728115720792,4352,2909,true,1,0.075647315979 +bf16,1,17,1,128,17,control_optimized_fused,0,11,100,59.1988563538,5508,0.0930423379649,4352,2909,true,1,0.0591988563538 +bf16,1,17,1,128,17,optimized_fused,0,12,100,29.3033146858,5508,0.187965083782,4352,2909,true,1,0.0293033146858 +bf16,1,17,1,128,17,control_shuffle32_fused,0,13,100,158.864459991,5508,0.0346710648832,4352,2909,true,1,0.158864459991 +bf16,1,17,1,128,17,shuffle32_fused,0,14,100,40.4393148422,5508,0.136204088064,4352,2909,true,1,0.0404393148422 +bf16,1,17,1,128,17,control_optimized_transform,1,0,100,9.90948498249,8704,0.878350390094,4352,2909,true,1,0.00990948498249 +bf16,1,17,1,128,17,optimized_transform,1,1,100,9.80434238911,8704,0.887769893641,4352,2909,true,1,0.00980434238911 +bf16,1,17,1,128,17,control_shuffle32_transform,1,2,100,9.18697178364,8704,0.947428620114,4352,2909,true,1,0.00918697178364 +bf16,1,17,1,128,17,shuffle32_transform,1,3,100,9.26422834396,8704,0.939527791937,4352,2909,true,1,0.00926422834396 +bf16,1,17,1,128,17,baseline_split,1,4,100,80.4281139374,14212,0.176704379902,4352,2909,true,1,0.0804281139374 +bf16,1,17,1,128,17,control_optimized_split,1,5,100,64.6246862411,14212,0.219915961324,4352,2909,true,1,0.0646246862411 +bf16,1,17,1,128,17,optimized_split,1,6,100,34.7334861755,14212,0.409172863564,4352,2909,true,1,0.0347334861755 +bf16,1,17,1,128,17,control_shuffle32_split,1,7,100,163.289356232,14212,0.0870356790423,4352,2909,true,1,0.163289356232 +bf16,1,17,1,128,17,shuffle32_split,1,8,100,45.5396556854,14212,0.312079654229,4352,2909,true,1,0.0455396556854 +bf16,1,17,1,128,17,baseline_fused,1,9,100,75.6454849243,5508,0.0728133345369,4352,2909,true,1,0.0756454849243 +bf16,1,17,1,128,17,control_optimized_fused,1,10,100,59.1995429993,5508,0.0930412587825,4352,2909,true,1,0.0591995429993 +bf16,1,17,1,128,17,optimized_fused,1,11,100,29.2852568626,5508,0.188080986478,4352,2909,true,1,0.0292852568626 +bf16,1,17,1,128,17,control_shuffle32_fused,1,12,100,158.856458664,5508,0.0346728112053,4352,2909,true,1,0.158856458664 +bf16,1,17,1,128,17,shuffle32_fused,1,13,100,40.4523420334,5508,0.136160225172,4352,2909,true,1,0.0404523420334 +bf16,1,17,1,128,17,baseline_transform,1,14,100,9.67268586159,8704,0.899853476537,4352,2909,true,1,0.00967268586159 +bf16,1,17,1,128,17,optimized_transform,2,0,100,9.90948498249,8704,0.878350390094,4352,2909,true,1,0.00990948498249 +bf16,1,17,1,128,17,control_shuffle32_transform,2,1,100,9.16411399841,8704,0.949791763995,4352,2909,true,1,0.00916411399841 +bf16,1,17,1,128,17,shuffle32_transform,2,2,100,9.1961145401,8704,0.946486688704,4352,2909,true,1,0.0091961145401 +bf16,1,17,1,128,17,baseline_split,2,3,100,80.4443454742,14212,0.176668725642,4352,2909,true,1,0.0804443454742 +bf16,1,17,1,128,17,control_optimized_split,2,4,100,64.6036577225,14212,0.219987544065,4352,2909,true,1,0.0646036577225 +bf16,1,17,1,128,17,optimized_split,2,5,100,34.9049162865,14212,0.407163274175,4352,2909,true,1,0.0349049162865 +bf16,1,17,1,128,17,control_shuffle32_split,2,6,100,163.268795013,14212,0.0870466398606,4352,2909,true,1,0.163268795013 +bf16,1,17,1,128,17,shuffle32_split,2,7,100,45.5814886093,14212,0.311793239616,4352,2909,true,1,0.0455814886093 +bf16,1,17,1,128,17,baseline_fused,2,8,100,75.5956602097,5508,0.0728613254349,4352,2909,true,1,0.0755956602097 +bf16,1,17,1,128,17,control_optimized_fused,2,9,100,59.1462850571,5508,0.0931250372646,4352,2909,true,1,0.0591462850571 +bf16,1,17,1,128,17,optimized_fused,2,10,100,29.2893719673,5508,0.188054561434,4352,2909,true,1,0.0292893719673 +bf16,1,17,1,128,17,control_shuffle32_fused,2,11,100,158.812570572,5508,0.0346823930887,4352,2909,true,1,0.158812570572 +bf16,1,17,1,128,17,shuffle32_fused,2,12,100,40.4107427597,5508,0.136300390041,4352,2909,true,1,0.0404107427597 +bf16,1,17,1,128,17,baseline_transform,2,13,100,9.61782872677,8704,0.904985963804,4352,2909,true,1,0.00961782872677 +bf16,1,17,1,128,17,control_optimized_transform,2,14,100,9.81028556824,8704,0.887232072854,4352,2909,true,1,0.00981028556824 +bf16,1,17,1,128,17,control_shuffle32_transform,3,0,100,9.30674254894,8704,0.935235927525,4352,2909,true,1,0.00930674254894 +bf16,1,17,1,128,17,shuffle32_transform,3,1,100,9.18171405792,8704,0.947971146247,4352,2909,true,1,0.00918171405792 +bf16,1,17,1,128,17,baseline_split,3,2,100,80.4383945465,14212,0.176681795803,4352,2909,true,1,0.0804383945465 +bf16,1,17,1,128,17,control_optimized_split,3,3,100,64.600687027,14212,0.219997660305,4352,2909,true,1,0.064600687027 +bf16,1,17,1,128,17,optimized_split,3,4,100,34.735314846,14212,0.409151322307,4352,2909,true,1,0.034735314846 +bf16,1,17,1,128,17,control_shuffle32_split,3,5,100,163.255996704,14212,0.0870534638048,4352,2909,true,1,0.163255996704 +bf16,1,17,1,128,17,shuffle32_split,3,6,100,46.0678863525,14212,0.308501238612,4352,2909,true,1,0.0460678863525 +bf16,1,17,1,128,17,baseline_fused,3,7,100,75.6539440155,5508,0.0728051930627,4352,2909,true,1,0.0756539440155 +bf16,1,17,1,128,17,control_optimized_fused,3,8,100,59.1529178619,5508,0.0931145951727,4352,2909,true,1,0.0591529178619 +bf16,1,17,1,128,17,optimized_fused,3,9,100,29.2724585533,5508,0.188163217994,4352,2909,true,1,0.0292724585533 +bf16,1,17,1,128,17,control_shuffle32_fused,3,10,100,158.875656128,5508,0.0346686215764,4352,2909,true,1,0.158875656128 +bf16,1,17,1,128,17,shuffle32_fused,3,11,100,40.4685735703,5508,0.136105612678,4352,2909,true,1,0.0404685735703 +bf16,1,17,1,128,17,baseline_transform,3,12,100,9.60411429405,8704,0.906278260911,4352,2909,true,1,0.00960411429405 +bf16,1,17,1,128,17,control_optimized_transform,3,13,100,9.87039983273,8704,0.88182851227,4352,2909,true,1,0.00987039983273 +bf16,1,17,1,128,17,optimized_transform,3,14,100,9.87405776978,8704,0.88150183065,4352,2909,true,1,0.00987405776978 +bf16,1,17,1,128,17,shuffle32_transform,4,0,100,9.27542865276,8704,0.938393288962,4352,2909,true,1,0.00927542865276 +bf16,1,17,1,128,17,baseline_split,4,1,100,80.4372596741,14212,0.17668428857,4352,2909,true,1,0.0804372596741 +bf16,1,17,1,128,17,control_optimized_split,4,2,100,64.6806812286,14212,0.219725576943,4352,2909,true,1,0.0646806812286 +bf16,1,17,1,128,17,optimized_split,4,3,100,34.626057148,14212,0.410442342288,4352,2909,true,1,0.034626057148 +bf16,1,17,1,128,17,control_shuffle32_split,4,4,100,163.201828003,14212,0.087082357924,4352,2909,true,1,0.163201828003 +bf16,1,17,1,128,17,shuffle32_split,4,5,100,45.4294872284,14212,0.312836460789,4352,2909,true,1,0.0454294872284 +bf16,1,17,1,128,17,baseline_fused,4,6,100,75.641374588,5508,0.0728172911981,4352,2909,true,1,0.075641374588 +bf16,1,17,1,128,17,control_optimized_fused,4,7,100,59.1913127899,5508,0.0930541956309,4352,2909,true,1,0.0591913127899 +bf16,1,17,1,128,17,optimized_fused,4,8,100,29.2676568031,5508,0.188194088685,4352,2909,true,1,0.0292676568031 +bf16,1,17,1,128,17,control_shuffle32_fused,4,9,100,158.820114136,5508,0.0346807457605,4352,2909,true,1,0.158820114136 +bf16,1,17,1,128,17,shuffle32_fused,4,10,100,40.4834270477,5508,0.136055675166,4352,2909,true,1,0.0404834270477 +bf16,1,17,1,128,17,baseline_transform,4,11,100,9.43291425705,8704,0.922726504537,4352,2909,true,1,0.00943291425705 +bf16,1,17,1,128,17,control_optimized_transform,4,12,100,9.75885748863,8704,0.891907685929,4352,2909,true,1,0.00975885748863 +bf16,1,17,1,128,17,optimized_transform,4,13,100,9.80159938335,8704,0.888018338597,4352,2909,true,1,0.00980159938335 +bf16,1,17,1,128,17,control_shuffle32_transform,4,14,100,9.14491415024,8704,0.951785862284,4352,2909,true,1,0.00914491415024 +bf16,1,257,1,128,257,baseline_transform,0,0,100,9.84274327755,131584,13.3686307048,65792,2909,true,1,0.00984274327755 +bf16,1,257,1,128,257,control_optimized_transform,0,1,100,9.93165791035,131584,13.248946066,65792,2909,true,1,0.00993165791035 +bf16,1,257,1,128,257,optimized_transform,0,2,100,9.97737169266,131584,13.1882427611,65792,2909,true,1,0.00997737169266 +bf16,1,257,1,128,257,control_shuffle32_transform,0,3,100,10.5247998238,131584,12.5022805377,65792,2909,true,1,0.0105247998238 +bf16,1,257,1,128,257,shuffle32_transform,0,4,100,10.4107427597,131584,12.6392518802,65792,2909,true,1,0.0104107427597 +bf16,1,257,1,128,257,baseline_split,0,5,100,81.8132591248,214852,2.6261268931,65792,2909,true,1,0.0818132591248 +bf16,1,257,1,128,257,control_optimized_split,0,6,100,65.701713562,214852,3.27011257929,65792,2909,true,1,0.065701713562 +bf16,1,257,1,128,257,optimized_split,0,7,100,35.07999897,214852,6.12462959829,65792,2909,true,1,0.03507999897 +bf16,1,257,1,128,257,control_shuffle32_split,0,8,100,176.889591217,214852,1.21461075534,65792,2909,true,1,0.176889591217 +bf16,1,257,1,128,257,shuffle32_split,0,9,100,51.5497159958,214852,4.16786001338,65792,2909,true,1,0.0515497159958 +bf16,1,257,1,128,257,baseline_fused,0,10,100,76.7339420319,83268,1.08515212167,65792,2909,true,1,0.0767339420319 +bf16,1,257,1,128,257,control_optimized_fused,0,11,100,60.2075433731,83268,1.38301606966,65792,2909,true,1,0.0602075433731 +bf16,1,257,1,128,257,optimized_fused,0,12,100,29.4975996017,83268,2.82287376343,65792,2909,true,1,0.0294975996017 +bf16,1,257,1,128,257,control_shuffle32_fused,0,13,100,171.211891174,83268,0.486344724241,65792,2909,true,1,0.171211891174 +bf16,1,257,1,128,257,shuffle32_fused,0,14,100,44.8573684692,83268,1.85628365732,65792,2909,true,1,0.0448573684692 +bf16,1,257,1,128,257,control_optimized_transform,1,0,100,10.0548577309,131584,13.086609828,65792,2909,true,1,0.0100548577309 +bf16,1,257,1,128,257,optimized_transform,1,1,100,10.0176000595,131584,13.1352818258,65792,2909,true,1,0.0100176000595 +bf16,1,257,1,128,257,control_shuffle32_transform,1,2,100,10.4397714138,131584,12.6041073874,65792,2909,true,1,0.0104397714138 +bf16,1,257,1,128,257,shuffle32_transform,1,3,100,10.4438865185,131584,12.5991411116,65792,2909,true,1,0.0104438865185 +bf16,1,257,1,128,257,baseline_split,1,4,100,81.8985176086,214852,2.6233930268,65792,2909,true,1,0.0818985176086 +bf16,1,257,1,128,257,control_optimized_split,1,5,100,65.7344007492,214852,3.26848647818,65792,2909,true,1,0.0657344007492 +bf16,1,257,1,128,257,optimized_split,1,6,100,35.1323413849,214852,6.11550473241,65792,2909,true,1,0.0351323413849 +bf16,1,257,1,128,257,control_shuffle32_split,1,7,100,177.524356842,214852,1.21026772789,65792,2909,true,1,0.177524356842 +bf16,1,257,1,128,257,shuffle32_split,1,8,100,51.6116571426,214852,4.1628580033,65792,2909,true,1,0.0516116571426 +bf16,1,257,1,128,257,baseline_fused,1,9,100,76.7737150192,83268,1.08458995346,65792,2909,true,1,0.0767737150192 +bf16,1,257,1,128,257,control_optimized_fused,1,10,100,60.1654863358,83268,1.38398282921,65792,2909,true,1,0.0601654863358 +bf16,1,257,1,128,257,optimized_fused,1,11,100,29.5465135574,83268,2.81820052434,65792,2909,true,1,0.0295465135574 +bf16,1,257,1,128,257,control_shuffle32_fused,1,12,100,171.169834137,83268,0.486464220871,65792,2909,true,1,0.171169834137 +bf16,1,257,1,128,257,shuffle32_fused,1,13,100,44.8578262329,83268,1.85626471438,65792,2909,true,1,0.0448578262329 +bf16,1,257,1,128,257,baseline_transform,1,14,100,10.039999485,131584,13.1059767679,65792,2909,true,1,0.010039999485 +bf16,1,257,1,128,257,optimized_transform,2,0,100,9.99062895775,131584,13.1707423583,65792,2909,true,1,0.00999062895775 +bf16,1,257,1,128,257,control_shuffle32_transform,2,1,100,10.4818284512,131584,12.5535349689,65792,2909,true,1,0.0104818284512 +bf16,1,257,1,128,257,shuffle32_transform,2,2,100,10.5958867073,131584,12.4184038236,65792,2909,true,1,0.0105958867073 +bf16,1,257,1,128,257,baseline_split,2,3,100,81.8390846252,214852,2.62529818098,65792,2909,true,1,0.0818390846252 +bf16,1,257,1,128,257,control_optimized_split,2,4,100,65.8219432831,214852,3.26413942347,65792,2909,true,1,0.0658219432831 +bf16,1,257,1,128,257,optimized_split,2,5,100,35.2381706238,214852,6.0971383076,65792,2909,true,1,0.0352381706238 +bf16,1,257,1,128,257,control_shuffle32_split,2,6,100,177.55405426,214852,1.21006530037,65792,2909,true,1,0.17755405426 +bf16,1,257,1,128,257,shuffle32_split,2,7,100,51.6395425797,214852,4.16061005321,65792,2909,true,1,0.0516395425797 +bf16,1,257,1,128,257,baseline_fused,2,8,100,76.7819452286,83268,1.08447369694,65792,2909,true,1,0.0767819452286 +bf16,1,257,1,128,257,control_optimized_fused,2,9,100,60.2203416824,83268,1.38272214461,65792,2909,true,1,0.0602203416824 +bf16,1,257,1,128,257,optimized_fused,2,10,100,29.5291423798,83268,2.81985839376,65792,2909,true,1,0.0295291423798 +bf16,1,257,1,128,257,control_shuffle32_fused,2,11,100,171.225147247,83268,0.486307071938,65792,2909,true,1,0.171225147247 +bf16,1,257,1,128,257,shuffle32_fused,2,12,100,44.8633146286,83268,1.8560376265,65792,2909,true,1,0.0448633146286 +bf16,1,257,1,128,257,baseline_transform,2,13,100,10.0009143353,131584,13.1571969911,65792,2909,true,1,0.0100009143353 +bf16,1,257,1,128,257,control_optimized_transform,2,14,100,9.93302881718,131584,13.2471175129,65792,2909,true,1,0.00993302881718 +bf16,1,257,1,128,257,control_shuffle32_transform,3,0,100,10.5510866642,131584,12.4711325182,65792,2909,true,1,0.0105510866642 +bf16,1,257,1,128,257,shuffle32_transform,3,1,100,10.5398857594,131584,12.4843857898,65792,2909,true,1,0.0105398857594 +bf16,1,257,1,128,257,baseline_split,3,2,100,81.77942276,214852,2.62721345723,65792,2909,true,1,0.08177942276 +bf16,1,257,1,128,257,control_optimized_split,3,3,100,65.7474279404,214852,3.26783886048,65792,2909,true,1,0.0657474279404 +bf16,1,257,1,128,257,optimized_split,3,4,100,35.1933717728,214852,6.10489956425,65792,2909,true,1,0.0351933717728 +bf16,1,257,1,128,257,control_shuffle32_split,3,5,100,176.931438446,214852,1.21432347969,65792,2909,true,1,0.176931438446 +bf16,1,257,1,128,257,shuffle32_split,3,6,100,52.2292613983,214852,4.11363274624,65792,2909,true,1,0.0522292613983 +bf16,1,257,1,128,257,baseline_fused,3,7,100,76.7730283737,83268,1.08459965386,65792,2909,true,1,0.0767730283737 +bf16,1,257,1,128,257,control_optimized_fused,3,8,100,60.2130270004,83268,1.38289011777,65792,2909,true,1,0.0602130270004 +bf16,1,257,1,128,257,optimized_fused,3,9,100,29.5051431656,83268,2.82215204084,65792,2909,true,1,0.0295051431656 +bf16,1,257,1,128,257,control_shuffle32_fused,3,10,100,171.215782166,83268,0.486333671738,65792,2909,true,1,0.171215782166 +bf16,1,257,1,128,257,shuffle32_fused,3,11,100,44.8171424866,83268,1.85794977948,65792,2909,true,1,0.0448171424866 +bf16,1,257,1,128,257,baseline_transform,3,12,100,9.93897080421,131584,13.2391977592,65792,2909,true,1,0.00993897080421 +bf16,1,257,1,128,257,control_optimized_transform,3,13,100,9.95062887669,131584,13.2236868273,65792,2909,true,1,0.00995062887669 +bf16,1,257,1,128,257,optimized_transform,3,14,100,10.0642287731,131584,13.074424575,65792,2909,true,1,0.0100642287731 +bf16,1,257,1,128,257,shuffle32_transform,4,0,100,10.5074286461,131584,12.5229496609,65792,2909,true,1,0.0105074286461 +bf16,1,257,1,128,257,baseline_split,4,1,100,81.8153095245,214852,2.62606107889,65792,2909,true,1,0.0818153095245 +bf16,1,257,1,128,257,control_optimized_split,4,2,100,65.7337141037,214852,3.26852062035,65792,2909,true,1,0.0657337141037 +bf16,1,257,1,128,257,optimized_split,4,3,100,35.1124572754,214852,6.11896792967,65792,2909,true,1,0.0351124572754 +bf16,1,257,1,128,257,control_shuffle32_split,4,4,100,176.900119781,214852,1.21453846535,65792,2909,true,1,0.176900119781 +bf16,1,257,1,128,257,shuffle32_split,4,5,100,51.6006851196,214852,4.16374316546,65792,2909,true,1,0.0516006851196 +bf16,1,257,1,128,257,baseline_fused,4,6,100,76.7837715149,83268,1.08444790295,65792,2909,true,1,0.0767837715149 +bf16,1,257,1,128,257,control_optimized_fused,4,7,100,60.2080011368,83268,1.38300555454,65792,2909,true,1,0.0602080011368 +bf16,1,257,1,128,257,optimized_fused,4,8,100,29.5412564278,83268,2.81870204822,65792,2909,true,1,0.0295412564278 +bf16,1,257,1,128,257,control_shuffle32_fused,4,9,100,171.216220856,83268,0.486332425654,65792,2909,true,1,0.171216220856 +bf16,1,257,1,128,257,shuffle32_fused,4,10,100,44.8185157776,83268,1.85789284976,65792,2909,true,1,0.0448185157776 +bf16,1,257,1,128,257,baseline_transform,4,11,100,10.0537145138,131584,13.0880979184,65792,2909,true,1,0.0100537145138 +bf16,1,257,1,128,257,control_optimized_transform,4,12,100,9.91428613663,131584,13.2721608179,65792,2909,true,1,0.00991428613663 +bf16,1,257,1,128,257,optimized_transform,4,13,100,10.1328003407,131584,12.9859461922,65792,2909,true,1,0.0101328003407 +bf16,1,257,1,128,257,control_shuffle32_transform,4,14,100,10.4502856731,131584,12.5914261213,65792,2909,true,1,0.0104502856731 +bf16,4,64,16,128,4096,baseline_transform,0,0,100,25.9455990791,2097152,80.8288139196,1048576,2909,true,1,0.0259455990791 +bf16,4,64,16,128,4096,control_optimized_transform,0,1,100,27.8354287148,2097152,75.3411065262,1048576,2909,true,1,0.0278354287148 +bf16,4,64,16,128,4096,optimized_transform,0,2,100,27.7954292297,2097152,75.4495274265,1048576,2909,true,1,0.0277954292297 +bf16,4,64,16,128,4096,control_shuffle32_transform,0,3,100,15.7051432133,2097152,133.532816067,1048576,2909,true,1,0.0157051432133 +bf16,4,64,16,128,4096,shuffle32_transform,0,4,100,15.8530282974,2097152,132.287154268,1048576,2909,true,1,0.0158530282974 +bf16,4,64,16,128,4096,baseline_split,0,5,100,525.378952026,3424256,6.51768782665,1048576,2909,true,1,0.525378952026 +bf16,4,64,16,128,4096,control_optimized_split,0,6,100,403.769111633,3424256,8.48072797384,1048576,2909,true,1,0.403769111633 +bf16,4,64,16,128,4096,optimized_split,0,7,100,169.018745422,3424256,20.2596226321,1048576,2909,true,1,0.169018745422 +bf16,4,64,16,128,4096,control_shuffle32_split,0,8,100,452.04410553,3424256,7.57504844795,1048576,2909,true,1,0.45204410553 +bf16,4,64,16,128,4096,shuffle32_split,0,9,100,105.940799713,3424256,32.3223537039,1048576,2909,true,1,0.105940799713 +bf16,4,64,16,128,4096,baseline_fused,0,10,100,518.184013367,1327104,2.56106704523,1048576,2909,true,1,0.518184013367 +bf16,4,64,16,128,4096,control_optimized_fused,0,11,100,394.161148071,1327104,3.36690718122,1048576,2909,true,1,0.394161148071 +bf16,4,64,16,128,4096,optimized_fused,0,12,100,154.577140808,1327104,8.58538327894,1048576,2909,true,1,0.154577140808 +bf16,4,64,16,128,4096,control_shuffle32_fused,0,13,100,436.674499512,1327104,3.03911495057,1048576,2909,true,1,0.436674499512 +bf16,4,64,16,128,4096,shuffle32_fused,0,14,100,92.9504013062,1327104,14.2775499767,1048576,2909,true,1,0.0929504013062 +bf16,4,64,16,128,4096,control_optimized_transform,1,0,100,27.7821731567,2097152,75.48552765,1048576,2909,true,1,0.0277821731567 +bf16,4,64,16,128,4096,optimized_transform,1,1,100,27.8283429146,2097152,75.3602902781,1048576,2909,true,1,0.0278283429146 +bf16,4,64,16,128,4096,control_shuffle32_transform,1,2,100,15.79657197,2097152,132.759943359,1048576,2909,true,1,0.01579657197 +bf16,4,64,16,128,4096,shuffle32_transform,1,3,100,15.8404564857,2097152,132.392144247,1048576,2909,true,1,0.0158404564857 +bf16,4,64,16,128,4096,baseline_split,1,4,100,527.211875916,3424256,6.49502819726,1048576,2909,true,1,0.527211875916 +bf16,4,64,16,128,4096,control_optimized_split,1,5,100,403.178520203,3424256,8.49315087093,1048576,2909,true,1,0.403178520203 +bf16,4,64,16,128,4096,optimized_split,1,6,100,168.47612381,3424256,20.3248740686,1048576,2909,true,1,0.16847612381 +bf16,4,64,16,128,4096,control_shuffle32_split,1,7,100,453.406867981,3424256,7.55228083608,1048576,2909,true,1,0.453406867981 +bf16,4,64,16,128,4096,shuffle32_split,1,8,100,105.432224274,3424256,32.4782676605,1048576,2909,true,1,0.105432224274 +bf16,4,64,16,128,4096,baseline_fused,1,9,100,516.771659851,1327104,2.56806652358,1048576,2909,true,1,0.516771659851 +bf16,4,64,16,128,4096,control_optimized_fused,1,10,100,391.019210815,1327104,3.39396112337,1048576,2909,true,1,0.391019210815 +bf16,4,64,16,128,4096,optimized_fused,1,11,100,154.226284027,1327104,8.60491457971,1048576,2909,true,1,0.154226284027 +bf16,4,64,16,128,4096,control_shuffle32_fused,1,12,100,435.0025177,1327104,3.05079613566,1048576,2909,true,1,0.4350025177 +bf16,4,64,16,128,4096,shuffle32_fused,1,13,100,92.8980541229,1327104,14.2855952423,1048576,2909,true,1,0.0928980541229 +bf16,4,64,16,128,4096,baseline_transform,1,14,100,25.8916568756,2097152,80.9972111895,1048576,2909,true,1,0.0258916568756 +bf16,4,64,16,128,4096,optimized_transform,2,0,100,27.8297138214,2097152,75.3565779892,1048576,2909,true,1,0.0278297138214 +bf16,4,64,16,128,4096,control_shuffle32_transform,2,1,100,15.8194291592,2097152,132.568121068,1048576,2909,true,1,0.0158194291592 +bf16,4,64,16,128,4096,shuffle32_transform,2,2,100,15.841370821,2097152,132.384502812,1048576,2909,true,1,0.015841370821 +bf16,4,64,16,128,4096,baseline_split,2,3,100,526.129150391,3424256,6.50839436944,1048576,2909,true,1,0.526129150391 +bf16,4,64,16,128,4096,control_optimized_split,2,4,100,404.186515808,3424256,8.47196991011,1048576,2909,true,1,0.404186515808 +bf16,4,64,16,128,4096,optimized_split,2,5,100,169.016685486,3424256,20.2598695517,1048576,2909,true,1,0.169016685486 +bf16,4,64,16,128,4096,control_shuffle32_split,2,6,100,453.034973145,3424256,7.5584804772,1048576,2909,true,1,0.453034973145 +bf16,4,64,16,128,4096,shuffle32_split,2,7,100,105.996341705,3424256,32.3054168182,1048576,2909,true,1,0.105996341705 +bf16,4,64,16,128,4096,baseline_fused,2,8,100,520.300559998,1327104,2.55064880193,1048576,2909,true,1,0.520300559998 +bf16,4,64,16,128,4096,control_optimized_fused,2,9,100,392.886161804,1327104,3.37783340066,1048576,2909,true,1,0.392886161804 +bf16,4,64,16,128,4096,optimized_fused,2,10,100,154.786052704,1327104,8.57379574463,1048576,2909,true,1,0.154786052704 +bf16,4,64,16,128,4096,control_shuffle32_fused,2,11,100,436.50970459,1327104,3.04026230355,1048576,2909,true,1,0.43650970459 +bf16,4,64,16,128,4096,shuffle32_fused,2,12,100,92.9067516327,1327104,14.2842578895,1048576,2909,true,1,0.0929067516327 +bf16,4,64,16,128,4096,baseline_transform,2,13,100,25.9570288658,2097152,80.7932221689,1048576,2909,true,1,0.0259570288658 +bf16,4,64,16,128,4096,control_optimized_transform,2,14,100,27.7501726151,2097152,75.5725749563,1048576,2909,true,1,0.0277501726151 +bf16,4,64,16,128,4096,control_shuffle32_transform,3,0,100,15.741943121,2097152,133.22065668,1048576,2909,true,1,0.015741943121 +bf16,4,64,16,128,4096,shuffle32_transform,3,1,100,15.7974863052,2097152,132.752259409,1048576,2909,true,1,0.0157974863052 +bf16,4,64,16,128,4096,baseline_split,3,2,100,525.952682495,3424256,6.51057806903,1048576,2909,true,1,0.525952682495 +bf16,4,64,16,128,4096,control_optimized_split,3,3,100,404.538078308,3424256,8.46460737224,1048576,2909,true,1,0.404538078308 +bf16,4,64,16,128,4096,optimized_split,3,4,100,169.20343399,3424256,20.2375088924,1048576,2909,true,1,0.16920343399 +bf16,4,64,16,128,4096,control_shuffle32_split,3,5,100,449.245491028,3424256,7.62223788193,1048576,2909,true,1,0.449245491028 +bf16,4,64,16,128,4096,shuffle32_split,3,6,100,105.568914413,3424256,32.4362149504,1048576,2909,true,1,0.105568914413 +bf16,4,64,16,128,4096,baseline_fused,3,7,100,521.910400391,1327104,2.54278128776,1048576,2909,true,1,0.521910400391 +bf16,4,64,16,128,4096,control_optimized_fused,3,8,100,393.708114624,1327104,3.3707814259,1048576,2909,true,1,0.393708114624 +bf16,4,64,16,128,4096,optimized_fused,3,9,100,154.315662384,1327104,8.59993068427,1048576,2909,true,1,0.154315662384 +bf16,4,64,16,128,4096,control_shuffle32_fused,3,10,100,437.182846069,1327104,3.03558113483,1048576,2909,true,1,0.437182846069 +bf16,4,64,16,128,4096,shuffle32_fused,3,11,100,92.8623962402,1327104,14.2910807144,1048576,2909,true,1,0.0928623962402 +bf16,4,64,16,128,4096,baseline_transform,3,12,100,25.9062862396,2097152,80.9514718012,1048576,2909,true,1,0.0259062862396 +bf16,4,64,16,128,4096,control_optimized_transform,3,13,100,27.7716565132,2097152,75.514112707,1048576,2909,true,1,0.0277716565132 +bf16,4,64,16,128,4096,optimized_transform,3,14,100,27.8043413162,2097152,75.4253436954,1048576,2909,true,1,0.0278043413162 +bf16,4,64,16,128,4096,shuffle32_transform,4,0,100,15.8978283405,2097152,131.914369377,1048576,2909,true,1,0.0158978283405 +bf16,4,64,16,128,4096,baseline_split,4,1,100,526.88457489,3424256,6.49906291281,1048576,2909,true,1,0.52688457489 +bf16,4,64,16,128,4096,control_optimized_split,4,2,100,401.786766052,3424256,8.52257040132,1048576,2909,true,1,0.401786766052 +bf16,4,64,16,128,4096,optimized_split,4,3,100,168.478622437,3424256,20.32457264,1048576,2909,true,1,0.168478622437 +bf16,4,64,16,128,4096,control_shuffle32_split,4,4,100,452.670860291,3424256,7.56456025864,1048576,2909,true,1,0.452670860291 +bf16,4,64,16,128,4096,shuffle32_split,4,5,100,105.649833679,3424256,32.4113714215,1048576,2909,true,1,0.105649833679 +bf16,4,64,16,128,4096,baseline_fused,4,6,100,518.654632568,1327104,2.55874317256,1048576,2909,true,1,0.518654632568 +bf16,4,64,16,128,4096,control_optimized_fused,4,7,100,391.950645447,1327104,3.38589568717,1048576,2909,true,1,0.391950645447 +bf16,4,64,16,128,4096,optimized_fused,4,8,100,153.920001984,1327104,8.62203731092,1048576,2909,true,1,0.153920001984 +bf16,4,64,16,128,4096,control_shuffle32_fused,4,9,100,436.244354248,1327104,3.04211157595,1048576,2909,true,1,0.436244354248 +bf16,4,64,16,128,4096,shuffle32_fused,4,10,100,92.7906322479,1327104,14.3021333927,1048576,2909,true,1,0.0927906322479 +bf16,4,64,16,128,4096,baseline_transform,4,11,100,25.8799982071,2097152,81.033699586,1048576,2909,true,1,0.0258799982071 +bf16,4,64,16,128,4096,control_optimized_transform,4,12,100,27.8141713142,2097152,75.3986871047,1048576,2909,true,1,0.0278141713142 +bf16,4,64,16,128,4096,optimized_transform,4,13,100,27.8077721596,2097152,75.4160379323,1048576,2909,true,1,0.0278077721596 +bf16,4,64,16,128,4096,control_shuffle32_transform,4,14,100,15.7737135887,2097152,132.952331625,1048576,2909,true,1,0.0157737135887 +bf16,16,64,16,128,16384,baseline_transform,0,0,100,78.8971424103,8388608,106.323343834,4194304,2909,true,1,0.0788971424103 +bf16,16,64,16,128,16384,control_optimized_transform,0,1,100,87.1840000153,8388608,96.2172875589,4194304,2909,true,1,0.0871840000153 +bf16,16,64,16,128,16384,optimized_transform,0,2,100,87.2006797791,8388608,96.1988830965,4194304,2909,true,1,0.0872006797791 +bf16,16,64,16,128,16384,control_shuffle32_transform,0,3,100,26.6102862358,8388608,315.239299783,4194304,2909,true,1,0.0266102862358 +bf16,16,64,16,128,16384,shuffle32_transform,0,4,100,26.612341404,8388608,315.214955072,4194304,2909,true,1,0.026612341404 +bf16,16,64,16,128,16384,baseline_split,0,5,100,1884.65118408,13697024,7.26767059904,4194304,2909,true,1,1.88465118408 +bf16,16,64,16,128,16384,control_optimized_split,0,6,100,1453.04443359,13697024,9.42643162407,4194304,2909,true,1,1.45304443359 +bf16,16,64,16,128,16384,optimized_split,0,7,100,601.886177063,13697024,22.7568343019,4194304,2909,true,1,0.601886177063 +bf16,16,64,16,128,16384,control_shuffle32_split,0,8,100,1605.94467163,13697024,8.52895136549,4194304,2909,true,1,1.60594467163 +bf16,16,64,16,128,16384,shuffle32_split,0,9,100,288.876800537,13697024,47.4147594218,4194304,2909,true,1,0.288876800537 +bf16,16,64,16,128,16384,baseline_fused,0,10,100,1864.63470459,5308416,2.84689327456,4194304,2909,true,1,1.86463470459 +bf16,16,64,16,128,16384,control_optimized_fused,0,11,100,1421.5927124,5308416,3.73413281715,4194304,2909,true,1,1.4215927124 +bf16,16,64,16,128,16384,optimized_fused,0,12,100,566.293487549,5308416,9.37396617958,4194304,2909,true,1,0.566293487549 +bf16,16,64,16,128,16384,control_shuffle32_fused,0,13,100,1496.78390503,5308416,3.54654802351,4194304,2909,true,1,1.49678390503 +bf16,16,64,16,128,16384,shuffle32_fused,0,14,100,265.213031769,5308416,20.0156680258,4194304,2909,true,1,0.265213031769 +bf16,16,64,16,128,16384,control_optimized_transform,1,0,100,87.1190929413,8388608,96.2889731377,4194304,2909,true,1,0.0871190929413 +bf16,16,64,16,128,16384,optimized_transform,1,1,100,87.1684551239,8388608,96.2344461431,4194304,2909,true,1,0.0871684551239 +bf16,16,64,16,128,16384,control_shuffle32_transform,1,2,100,26.6507434845,8388608,314.760749728,4194304,2909,true,1,0.0266507434845 +bf16,16,64,16,128,16384,shuffle32_transform,1,3,100,26.5883421898,8388608,315.499474925,4194304,2909,true,1,0.0265883421898 +bf16,16,64,16,128,16384,baseline_split,1,4,100,1884.51858521,13697024,7.26818196835,4194304,2909,true,1,1.88451858521 +bf16,16,64,16,128,16384,control_optimized_split,1,5,100,1453.40209961,13697024,9.42411188458,4194304,2909,true,1,1.45340209961 +bf16,16,64,16,128,16384,optimized_split,1,6,100,601.863288879,13697024,22.7576997187,4194304,2909,true,1,0.601863288879 +bf16,16,64,16,128,16384,control_shuffle32_split,1,7,100,1607.79312134,13697024,8.51914578948,4194304,2909,true,1,1.60779312134 +bf16,16,64,16,128,16384,shuffle32_split,1,8,100,288.958873749,13697024,47.4012921711,4194304,2909,true,1,0.288958873749 +bf16,16,64,16,128,16384,baseline_fused,1,9,100,1864.72290039,5308416,2.84675862504,4194304,2909,true,1,1.86472290039 +bf16,16,64,16,128,16384,control_optimized_fused,1,10,100,1421.62002563,5308416,3.73406107418,4194304,2909,true,1,1.42162002563 +bf16,16,64,16,128,16384,optimized_fused,1,11,100,566.326408386,5308416,9.37342126624,4194304,2909,true,1,0.566326408386 +bf16,16,64,16,128,16384,control_shuffle32_fused,1,12,100,1498.11355591,5308416,3.54340028435,4194304,2909,true,1,1.49811355591 +bf16,16,64,16,128,16384,shuffle32_fused,1,13,100,265.200233459,5308416,20.0166339628,4194304,2909,true,1,0.265200233459 +bf16,16,64,16,128,16384,baseline_transform,1,14,100,78.8795375824,8388608,106.347073742,4194304,2909,true,1,0.0788795375824 +bf16,16,64,16,128,16384,optimized_transform,2,0,100,87.1782875061,8388608,96.2235923642,4194304,2909,true,1,0.0871782875061 +bf16,16,64,16,128,16384,control_shuffle32_transform,2,1,100,26.582171917,8388608,315.572708889,4194304,2909,true,1,0.026582171917 +bf16,16,64,16,128,16384,shuffle32_transform,2,2,100,26.6457128525,8388608,314.82017563,4194304,2909,true,1,0.0266457128525 +bf16,16,64,16,128,16384,baseline_split,2,3,100,1884.55200195,13697024,7.26805308944,4194304,2909,true,1,1.88455200195 +bf16,16,64,16,128,16384,control_optimized_split,2,4,100,1453.00811768,13697024,9.42666722462,4194304,2909,true,1,1.45300811768 +bf16,16,64,16,128,16384,optimized_split,2,5,100,601.76410675,13697024,22.7614506189,4194304,2909,true,1,0.60176410675 +bf16,16,64,16,128,16384,control_shuffle32_split,2,6,100,1605.63156128,13697024,8.53061457579,4194304,2909,true,1,1.60563156128 +bf16,16,64,16,128,16384,shuffle32_split,2,7,100,288.868808746,13697024,47.4160711897,4194304,2909,true,1,0.288868808746 +bf16,16,64,16,128,16384,baseline_fused,2,8,100,1864.70397949,5308416,2.84678751072,4194304,2909,true,1,1.86470397949 +bf16,16,64,16,128,16384,control_optimized_fused,2,9,100,1421.40380859,5308416,3.73462908141,4194304,2909,true,1,1.42140380859 +bf16,16,64,16,128,16384,optimized_fused,2,10,100,566.246185303,5308416,9.37474924827,4194304,2909,true,1,0.566246185303 +bf16,16,64,16,128,16384,control_shuffle32_fused,2,11,100,1496.22909546,5308416,3.54786310205,4194304,2909,true,1,1.49622909546 +bf16,16,64,16,128,16384,shuffle32_fused,2,12,100,265.209369659,5308416,20.0159444096,4194304,2909,true,1,0.265209369659 +bf16,16,64,16,128,16384,baseline_transform,2,13,100,78.8690233231,8388608,106.36125118,4194304,2909,true,1,0.0788690233231 +bf16,16,64,16,128,16384,control_optimized_transform,2,14,100,87.1028518677,8388608,96.3069270423,4194304,2909,true,1,0.0871028518677 +bf16,16,64,16,128,16384,control_shuffle32_transform,3,0,100,26.623313427,8388608,315.085048411,4194304,2909,true,1,0.026623313427 +bf16,16,64,16,128,16384,shuffle32_transform,3,1,100,26.6898274422,8388608,314.299821465,4194304,2909,true,1,0.0266898274422 +bf16,16,64,16,128,16384,baseline_split,3,2,100,1884.29702759,13697024,7.26903656879,4194304,2909,true,1,1.88429702759 +bf16,16,64,16,128,16384,control_optimized_split,3,3,100,1453.04321289,13697024,9.42643954322,4194304,2909,true,1,1.45304321289 +bf16,16,64,16,128,16384,optimized_split,3,4,100,601.886863708,13697024,22.7568083404,4194304,2909,true,1,0.601886863708 +bf16,16,64,16,128,16384,control_shuffle32_split,3,5,100,1605.01144409,13697024,8.53391049043,4194304,2909,true,1,1.60501144409 +bf16,16,64,16,128,16384,shuffle32_split,3,6,100,288.835887909,13697024,47.4214755623,4194304,2909,true,1,0.288835887909 +bf16,16,64,16,128,16384,baseline_fused,3,7,100,1864.76287842,5308416,2.84669759434,4194304,2909,true,1,1.86476287842 +bf16,16,64,16,128,16384,control_optimized_fused,3,8,100,1421.77032471,5308416,3.73366633679,4194304,2909,true,1,1.42177032471 +bf16,16,64,16,128,16384,optimized_fused,3,9,100,566.21963501,5308416,9.37518883447,4194304,2909,true,1,0.56621963501 +bf16,16,64,16,128,16384,control_shuffle32_fused,3,10,100,1497.91397095,5308416,3.54387241388,4194304,2909,true,1,1.49791397095 +bf16,16,64,16,128,16384,shuffle32_fused,3,11,100,265.217151642,5308416,20.0153571032,4194304,2909,true,1,0.265217151642 +bf16,16,64,16,128,16384,baseline_transform,3,12,100,78.8706254959,8388608,106.359090565,4194304,2909,true,1,0.0788706254959 +bf16,16,64,16,128,16384,control_optimized_transform,3,13,100,87.1753120422,8388608,96.2268766636,4194304,2909,true,1,0.0871753120422 +bf16,16,64,16,128,16384,optimized_transform,3,14,100,87.2203445435,8388608,96.177194024,4194304,2909,true,1,0.0872203445435 +bf16,16,64,16,128,16384,shuffle32_transform,4,0,100,26.6217136383,8388608,315.103982936,4194304,2909,true,1,0.0266217136383 +bf16,16,64,16,128,16384,baseline_split,4,1,100,1884.77371216,13697024,7.26719813187,4194304,2909,true,1,1.88477371216 +bf16,16,64,16,128,16384,control_optimized_split,4,2,100,1453.0065918,13697024,9.42667712406,4194304,2909,true,1,1.4530065918 +bf16,16,64,16,128,16384,optimized_split,4,3,100,601.787414551,13697024,22.7605690462,4194304,2909,true,1,0.601787414551 +bf16,16,64,16,128,16384,control_shuffle32_split,4,4,100,1606.22406006,13697024,8.52746783005,4194304,2909,true,1,1.60622406006 +bf16,16,64,16,128,16384,shuffle32_split,4,5,100,288.782615662,13697024,47.430223487,4194304,2909,true,1,0.288782615662 +bf16,16,64,16,128,16384,baseline_fused,4,6,100,1864.77981567,5308416,2.84667173861,4194304,2909,true,1,1.86477981567 +bf16,16,64,16,128,16384,control_optimized_fused,4,7,100,1421.4352417,5308416,3.73454649517,4194304,2909,true,1,1.4214352417 +bf16,16,64,16,128,16384,optimized_fused,4,8,100,566.316795349,5308416,9.37358037691,4194304,2909,true,1,0.566316795349 +bf16,16,64,16,128,16384,control_shuffle32_fused,4,9,100,1496.74667358,5308416,3.54663624359,4194304,2909,true,1,1.49674667358 +bf16,16,64,16,128,16384,shuffle32_fused,4,10,100,265.215072632,5308416,20.0155140027,4194304,2909,true,1,0.265215072632 +bf16,16,64,16,128,16384,baseline_transform,4,11,100,78.9165687561,8388608,106.297170952,4194304,2909,true,1,0.0789165687561 +bf16,16,64,16,128,16384,control_optimized_transform,4,12,100,87.1616077423,8388608,96.2420062833,4194304,2909,true,1,0.0871616077423 +bf16,16,64,16,128,16384,optimized_transform,4,13,100,87.1871948242,8388608,96.2137618593,4194304,2909,true,1,0.0871871948242 +bf16,16,64,16,128,16384,control_shuffle32_transform,4,14,100,26.6171431541,8388608,315.158090086,4194304,2909,true,1,0.0266171431541 +bf16,1,1,1,256,1,baseline_transform,0,0,100,9.74011421204,1024,0.105132237437,512,2909,true,1,0.00974011421204 +bf16,1,1,1,256,1,control_optimized_transform,0,1,100,10.4962289333,1024,0.0975588477065,512,2909,true,1,0.0104962289333 +bf16,1,1,1,256,1,optimized_transform,0,2,100,10.4194283485,1024,0.0982779444079,512,2909,true,1,0.0104194283485 +bf16,1,1,1,256,1,control_shuffle32_transform,0,3,100,11.5691423416,1024,0.0885113148203,512,2909,true,1,0.0115691423416 +bf16,1,1,1,256,1,shuffle32_transform,0,4,100,11.5488004684,1024,0.0886672172403,512,2909,true,1,0.0115488004684 +bf16,1,1,1,256,1,baseline_split,0,5,100,100.553607941,1668,0.0165881665925,512,2909,true,1,0.100553607941 +bf16,1,1,1,256,1,control_optimized_split,0,6,100,66.9113111496,1668,0.0249285206244,512,2909,true,1,0.0669113111496 +bf16,1,1,1,256,1,optimized_split,0,7,100,35.4729151726,1668,0.047021790904,512,2909,true,1,0.0354729151726 +bf16,1,1,1,256,1,control_shuffle32_split,0,8,100,304.881362915,1668,0.00547098052846,512,2909,true,1,0.304881362915 +bf16,1,1,1,256,1,shuffle32_split,0,9,100,67.96251297,1668,0.0245429417941,512,2909,true,1,0.06796251297 +bf16,1,1,1,256,1,baseline_fused,0,10,100,96.0182857513,644,0.00670705579631,512,2909,true,1,0.0960182857513 +bf16,1,1,1,256,1,control_optimized_fused,0,11,100,61.1515426636,644,0.0105312142907,512,2909,true,1,0.0611515426636 +bf16,1,1,1,256,1,optimized_fused,0,12,100,29.9606871605,644,0.0214948340988,512,2909,true,1,0.0299606871605 +bf16,1,1,1,256,1,control_shuffle32_fused,0,13,100,300.692806244,644,0.00214172067515,512,2909,true,1,0.300692806244 +bf16,1,1,1,256,1,shuffle32_fused,0,14,100,61.5433120728,644,0.0104641752013,512,2909,true,1,0.0615433120728 +bf16,1,1,1,256,1,control_optimized_transform,1,0,100,10.5204570293,1024,0.0973341744702,512,2909,true,1,0.0105204570293 +bf16,1,1,1,256,1,optimized_transform,1,1,100,10.4884564877,1024,0.0976311434581,512,2909,true,1,0.0104884564877 +bf16,1,1,1,256,1,control_shuffle32_transform,1,2,100,11.5126860142,1024,0.0889453598178,512,2909,true,1,0.0115126860142 +bf16,1,1,1,256,1,shuffle32_transform,1,3,100,11.5961134434,1024,0.0883054486316,512,2909,true,1,0.0115961134434 +bf16,1,1,1,256,1,baseline_split,1,4,100,100.565490723,1668,0.0165862065408,512,2909,true,1,0.100565490723 +bf16,1,1,1,256,1,control_optimized_split,1,5,100,66.9478845596,1668,0.0249149022553,512,2909,true,1,0.0669478845596 +bf16,1,1,1,256,1,optimized_split,1,6,100,35.5522298813,1668,0.0469168883519,512,2909,true,1,0.0355522298813 +bf16,1,1,1,256,1,control_shuffle32_split,1,7,100,304.911327362,1668,0.00547044288066,512,2909,true,1,0.304911327362 +bf16,1,1,1,256,1,shuffle32_split,1,8,100,68.0301713943,1668,0.0245185329658,512,2909,true,1,0.0680301713943 +bf16,1,1,1,256,1,baseline_fused,1,9,100,96.0182857513,644,0.00670705579631,512,2909,true,1,0.0960182857513 +bf16,1,1,1,256,1,control_optimized_fused,1,10,100,61.1728000641,644,0.0105275547192,512,2909,true,1,0.0611728000641 +bf16,1,1,1,256,1,optimized_fused,1,11,100,29.9282288551,644,0.021518146066,512,2909,true,1,0.0299282288551 +bf16,1,1,1,256,1,control_shuffle32_fused,1,12,100,300.696239471,644,0.00214169622185,512,2909,true,1,0.300696239471 +bf16,1,1,1,256,1,shuffle32_fused,1,13,100,61.5446853638,644,0.0104639417066,512,2909,true,1,0.0615446853638 +bf16,1,1,1,256,1,baseline_transform,1,14,100,9.74034309387,1024,0.105129767004,512,2909,true,1,0.00974034309387 +bf16,1,1,1,256,1,optimized_transform,2,0,100,10.4928004742,1024,0.0975907244707,512,2909,true,1,0.0104928004742 +bf16,1,1,1,256,1,control_shuffle32_transform,2,1,100,11.4802277088,1024,0.0891968370291,512,2909,true,1,0.0114802277088 +bf16,1,1,1,256,1,shuffle32_transform,2,2,100,11.4889144897,1024,0.0891293952021,512,2909,true,1,0.0114889144897 +bf16,1,1,1,256,1,baseline_split,2,3,100,100.574636459,1668,0.016584698277,512,2909,true,1,0.100574636459 +bf16,1,1,1,256,1,control_optimized_split,2,4,100,66.9531440735,1668,0.0249129450615,512,2909,true,1,0.0669531440735 +bf16,1,1,1,256,1,optimized_split,2,5,100,35.5030846596,1668,0.046981833156,512,2909,true,1,0.0355030846596 +bf16,1,1,1,256,1,control_shuffle32_split,2,6,100,304.895553589,1668,0.00547072589405,512,2909,true,1,0.304895553589 +bf16,1,1,1,256,1,shuffle32_split,2,7,100,68.676571846,1668,0.0242877586223,512,2909,true,1,0.068676571846 +bf16,1,1,1,256,1,baseline_fused,2,8,100,96.0130214691,644,0.00670742353637,512,2909,true,1,0.0960130214691 +bf16,1,1,1,256,1,control_optimized_fused,2,9,100,61.1561155319,644,0.010530426833,512,2909,true,1,0.0611561155319 +bf16,1,1,1,256,1,optimized_fused,2,10,100,29.952685833,644,0.0215005760616,512,2909,true,1,0.029952685833 +bf16,1,1,1,256,1,control_shuffle32_fused,2,11,100,300.70400238,644,0.00214164093229,512,2909,true,1,0.30070400238 +bf16,1,1,1,256,1,shuffle32_fused,2,12,100,61.5460586548,644,0.0104637082224,512,2909,true,1,0.0615460586548 +bf16,1,1,1,256,1,baseline_transform,2,13,100,9.71131443977,1024,0.105444016498,512,2909,true,1,0.00971131443977 +bf16,1,1,1,256,1,control_optimized_transform,2,14,100,10.5348563194,1024,0.0972011358249,512,2909,true,1,0.0105348563194 +bf16,1,1,1,256,1,control_shuffle32_transform,3,0,100,11.6068565845,1024,0.0882237143658,512,2909,true,1,0.0116068565845 +bf16,1,1,1,256,1,shuffle32_transform,3,1,100,11.5353143215,1024,0.0887708797054,512,2909,true,1,0.0115353143215 +bf16,1,1,1,256,1,baseline_split,3,2,100,100.549259186,1668,0.0165888840306,512,2909,true,1,0.100549259186 +bf16,1,1,1,256,1,control_optimized_split,3,3,100,66.9426298141,1668,0.0249168579817,512,2909,true,1,0.0669426298141 +bf16,1,1,1,256,1,optimized_split,3,4,100,36.4027428627,1668,0.0458207230782,512,2909,true,1,0.0364027428627 +bf16,1,1,1,256,1,control_shuffle32_split,3,5,100,304.910163879,1668,0.00547046375489,512,2909,true,1,0.304910163879 +bf16,1,1,1,256,1,shuffle32_split,3,6,100,68.6089134216,1668,0.0243117099049,512,2909,true,1,0.0686089134216 +bf16,1,1,1,256,1,baseline_fused,3,7,100,96.010055542,644,0.00670763074101,512,2909,true,1,0.096010055542 +bf16,1,1,1,256,1,control_optimized_fused,3,8,100,61.1531400681,644,0.0105309392009,512,2909,true,1,0.0611531400681 +bf16,1,1,1,256,1,optimized_fused,3,9,100,29.967083931,644,0.0214902458138,512,2909,true,1,0.029967083931 +bf16,1,1,1,256,1,control_shuffle32_fused,3,10,100,300.700569153,644,0.00214166538432,512,2909,true,1,0.300700569153 +bf16,1,1,1,256,1,shuffle32_fused,3,11,100,61.4980554581,644,0.0104718758212,512,2909,true,1,0.0614980554581 +bf16,1,1,1,256,1,baseline_transform,3,12,100,9.92548525333,1024,0.103168759397,512,2909,true,1,0.00992548525333 +bf16,1,1,1,256,1,control_optimized_transform,3,13,100,10.5247998238,1024,0.0972940119667,512,2909,true,1,0.0105247998238 +bf16,1,1,1,256,1,optimized_transform,3,14,100,10.4381716251,1024,0.0981014718645,512,2909,true,1,0.0104381716251 +bf16,1,1,1,256,1,shuffle32_transform,4,0,100,11.5243422985,1024,0.0888553961238,512,2909,true,1,0.0115243422985 +bf16,1,1,1,256,1,baseline_split,4,1,100,100.56137085,1668,0.0165868860568,512,2909,true,1,0.10056137085 +bf16,1,1,1,256,1,control_optimized_split,4,2,100,66.896686554,1668,0.0249339703642,512,2909,true,1,0.066896686554 +bf16,1,1,1,256,1,optimized_split,4,3,100,35.4683423042,1668,0.0470278533373,512,2909,true,1,0.0354683423042 +bf16,1,1,1,256,1,control_shuffle32_split,4,4,100,304.919776917,1668,0.00547029129061,512,2909,true,1,0.304919776917 +bf16,1,1,1,256,1,shuffle32_split,4,5,100,67.985830307,1668,0.0245345242158,512,2909,true,1,0.067985830307 +bf16,1,1,1,256,1,baseline_fused,4,6,100,96.0107421875,644,0.00670758276967,512,2909,true,1,0.0960107421875 +bf16,1,1,1,256,1,control_optimized_fused,4,7,100,61.1558818817,644,0.0105304670652,512,2909,true,1,0.0611558818817 +bf16,1,1,1,256,1,optimized_fused,4,8,100,29.9485707283,644,0.0215035303635,512,2909,true,1,0.0299485707283 +bf16,1,1,1,256,1,control_shuffle32_fused,4,9,100,300.69852829,644,0.00214167991996,512,2909,true,1,0.30069852829 +bf16,1,1,1,256,1,shuffle32_fused,4,10,100,61.5433120728,644,0.0104641752013,512,2909,true,1,0.0615433120728 +bf16,1,1,1,256,1,baseline_transform,4,11,100,9.80480015278,1024,0.10443864067,512,2909,true,1,0.00980480015278 +bf16,1,1,1,256,1,control_optimized_transform,4,12,100,10.743085146,1024,0.0953171259548,512,2909,true,1,0.010743085146 +bf16,1,1,1,256,1,optimized_transform,4,13,100,10.4653716087,1024,0.0978465016135,512,2909,true,1,0.0104653716087 +bf16,1,1,1,256,1,control_shuffle32_transform,4,14,100,11.5504002571,1024,0.0886549363837,512,2909,true,1,0.0115504002571 +bf16,1,17,1,256,17,baseline_transform,0,0,100,9.97988641262,17408,1.74430843,8704,2909,true,1,0.00997988641262 +bf16,1,17,1,256,17,control_optimized_transform,0,1,100,10.677485466,17408,1.63034640088,8704,2909,true,1,0.010677485466 +bf16,1,17,1,256,17,optimized_transform,0,2,100,10.7003438473,17408,1.62686360817,8704,2909,true,1,0.0107003438473 +bf16,1,17,1,256,17,control_shuffle32_transform,0,3,100,11.7323434353,17408,1.48376154312,8704,2909,true,1,0.0117323434353 +bf16,1,17,1,256,17,shuffle32_transform,0,4,100,11.7456007004,17408,1.48208682077,8704,2909,true,1,0.0117456007004 +bf16,1,17,1,256,17,baseline_split,0,5,100,101.296691895,28356,0.279930168199,8704,2909,true,1,0.101296691895 +bf16,1,17,1,256,17,control_optimized_split,0,6,100,67.3522281647,28356,0.42101057044,8704,2909,true,1,0.0673522281647 +bf16,1,17,1,256,17,optimized_split,0,7,100,36.8543982506,28356,0.76940613186,8704,2909,true,1,0.0368543982506 +bf16,1,17,1,256,17,control_shuffle32_split,0,8,100,305.980567932,28356,0.0926725516971,8704,2909,true,1,0.305980567932 +bf16,1,17,1,256,17,shuffle32_split,0,9,100,68.1437683105,28356,0.41612022204,8704,2909,true,1,0.0681437683105 +bf16,1,17,1,256,17,baseline_fused,0,10,100,96.5369129181,10948,0.113407396912,8704,2909,true,1,0.0965369129181 +bf16,1,17,1,256,17,control_optimized_fused,0,11,100,61.5819454193,10948,0.177779378768,8704,2909,true,1,0.0615819454193 +bf16,1,17,1,256,17,optimized_fused,0,12,100,30.256228447,10948,0.361842852264,8704,2909,true,1,0.030256228447 +bf16,1,17,1,256,17,control_shuffle32_fused,0,13,100,301.218738556,10948,0.0363456803932,8704,2909,true,1,0.301218738556 +bf16,1,17,1,256,17,shuffle32_fused,0,14,100,61.4667463303,10948,0.178112567423,8704,2909,true,1,0.0614667463303 +bf16,1,17,1,256,17,control_optimized_transform,1,0,100,10.6829714775,17408,1.62950917136,8704,2909,true,1,0.0106829714775 +bf16,1,17,1,256,17,optimized_transform,1,1,100,10.8032000065,17408,1.61137440662,8704,2909,true,1,0.0108032000065 +bf16,1,17,1,256,17,control_shuffle32_transform,1,2,100,11.7245709896,17408,1.48474515745,8704,2909,true,1,0.0117245709896 +bf16,1,17,1,256,17,shuffle32_transform,1,3,100,11.805485487,17408,1.47456875189,8704,2909,true,1,0.011805485487 +bf16,1,17,1,256,17,baseline_split,1,4,100,101.292572021,28356,0.279941553799,8704,2909,true,1,0.101292572021 +bf16,1,17,1,256,17,control_optimized_split,1,5,100,67.3721122742,28356,0.420886313978,8704,2909,true,1,0.0673721122742 +bf16,1,17,1,256,17,optimized_split,1,6,100,36.3766860962,28356,0.779510258989,8704,2909,true,1,0.0363766860962 +bf16,1,17,1,256,17,control_shuffle32_split,1,7,100,305.404567719,28356,0.0928473343141,8704,2909,true,1,0.305404567719 +bf16,1,17,1,256,17,shuffle32_split,1,8,100,68.0891418457,28356,0.41645406641,8704,2909,true,1,0.0680891418457 +bf16,1,17,1,256,17,baseline_fused,1,9,100,96.5405750275,10948,0.113403094988,8704,2909,true,1,0.0965405750275 +bf16,1,17,1,256,17,control_optimized_fused,1,10,100,61.5798854828,10948,0.177785325747,8704,2909,true,1,0.0615798854828 +bf16,1,17,1,256,17,optimized_fused,1,11,100,30.1954293251,10948,0.362571430336,8704,2909,true,1,0.0301954293251 +bf16,1,17,1,256,17,control_shuffle32_fused,1,12,100,301.233825684,10948,0.0363438600401,8704,2909,true,1,0.301233825684 +bf16,1,17,1,256,17,shuffle32_fused,1,13,100,61.4272022247,10948,0.178227228386,8704,2909,true,1,0.0614272022247 +bf16,1,17,1,256,17,baseline_transform,1,14,100,10.0178277493,17408,1.73770206833,8704,2909,true,1,0.0100178277493 +bf16,1,17,1,256,17,optimized_transform,2,0,100,10.7325720787,17408,1.62197839179,8704,2909,true,1,0.0107325720787 +bf16,1,17,1,256,17,control_shuffle32_transform,2,1,100,11.7625141144,17408,1.47995571616,8704,2909,true,1,0.0117625141144 +bf16,1,17,1,256,17,shuffle32_transform,2,2,100,11.9620573521,17408,1.45526806031,8704,2909,true,1,0.0119620573521 +bf16,1,17,1,256,17,baseline_split,2,3,100,101.322746277,28356,0.279858186261,8704,2909,true,1,0.101322746277 +bf16,1,17,1,256,17,control_optimized_split,2,4,100,67.4409151077,28356,0.420456928182,8704,2909,true,1,0.0674409151077 +bf16,1,17,1,256,17,optimized_split,2,5,100,36.1821699142,28356,0.783700924163,8704,2909,true,1,0.0361821699142 +bf16,1,17,1,256,17,control_shuffle32_split,2,6,100,305.383090973,28356,0.0928538640095,8704,2909,true,1,0.305383090973 +bf16,1,17,1,256,17,shuffle32_split,2,7,100,68.1046867371,28356,0.41635901079,8704,2909,true,1,0.0681046867371 +bf16,1,17,1,256,17,baseline_fused,2,8,100,96.493024826,10948,0.113458978198,8704,2909,true,1,0.096493024826 +bf16,1,17,1,256,17,control_optimized_fused,2,9,100,61.5369176865,10948,0.177909463321,8704,2909,true,1,0.0615369176865 +bf16,1,17,1,256,17,optimized_fused,2,10,100,30.2543997765,10948,0.361864723177,8704,2909,true,1,0.0302543997765 +bf16,1,17,1,256,17,control_shuffle32_fused,2,11,100,301.213703156,10948,0.0363462879853,8704,2909,true,1,0.301213703156 +bf16,1,17,1,256,17,shuffle32_fused,2,12,100,61.4756584167,10948,0.178086746559,8704,2909,true,1,0.0614756584167 +bf16,1,17,1,256,17,baseline_transform,2,13,100,10.0336003304,17408,1.734970442,8704,2909,true,1,0.0100336003304 +bf16,1,17,1,256,17,control_optimized_transform,2,14,100,10.7334852219,17408,1.6218404032,8704,2909,true,1,0.0107334852219 +bf16,1,17,1,256,17,control_shuffle32_transform,3,0,100,11.7248010635,17408,1.48471602253,8704,2909,true,1,0.0117248010635 +bf16,1,17,1,256,17,shuffle32_transform,3,1,100,11.7791998386,17408,1.47785929762,8704,2909,true,1,0.0117791998386 +bf16,1,17,1,256,17,baseline_split,3,2,100,101.293716431,28356,0.279938391039,8704,2909,true,1,0.101293716431 +bf16,1,17,1,256,17,control_optimized_split,3,3,100,67.4043416977,28356,0.420685066953,8704,2909,true,1,0.0674043416977 +bf16,1,17,1,256,17,optimized_split,3,4,100,36.2061715126,28356,0.783181397407,8704,2909,true,1,0.0362061715126 +bf16,1,17,1,256,17,control_shuffle32_split,3,5,100,306.046619415,28356,0.0926525509551,8704,2909,true,1,0.306046619415 +bf16,1,17,1,256,17,shuffle32_split,3,6,100,68.722281456,28356,0.412617267635,8704,2909,true,1,0.068722281456 +bf16,1,17,1,256,17,baseline_fused,3,7,100,96.5451431274,10948,0.113397729242,8704,2909,true,1,0.0965451431274 +bf16,1,17,1,256,17,control_optimized_fused,3,8,100,61.5892553329,10948,0.177758278466,8704,2909,true,1,0.0615892553329 +bf16,1,17,1,256,17,optimized_fused,3,9,100,30.1972556114,10948,0.36254950254,8704,2909,true,1,0.0301972556114 +bf16,1,17,1,256,17,control_shuffle32_fused,3,10,100,301.223773956,10948,0.0363450728215,8704,2909,true,1,0.301223773956 +bf16,1,17,1,256,17,shuffle32_fused,3,11,100,61.4724588394,10948,0.178096015788,8704,2909,true,1,0.0614724588394 +bf16,1,17,1,256,17,baseline_transform,3,12,100,10.1115429401,17408,1.72159680308,8704,2909,true,1,0.0101115429401 +bf16,1,17,1,256,17,control_optimized_transform,3,13,100,10.8870851994,17408,1.598958737,8704,2909,true,1,0.0108870851994 +bf16,1,17,1,256,17,optimized_transform,3,14,100,10.7581710815,17408,1.61811890405,8704,2909,true,1,0.0107581710815 +bf16,1,17,1,256,17,shuffle32_transform,4,0,100,11.8105149269,17408,1.47394081526,8704,2909,true,1,0.0118105149269 +bf16,1,17,1,256,17,baseline_split,4,1,100,101.31108284,28356,0.27989040493,8704,2909,true,1,0.10131108284 +bf16,1,17,1,256,17,control_optimized_split,4,2,100,67.372341156,28356,0.420884884115,8704,2909,true,1,0.067372341156 +bf16,1,17,1,256,17,optimized_split,4,3,100,36.2523436546,28356,0.782183912581,8704,2909,true,1,0.0362523436546 +bf16,1,17,1,256,17,control_shuffle32_split,4,4,100,305.429019928,28356,0.0928399010896,8704,2909,true,1,0.305429019928 +bf16,1,17,1,256,17,shuffle32_split,4,5,100,68.1458282471,28356,0.416107643408,8704,2909,true,1,0.0681458282471 +bf16,1,17,1,256,17,baseline_fused,4,6,100,96.5282344818,10948,0.113417592881,8704,2909,true,1,0.0965282344818 +bf16,1,17,1,256,17,control_optimized_fused,4,7,100,61.5778303146,10948,0.177791259355,8704,2909,true,1,0.0615778303146 +bf16,1,17,1,256,17,optimized_fused,4,8,100,30.2326869965,10948,0.362124610402,8704,2909,true,1,0.0302326869965 +bf16,1,17,1,256,17,control_shuffle32_fused,4,9,100,301.218967438,10948,0.0363456527759,8704,2909,true,1,0.301218967438 +bf16,1,17,1,256,17,shuffle32_fused,4,10,100,61.4754295349,10948,0.178087409601,8704,2909,true,1,0.0614754295349 +bf16,1,17,1,256,17,baseline_transform,4,11,100,10.0747430325,17408,1.72788526158,8704,2909,true,1,0.0100747430325 +bf16,1,17,1,256,17,control_optimized_transform,4,12,100,10.6804573536,17408,1.62989274932,8704,2909,true,1,0.0106804573536 +bf16,1,17,1,256,17,optimized_transform,4,13,100,10.7019424438,17408,1.62662059634,8704,2909,true,1,0.0107019424438 +bf16,1,17,1,256,17,control_shuffle32_transform,4,14,100,11.767771244,17408,1.4792945613,8704,2909,true,1,0.011767771244 +bf16,1,257,1,256,257,baseline_transform,0,0,100,10.4934859276,263168,25.0791778648,131584,2909,true,1,0.0104934859276 +bf16,1,257,1,256,257,control_optimized_transform,0,1,100,10.9472000599,263168,24.0397543262,131584,2909,true,1,0.0109472000599 +bf16,1,257,1,256,257,optimized_transform,0,2,100,10.9801137447,263168,23.9676934245,131584,2909,true,1,0.0109801137447 +bf16,1,257,1,256,257,control_shuffle32_transform,0,3,100,13.6841142178,263168,19.2316430433,131584,2909,true,1,0.0136841142178 +bf16,1,257,1,256,257,shuffle32_transform,0,4,100,13.8043427467,263168,19.0641455974,131584,2909,true,1,0.0138043427467 +bf16,1,257,1,256,257,baseline_split,0,5,100,104.06170845,428676,4.11944034346,131584,2909,true,1,0.10406170845 +bf16,1,257,1,256,257,control_optimized_split,0,6,100,69.7078847885,428676,6.14960561923,131584,2909,true,1,0.0697078847885 +bf16,1,257,1,256,257,optimized_split,0,7,100,36.8292570114,428676,11.6395505852,131584,2909,true,1,0.0368292570114 +bf16,1,257,1,256,257,control_shuffle32_split,0,8,100,328.982391357,428676,1.30303630608,131584,2909,true,1,0.328982391357 +bf16,1,257,1,256,257,shuffle32_split,0,9,100,76.7714309692,428676,5.58379588068,131584,2909,true,1,0.0767714309692 +bf16,1,257,1,256,257,baseline_fused,0,10,100,98.8267421722,165508,1.67472888777,131584,2909,true,1,0.0988267421722 +bf16,1,257,1,256,257,control_optimized_fused,0,11,100,63.7739419937,165508,2.59522925549,131584,2909,true,1,0.0637739419937 +bf16,1,257,1,256,257,optimized_fused,0,12,100,30.7522296906,165508,5.38198373469,131584,2909,true,1,0.0307522296906 +bf16,1,257,1,256,257,control_shuffle32_fused,0,13,100,321.570739746,165508,0.514686131365,131584,2909,true,1,0.321570739746 +bf16,1,257,1,256,257,shuffle32_fused,0,14,100,68.3753156662,165508,2.42058114668,131584,2909,true,1,0.0683753156662 +bf16,1,257,1,256,257,control_optimized_transform,1,0,100,11.0127997398,263168,23.8965572985,131584,2909,true,1,0.0110127997398 +bf16,1,257,1,256,257,optimized_transform,1,1,100,11.0144007206,263168,23.8930838523,131584,2909,true,1,0.0110144007206 +bf16,1,257,1,256,257,control_shuffle32_transform,1,2,100,13.6436569691,263168,19.2886702294,131584,2909,true,1,0.0136436569691 +bf16,1,257,1,256,257,shuffle32_transform,1,3,100,13.8498282433,263168,19.001535281,131584,2909,true,1,0.0138498282433 +bf16,1,257,1,256,257,baseline_split,1,4,100,104.059200287,428676,4.11953963531,131584,2909,true,1,0.104059200287 +bf16,1,257,1,256,257,control_optimized_split,1,5,100,69.6687984467,428676,6.1530557374,131584,2909,true,1,0.0696687984467 +bf16,1,257,1,256,257,optimized_split,1,6,100,36.8761134148,428676,11.6247608629,131584,2909,true,1,0.0368761134148 +bf16,1,257,1,256,257,control_shuffle32_split,1,7,100,328.957977295,428676,1.30313301269,131584,2909,true,1,0.328957977295 +bf16,1,257,1,256,257,shuffle32_split,1,8,100,76.7730283737,428676,5.58367969951,131584,2909,true,1,0.0767730283737 +bf16,1,257,1,256,257,baseline_fused,1,9,100,98.8912010193,165508,1.67363727302,131584,2909,true,1,0.0988912010193 +bf16,1,257,1,256,257,control_optimized_fused,1,10,100,63.7702894211,165508,2.59537790251,131584,2909,true,1,0.0637702894211 +bf16,1,257,1,256,257,optimized_fused,1,11,100,30.700571537,165508,5.39103970102,131584,2909,true,1,0.030700571537 +bf16,1,257,1,256,257,control_shuffle32_fused,1,12,100,321.569824219,165508,0.514687596705,131584,2909,true,1,0.321569824219 +bf16,1,257,1,256,257,shuffle32_fused,1,13,100,68.3762264252,165508,2.42054890498,131584,2909,true,1,0.0683762264252 +bf16,1,257,1,256,257,baseline_transform,1,14,100,10.570743084,263168,24.8958846043,131584,2909,true,1,0.010570743084 +bf16,1,257,1,256,257,optimized_transform,2,0,100,11.0948574543,263168,23.7198180404,131584,2909,true,1,0.0110948574543 +bf16,1,257,1,256,257,control_shuffle32_transform,2,1,100,13.7026286125,263168,19.2056580852,131584,2909,true,1,0.0137026286125 +bf16,1,257,1,256,257,shuffle32_transform,2,2,100,13.8411426544,263168,19.0134591176,131584,2909,true,1,0.0138411426544 +bf16,1,257,1,256,257,baseline_split,2,3,100,104.09828186,428676,4.1179930383,131584,2909,true,1,0.10409828186 +bf16,1,257,1,256,257,control_optimized_split,2,4,100,69.7476577759,428676,6.1460988608,131584,2909,true,1,0.0697476577759 +bf16,1,257,1,256,257,optimized_split,2,5,100,36.9120001793,428676,11.6134589813,131584,2909,true,1,0.0369120001793 +bf16,1,257,1,256,257,control_shuffle32_split,2,6,100,328.300094604,428676,1.30574436939,131584,2909,true,1,0.328300094604 +bf16,1,257,1,256,257,shuffle32_split,2,7,100,76.7629718781,428676,5.58441120129,131584,2909,true,1,0.0767629718781 +bf16,1,257,1,256,257,baseline_fused,2,8,100,98.8772583008,165508,1.67387327323,131584,2909,true,1,0.0988772583008 +bf16,1,257,1,256,257,control_optimized_fused,2,9,100,63.7700557709,165508,2.59538741184,131584,2909,true,1,0.0637700557709 +bf16,1,257,1,256,257,optimized_fused,2,10,100,30.745370388,165508,5.38318445708,131584,2909,true,1,0.030745370388 +bf16,1,257,1,256,257,control_shuffle32_fused,2,11,100,321.594276428,165508,0.514648462772,131584,2909,true,1,0.321594276428 +bf16,1,257,1,256,257,shuffle32_fused,2,12,100,68.3780574799,165508,2.42048408656,131584,2909,true,1,0.0683780574799 +bf16,1,257,1,256,257,baseline_transform,2,13,100,10.5067420006,263168,25.0475361426,131584,2909,true,1,0.0105067420006 +bf16,1,257,1,256,257,control_optimized_transform,2,14,100,11.0873138905,263168,23.7359564814,131584,2909,true,1,0.0110873138905 +bf16,1,257,1,256,257,control_shuffle32_transform,3,0,100,13.7325716019,263168,19.1637813827,131584,2909,true,1,0.0137325716019 +bf16,1,257,1,256,257,shuffle32_transform,3,1,100,13.9622855186,263168,18.8484900734,131584,2909,true,1,0.0139622855186 +bf16,1,257,1,256,257,baseline_split,3,2,100,104.105148315,428676,4.11772142816,131584,2909,true,1,0.104105148315 +bf16,1,257,1,256,257,control_optimized_split,3,3,100,69.6687984467,428676,6.1530557374,131584,2909,true,1,0.0696687984467 +bf16,1,257,1,256,257,optimized_split,3,4,100,36.8859434128,428676,11.6216628975,131584,2909,true,1,0.0368859434128 +bf16,1,257,1,256,257,control_shuffle32_split,3,5,100,328.416213989,428676,1.30528269233,131584,2909,true,1,0.328416213989 +bf16,1,257,1,256,257,shuffle32_split,3,6,100,76.7878866196,428676,5.58259927277,131584,2909,true,1,0.0767878866196 +bf16,1,257,1,256,257,baseline_fused,3,7,100,98.8349723816,165508,1.67458942935,131584,2909,true,1,0.0988349723816 +bf16,1,257,1,256,257,control_optimized_fused,3,8,100,63.7778282166,165508,2.59507111841,131584,2909,true,1,0.0637778282166 +bf16,1,257,1,256,257,optimized_fused,3,9,100,30.7565712929,165508,5.38122401304,131584,2909,true,1,0.0307565712929 +bf16,1,257,1,256,257,control_shuffle32_fused,3,10,100,321.576652527,165508,0.514676667909,131584,2909,true,1,0.321576652527 +bf16,1,257,1,256,257,shuffle32_fused,3,11,100,68.3282279968,165508,2.42224926436,131584,2909,true,1,0.0683282279968 +bf16,1,257,1,256,257,baseline_transform,3,12,100,10.4859423637,263168,25.0972197702,131584,2909,true,1,0.0104859423637 +bf16,1,257,1,256,257,control_optimized_transform,3,13,100,11.0020565987,263168,23.9198914894,131584,2909,true,1,0.0110020565987 +bf16,1,257,1,256,257,optimized_transform,3,14,100,11.0557711124,263168,23.8036765888,131584,2909,true,1,0.0110557711124 +bf16,1,257,1,256,257,shuffle32_transform,4,0,100,13.8948571682,263168,18.9399571953,131584,2909,true,1,0.0138948571682 +bf16,1,257,1,256,257,baseline_split,4,1,100,104.077711105,428676,4.11880695153,131584,2909,true,1,0.104077711105 +bf16,1,257,1,256,257,control_optimized_split,4,2,100,69.6669721603,428676,6.15321703681,131584,2909,true,1,0.0696669721603 +bf16,1,257,1,256,257,optimized_split,4,3,100,36.8489146233,428676,11.6333412906,131584,2909,true,1,0.0368489146233 +bf16,1,257,1,256,257,control_shuffle32_split,4,4,100,328.396568298,428676,1.30536077835,131584,2909,true,1,0.328396568298 +bf16,1,257,1,256,257,shuffle32_split,4,5,100,76.8022823334,428676,5.58155287807,131584,2909,true,1,0.0768022823334 +bf16,1,257,1,256,257,baseline_fused,4,6,100,98.8751983643,165508,1.67390814621,131584,2909,true,1,0.0988751983643 +bf16,1,257,1,256,257,control_optimized_fused,4,7,100,63.7184000015,165508,2.59749146237,131584,2909,true,1,0.0637184000015 +bf16,1,257,1,256,257,optimized_fused,4,8,100,30.6390857697,165508,5.40185830753,131584,2909,true,1,0.0306390857697 +bf16,1,257,1,256,257,control_shuffle32_fused,4,9,100,321.585388184,165508,0.514662687054,131584,2909,true,1,0.321585388184 +bf16,1,257,1,256,257,shuffle32_fused,4,10,100,68.327088356,165508,2.42228966552,131584,2909,true,1,0.068327088356 +bf16,1,257,1,256,257,baseline_transform,4,11,100,10.6413710117,263168,24.7306479315,131584,2909,true,1,0.0106413710117 +bf16,1,257,1,256,257,control_optimized_transform,4,12,100,11.0182857513,263168,23.8846591874,131584,2909,true,1,0.0110182857513 +bf16,1,257,1,256,257,optimized_transform,4,13,100,11.089142561,263168,23.7320422705,131584,2909,true,1,0.011089142561 +bf16,1,257,1,256,257,control_shuffle32_transform,4,14,100,13.742171526,263168,19.1503940628,131584,2909,true,1,0.013742171526 +bf16,4,64,16,256,4096,baseline_transform,0,0,100,49.2649173737,4194304,85.137745552,2097152,2909,true,1,0.0492649173737 +bf16,4,64,16,256,4096,control_optimized_transform,0,1,100,37.3028564453,4194304,112.439217789,2097152,2909,true,1,0.0373028564453 +bf16,4,64,16,256,4096,optimized_transform,0,2,100,37.4217152596,4194304,112.082088459,2097152,2909,true,1,0.0374217152596 +bf16,4,64,16,256,4096,control_shuffle32_transform,0,3,100,22.4383997917,4194304,186.925272699,2097152,2909,true,1,0.0224383997917 +bf16,4,64,16,256,4096,shuffle32_transform,0,4,100,22.4443435669,4194304,186.875770614,2097152,2909,true,1,0.0224443435669 +bf16,4,64,16,256,4096,baseline_split,0,5,100,1145.11863708,6832128,5.96630582958,2097152,2909,true,1,1.14511863708 +bf16,4,64,16,256,4096,control_optimized_split,0,6,100,499.791297913,6832128,13.6699618992,2097152,2909,true,1,0.499791297913 +bf16,4,64,16,256,4096,optimized_split,0,7,100,188.719539642,6832128,36.2025469803,2097152,2909,true,1,0.188719539642 +bf16,4,64,16,256,4096,control_shuffle32_split,0,8,100,842.68371582,6832128,8.10758279973,2097152,2909,true,1,0.84268371582 +bf16,4,64,16,256,4096,shuffle32_split,0,9,100,149.500570297,6832128,45.6996785124,2097152,2909,true,1,0.149500570297 +bf16,4,64,16,256,4096,baseline_fused,0,10,100,1138.36502075,2637824,2.31720401797,2097152,2909,true,1,1.13836502075 +bf16,4,64,16,256,4096,control_optimized_fused,0,11,100,484.753837585,2637824,5.4415742496,2097152,2909,true,1,0.484753837585 +bf16,4,64,16,256,4096,optimized_fused,0,12,100,171.889839172,2637824,15.3460147074,2097152,2909,true,1,0.171889839172 +bf16,4,64,16,256,4096,control_shuffle32_fused,0,13,100,831.012573242,2637824,3.17422874808,2097152,2909,true,1,0.831012573242 +bf16,4,64,16,256,4096,shuffle32_fused,0,14,100,131.430854797,2637824,20.0700513138,2097152,2909,true,1,0.131430854797 +bf16,4,64,16,256,4096,control_optimized_transform,1,0,100,37.3933720589,4194304,112.167043758,2097152,2909,true,1,0.0373933720589 +bf16,4,64,16,256,4096,optimized_transform,1,1,100,37.353143692,4194304,112.287844755,2097152,2909,true,1,0.037353143692 +bf16,4,64,16,256,4096,control_shuffle32_transform,1,2,100,21.9309735298,4194304,191.250242234,2097152,2909,true,1,0.0219309735298 +bf16,4,64,16,256,4096,shuffle32_transform,1,3,100,22.5145125389,4194304,186.293351577,2097152,2909,true,1,0.0225145125389 +bf16,4,64,16,256,4096,baseline_split,1,4,100,1145.01739502,6832128,5.9668333684,2097152,2909,true,1,1.14501739502 +bf16,4,64,16,256,4096,control_optimized_split,1,5,100,500.291900635,6832128,13.6562834444,2097152,2909,true,1,0.500291900635 +bf16,4,64,16,256,4096,optimized_split,1,6,100,188.736915588,6832128,36.1992140155,2097152,2909,true,1,0.188736915588 +bf16,4,64,16,256,4096,control_shuffle32_split,1,7,100,842.790374756,6832128,8.10655674844,2097152,2909,true,1,0.842790374756 +bf16,4,64,16,256,4096,shuffle32_split,1,8,100,149.318180084,6832128,45.7555000747,2097152,2909,true,1,0.149318180084 +bf16,4,64,16,256,4096,baseline_fused,1,9,100,1136.62902832,2637824,2.32074312223,2097152,2909,true,1,1.13662902832 +bf16,4,64,16,256,4096,control_optimized_fused,1,10,100,488.225135803,2637824,5.40288446161,2097152,2909,true,1,0.488225135803 +bf16,4,64,16,256,4096,optimized_fused,1,11,100,171.692333221,2637824,15.3636679665,2097152,2909,true,1,0.171692333221 +bf16,4,64,16,256,4096,control_shuffle32_fused,1,12,100,833.356323242,2637824,3.16530147601,2097152,2909,true,1,0.833356323242 +bf16,4,64,16,256,4096,shuffle32_fused,1,13,100,130.929365158,2637824,20.1469242352,2097152,2909,true,1,0.130929365158 +bf16,4,64,16,256,4096,baseline_transform,1,14,100,49.2432022095,4194304,85.1752894168,2097152,2909,true,1,0.0492432022095 +bf16,4,64,16,256,4096,optimized_transform,2,0,100,37.395427227,4194304,112.160879311,2097152,2909,true,1,0.037395427227 +bf16,4,64,16,256,4096,control_shuffle32_transform,2,1,100,22.4105143547,4194304,187.1578641,2097152,2909,true,1,0.0224105143547 +bf16,4,64,16,256,4096,shuffle32_transform,2,2,100,21.8896007538,4194304,191.611717691,2097152,2909,true,1,0.0218896007538 +bf16,4,64,16,256,4096,baseline_split,2,3,100,1144.11514282,6832128,5.97153882882,2097152,2909,true,1,1.14411514282 +bf16,4,64,16,256,4096,control_optimized_split,2,4,100,501.005935669,6832128,13.6368204718,2097152,2909,true,1,0.501005935669 +bf16,4,64,16,256,4096,optimized_split,2,5,100,188.8048172,6832128,36.1861953595,2097152,2909,true,1,0.1888048172 +bf16,4,64,16,256,4096,control_shuffle32_split,2,6,100,840.151748657,6832128,8.13201663976,2097152,2909,true,1,0.840151748657 +bf16,4,64,16,256,4096,shuffle32_split,2,7,100,149.463768005,6832128,45.7109310917,2097152,2909,true,1,0.149463768005 +bf16,4,64,16,256,4096,baseline_fused,2,8,100,1137.22442627,2637824,2.3195280888,2097152,2909,true,1,1.13722442627 +bf16,4,64,16,256,4096,control_optimized_fused,2,9,100,484.704437256,2637824,5.44212884647,2097152,2909,true,1,0.484704437256 +bf16,4,64,16,256,4096,optimized_fused,2,10,100,171.552448273,2637824,15.3761955983,2097152,2909,true,1,0.171552448273 +bf16,4,64,16,256,4096,control_shuffle32_fused,2,11,100,839.16519165,2637824,3.14339062946,2097152,2909,true,1,0.83916519165 +bf16,4,64,16,256,4096,shuffle32_fused,2,12,100,131.356563568,2637824,20.0814023171,2097152,2909,true,1,0.131356563568 +bf16,4,64,16,256,4096,baseline_transform,2,13,100,49.296913147,4194304,85.0824875686,2097152,2909,true,1,0.049296913147 +bf16,4,64,16,256,4096,control_optimized_transform,2,14,100,37.2848010063,4194304,112.493667307,2097152,2909,true,1,0.0372848010063 +bf16,4,64,16,256,4096,control_shuffle32_transform,3,0,100,22.2790837288,4194304,188.261961356,2097152,2909,true,1,0.0222790837288 +bf16,4,64,16,256,4096,shuffle32_transform,3,1,100,22.4146294594,4194304,187.123503764,2097152,2909,true,1,0.0224146294594 +bf16,4,64,16,256,4096,baseline_split,3,2,100,1145.09552002,6832128,5.96642627672,2097152,2909,true,1,1.14509552002 +bf16,4,64,16,256,4096,control_optimized_split,3,3,100,500.098724365,6832128,13.6615585426,2097152,2909,true,1,0.500098724365 +bf16,4,64,16,256,4096,optimized_split,3,4,100,188.804111481,6832128,36.1863306176,2097152,2909,true,1,0.188804111481 +bf16,4,64,16,256,4096,control_shuffle32_split,3,5,100,837.845077515,6832128,8.1544048934,2097152,2909,true,1,0.837845077515 +bf16,4,64,16,256,4096,shuffle32_split,3,6,100,149.30811882,6832128,45.7585833509,2097152,2909,true,1,0.14930811882 +bf16,4,64,16,256,4096,baseline_fused,3,7,100,1137.36824036,2637824,2.31923479697,2097152,2909,true,1,1.13736824036 +bf16,4,64,16,256,4096,control_optimized_fused,3,8,100,485.969390869,2637824,5.42796326181,2097152,2909,true,1,0.485969390869 +bf16,4,64,16,256,4096,optimized_fused,3,9,100,171.728000641,2637824,15.3604769761,2097152,2909,true,1,0.171728000641 +bf16,4,64,16,256,4096,control_shuffle32_fused,3,10,100,836.472015381,2637824,3.15351135662,2097152,2909,true,1,0.836472015381 +bf16,4,64,16,256,4096,shuffle32_fused,3,11,100,131.472921371,2637824,20.0636296241,2097152,2909,true,1,0.131472921371 +bf16,4,64,16,256,4096,baseline_transform,3,12,100,49.2054891586,4194304,85.2405711582,2097152,2909,true,1,0.0492054891586 +bf16,4,64,16,256,4096,control_optimized_transform,3,13,100,37.355659008,4194304,112.280283935,2097152,2909,true,1,0.037355659008 +bf16,4,64,16,256,4096,optimized_transform,3,14,100,37.424685955,4194304,112.07319161,2097152,2909,true,1,0.037424685955 +bf16,4,64,16,256,4096,shuffle32_transform,4,0,100,22.337372303,4194304,187.7706985,2097152,2909,true,1,0.022337372303 +bf16,4,64,16,256,4096,baseline_split,4,1,100,1144.90394592,6832128,5.96742462486,2097152,2909,true,1,1.14490394592 +bf16,4,64,16,256,4096,control_optimized_split,4,2,100,501.928215027,6832128,13.6117631874,2097152,2909,true,1,0.501928215027 +bf16,4,64,16,256,4096,optimized_split,4,3,100,188.961372375,6832128,36.1562149666,2097152,2909,true,1,0.188961372375 +bf16,4,64,16,256,4096,control_shuffle32_split,4,4,100,839.621276855,6832128,8.13715443895,2097152,2909,true,1,0.839621276855 +bf16,4,64,16,256,4096,shuffle32_split,4,5,100,149.401607513,6832128,45.7299497222,2097152,2909,true,1,0.149401607513 +bf16,4,64,16,256,4096,baseline_fused,4,6,100,1136.54586792,2637824,2.32091292965,2097152,2909,true,1,1.13654586792 +bf16,4,64,16,256,4096,control_optimized_fused,4,7,100,485.598869324,2637824,5.43210490517,2097152,2909,true,1,0.485598869324 +bf16,4,64,16,256,4096,optimized_fused,4,8,100,171.668815613,2637824,15.3657726978,2097152,2909,true,1,0.171668815613 +bf16,4,64,16,256,4096,control_shuffle32_fused,4,9,100,832.301940918,2637824,3.16931136444,2097152,2909,true,1,0.832301940918 +bf16,4,64,16,256,4096,shuffle32_fused,4,10,100,130.382404327,2637824,20.2314416091,2097152,2909,true,1,0.130382404327 +bf16,4,64,16,256,4096,baseline_transform,4,11,100,49.2322301865,4194304,85.1942718035,2097152,2909,true,1,0.0492322301865 +bf16,4,64,16,256,4096,control_optimized_transform,4,12,100,37.2429704666,4194304,112.620017884,2097152,2909,true,1,0.0372429704666 +bf16,4,64,16,256,4096,optimized_transform,4,13,100,37.3298287392,4194304,112.357975958,2097152,2909,true,1,0.0373298287392 +bf16,4,64,16,256,4096,control_shuffle32_transform,4,14,100,23.2015991211,4194304,180.776505021,2097152,2909,true,1,0.0232015991211 +bf16,16,64,16,256,16384,baseline_transform,0,0,100,170.639762878,16777216,98.3194990253,8388608,2909,true,1,0.170639762878 +bf16,16,64,16,256,16384,control_optimized_transform,0,1,100,120.598402023,16777216,139.116403854,8388608,2909,true,1,0.120598402023 +bf16,16,64,16,256,16384,optimized_transform,0,2,100,120.683202744,16777216,139.018650637,8388608,2909,true,1,0.120683202744 +bf16,16,64,16,256,16384,control_shuffle32_transform,0,3,100,43.0507469177,16777216,389.707895941,8388608,2909,true,1,0.0430507469177 +bf16,16,64,16,256,16384,shuffle32_transform,0,4,100,43.2370281219,16777216,388.028889328,8388608,2909,true,1,0.0432370281219 +bf16,16,64,16,256,16384,baseline_split,0,5,100,4268.43078613,27328512,6.40247279838,8388608,2909,true,1,4.26843078613 +bf16,16,64,16,256,16384,control_optimized_split,0,6,100,1869.99084473,27328512,14.6142490895,8388608,2909,true,1,1.86999084473 +bf16,16,64,16,256,16384,optimized_split,0,7,100,677.841415405,27328512,40.3169699858,8388608,2909,true,1,0.677841415405 +bf16,16,64,16,256,16384,control_shuffle32_split,0,8,100,3209.85198975,27328512,8.51394771077,8388608,2909,true,1,3.20985198975 +bf16,16,64,16,256,16384,shuffle32_split,0,9,100,436.122283936,27328512,62.662498585,8388608,2909,true,1,0.436122283936 +bf16,16,64,16,256,16384,baseline_fused,0,10,100,4270.7989502,10551296,2.47056724586,8388608,2909,true,1,4.2707989502 +bf16,16,64,16,256,16384,control_optimized_fused,0,11,100,1823.46969604,10551296,5.78638406927,8388608,2909,true,1,1.82346969604 +bf16,16,64,16,256,16384,optimized_fused,0,12,100,631.613960266,10551296,16.7052925739,8388608,2909,true,1,0.631613960266 +bf16,16,64,16,256,16384,control_shuffle32_fused,0,13,100,3041.81427002,10551296,3.46875090435,8388608,2909,true,1,3.04181427002 +bf16,16,64,16,256,16384,shuffle32_fused,0,14,100,391.185836792,10551296,26.9725920717,8388608,2909,true,1,0.391185836792 +bf16,16,64,16,256,16384,control_optimized_transform,1,0,100,120.690059662,16777216,139.010752393,8388608,2909,true,1,0.120690059662 +bf16,16,64,16,256,16384,optimized_transform,1,1,100,120.731420517,16777216,138.963129301,8388608,2909,true,1,0.120731420517 +bf16,16,64,16,256,16384,control_shuffle32_transform,1,2,100,43.0806875229,16777216,389.437053229,8388608,2909,true,1,0.0430806875229 +bf16,16,64,16,256,16384,shuffle32_transform,1,3,100,43.1762266159,16777216,388.575318294,8388608,2909,true,1,0.0431762266159 +bf16,16,64,16,256,16384,baseline_split,1,4,100,4270.22796631,27328512,6.39977823564,8388608,2909,true,1,4.27022796631 +bf16,16,64,16,256,16384,control_optimized_split,1,5,100,1870.10635376,27328512,14.6133464255,8388608,2909,true,1,1.87010635376 +bf16,16,64,16,256,16384,optimized_split,1,6,100,677.92388916,27328512,40.3120651698,8388608,2909,true,1,0.67792388916 +bf16,16,64,16,256,16384,control_shuffle32_split,1,7,100,3178.72833252,27328512,8.59730972302,8388608,2909,true,1,3.17872833252 +bf16,16,64,16,256,16384,shuffle32_split,1,8,100,436.400222778,27328512,62.6225894799,8388608,2909,true,1,0.436400222778 +bf16,16,64,16,256,16384,baseline_fused,1,9,100,4274.03869629,10551296,2.46869454157,8388608,2909,true,1,4.27403869629 +bf16,16,64,16,256,16384,control_optimized_fused,1,10,100,1823.49151611,10551296,5.78631482887,8388608,2909,true,1,1.82349151611 +bf16,16,64,16,256,16384,optimized_fused,1,11,100,631.592216492,10551296,16.7058676857,8388608,2909,true,1,0.631592216492 +bf16,16,64,16,256,16384,control_shuffle32_fused,1,12,100,3042.56256104,10551296,3.46789779613,8388608,2909,true,1,3.04256256104 +bf16,16,64,16,256,16384,shuffle32_fused,1,13,100,391.27407074,10551296,26.9665096388,8388608,2909,true,1,0.39127407074 +bf16,16,64,16,256,16384,baseline_transform,1,14,100,170.64617157,16777216,98.3158065936,8388608,2909,true,1,0.17064617157 +bf16,16,64,16,256,16384,optimized_transform,2,0,100,120.743312836,16777216,138.949442466,8388608,2909,true,1,0.120743312836 +bf16,16,64,16,256,16384,control_shuffle32_transform,2,1,100,43.1081151962,16777216,389.189272684,8388608,2909,true,1,0.0431081151962 +bf16,16,64,16,256,16384,shuffle32_transform,2,2,100,43.1792020798,16777216,388.548541703,8388608,2909,true,1,0.0431792020798 +bf16,16,64,16,256,16384,baseline_split,2,3,100,4269.07989502,27328512,6.40149930946,8388608,2909,true,1,4.26907989502 +bf16,16,64,16,256,16384,control_optimized_split,2,4,100,1870.02807617,27328512,14.6139581262,8388608,2909,true,1,1.87002807617 +bf16,16,64,16,256,16384,optimized_split,2,5,100,677.95501709,27328512,40.3102142636,8388608,2909,true,1,0.67795501709 +bf16,16,64,16,256,16384,control_shuffle32_split,2,6,100,3205.2822876,27328512,8.52608586325,8388608,2909,true,1,3.2052822876 +bf16,16,64,16,256,16384,shuffle32_split,2,7,100,436.1769104,27328512,62.6546507813,8388608,2909,true,1,0.4361769104 +bf16,16,64,16,256,16384,baseline_fused,2,8,100,4272.46765137,10551296,2.4696023144,8388608,2909,true,1,4.27246765137 +bf16,16,64,16,256,16384,control_optimized_fused,2,9,100,1823.51791382,10551296,5.78623106471,8388608,2909,true,1,1.82351791382 +bf16,16,64,16,256,16384,optimized_fused,2,10,100,631.600914001,10551296,16.7056376362,8388608,2909,true,1,0.631600914001 +bf16,16,64,16,256,16384,control_shuffle32_fused,2,11,100,3041.89697266,10551296,3.46865659647,8388608,2909,true,1,3.04189697266 +bf16,16,64,16,256,16384,shuffle32_fused,2,12,100,391.127319336,10551296,26.9766275031,8388608,2909,true,1,0.391127319336 +bf16,16,64,16,256,16384,baseline_transform,2,13,100,170.65530777,16777216,98.3105431601,8388608,2909,true,1,0.17065530777 +bf16,16,64,16,256,16384,control_optimized_transform,2,14,100,120.694627762,16777216,139.005491057,8388608,2909,true,1,0.120694627762 +bf16,16,64,16,256,16384,control_shuffle32_transform,3,0,100,43.1051445007,16777216,389.216094606,8388608,2909,true,1,0.0431051445007 +bf16,16,64,16,256,16384,shuffle32_transform,3,1,100,43.2260560989,16777216,388.127382281,8388608,2909,true,1,0.0432260560989 +bf16,16,64,16,256,16384,baseline_split,3,2,100,4269.793396,27328512,6.40042959119,8388608,2909,true,1,4.269793396 +bf16,16,64,16,256,16384,control_optimized_split,3,3,100,1870.08743286,27328512,14.6134942783,8388608,2909,true,1,1.87008743286 +bf16,16,64,16,256,16384,optimized_split,3,4,100,677.908554077,27328512,40.3129770758,8388608,2909,true,1,0.677908554077 +bf16,16,64,16,256,16384,control_shuffle32_split,3,5,100,3180.52856445,27328512,8.59244350308,8388608,2909,true,1,3.18052856445 +bf16,16,64,16,256,16384,shuffle32_split,3,6,100,436.222419739,27328512,62.6481142725,8388608,2909,true,1,0.436222419739 +bf16,16,64,16,256,16384,baseline_fused,3,7,100,4268.67980957,10551296,2.47179373265,8388608,2909,true,1,4.26867980957 +bf16,16,64,16,256,16384,control_optimized_fused,3,8,100,1823.46420288,10551296,5.78640150069,8388608,2909,true,1,1.82346420288 +bf16,16,64,16,256,16384,optimized_fused,3,9,100,631.605682373,10551296,16.7055115153,8388608,2909,true,1,0.631605682373 +bf16,16,64,16,256,16384,control_shuffle32_fused,3,10,100,3042.02423096,10551296,3.46851149068,8388608,2909,true,1,3.04202423096 +bf16,16,64,16,256,16384,shuffle32_fused,3,11,100,391.282958984,10551296,26.9658970771,8388608,2909,true,1,0.391282958984 +bf16,16,64,16,256,16384,baseline_transform,3,12,100,170.631999969,16777216,98.3239720744,8388608,2909,true,1,0.170631999969 +bf16,16,64,16,256,16384,control_optimized_transform,3,13,100,120.664234161,16777216,139.040504559,8388608,2909,true,1,0.120664234161 +bf16,16,64,16,256,16384,optimized_transform,3,14,100,120.69234848,16777216,139.008116184,8388608,2909,true,1,0.12069234848 +bf16,16,64,16,256,16384,shuffle32_transform,4,0,100,43.1785106659,16777216,388.554763498,8388608,2909,true,1,0.0431785106659 +bf16,16,64,16,256,16384,baseline_split,4,1,100,4271.5838623,27328512,6.39774680328,8388608,2909,true,1,4.2715838623 +bf16,16,64,16,256,16384,control_optimized_split,4,2,100,1870.13381958,27328512,14.6131318058,8388608,2909,true,1,1.87013381958 +bf16,16,64,16,256,16384,optimized_split,4,3,100,677.829666138,27328512,40.3176688263,8388608,2909,true,1,0.677829666138 +bf16,16,64,16,256,16384,control_shuffle32_split,4,4,100,3183.60565186,27328512,8.58413854872,8388608,2909,true,1,3.18360565186 +bf16,16,64,16,256,16384,shuffle32_split,4,5,100,436.189231873,27328512,62.6528809129,8388608,2909,true,1,0.436189231873 +bf16,16,64,16,256,16384,baseline_fused,4,6,100,4272.67547607,10551296,2.46948219192,8388608,2909,true,1,4.27267547607 +bf16,16,64,16,256,16384,control_optimized_fused,4,7,100,1823.50830078,10551296,5.78626156814,8388608,2909,true,1,1.82350830078 +bf16,16,64,16,256,16384,optimized_fused,4,8,100,631.596832275,10551296,16.7057455972,8388608,2909,true,1,0.631596832275 +bf16,16,64,16,256,16384,control_shuffle32_fused,4,9,100,3044.8550415,10551296,3.46528680551,8388608,2909,true,1,3.0448550415 +bf16,16,64,16,256,16384,shuffle32_fused,4,10,100,390.836105347,10551296,26.9967279268,8388608,2909,true,1,0.390836105347 +bf16,16,64,16,256,16384,baseline_transform,4,11,100,170.526866913,16777216,98.3845906732,8388608,2909,true,1,0.170526866913 +bf16,16,64,16,256,16384,control_optimized_transform,4,12,100,120.613260269,16777216,139.099266221,8388608,2909,true,1,0.120613260269 +bf16,16,64,16,256,16384,optimized_transform,4,13,100,120.690507889,16777216,139.010236128,8388608,2909,true,1,0.120690507889 +bf16,16,64,16,256,16384,control_shuffle32_transform,4,14,100,43.0806875229,16777216,389.437053229,8388608,2909,true,1,0.0430806875229 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round2.log b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round2.log new file mode 100644 index 00000000..22c2e112 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round2.log @@ -0,0 +1,32 @@ +DEVICE name=MTT S4000 warp=128 runtime=50100 driver=50100 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=256 +PASS requested validation/benchmark operations completed diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round3.csv b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round3.csv new file mode 100644 index 00000000..4d5c1c4c --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round3.csv @@ -0,0 +1,2251 @@ +dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale,kernel_ms +fp16,1,1,1,64,1,baseline_transform,0,0,100,9.09074246883,256,0.028160516138,128,2909,true,1,0.00909074246883 +fp16,1,1,1,64,1,control_optimized_transform,0,1,100,9.24388587475,256,0.0276939810236,128,2909,true,1,0.00924388587475 +fp16,1,1,1,64,1,optimized_transform,0,2,100,9.22537088394,256,0.0277495618572,128,2909,true,1,0.00922537088394 +fp16,1,1,1,64,1,control_shuffle32_transform,0,3,100,8.01919996738,256,0.0319233840085,128,2909,true,1,0.00801919996738 +fp16,1,1,1,64,1,shuffle32_transform,0,4,100,7.86331415176,256,0.0325562472844,128,2909,true,1,0.00786331415176 +fp16,1,1,1,64,1,baseline_split,0,5,100,69.9149703979,420,0.00600729711547,128,2909,true,1,0.0699149703979 +fp16,1,1,1,64,1,control_optimized_split,0,6,100,64.0070867538,420,0.00656177341136,128,2909,true,1,0.0640070867538 +fp16,1,1,1,64,1,optimized_split,0,7,100,33.7442278862,420,0.012446573127,128,2909,true,1,0.0337442278862 +fp16,1,1,1,64,1,control_shuffle32_split,0,8,100,93.9005661011,420,0.00447281648492,128,2909,true,1,0.0939005661011 +fp16,1,1,1,64,1,shuffle32_split,0,9,100,35.4717707634,420,0.0118404012814,128,2909,true,1,0.0354717707634 +fp16,1,1,1,64,1,baseline_fused,0,10,100,65.4450273514,164,0.00250591995507,128,2909,true,1,0.0654450273514 +fp16,1,1,1,64,1,control_optimized_fused,0,11,100,58.670630455,164,0.00279526568452,128,2909,true,1,0.058670630455 +fp16,1,1,1,64,1,optimized_fused,0,12,100,28.5595417023,164,0.00574238906596,128,2909,true,1,0.0285595417023 +fp16,1,1,1,64,1,control_shuffle32_fused,0,13,100,89.2463970184,164,0.00183760919745,128,2909,true,1,0.0892463970184 +fp16,1,1,1,64,1,shuffle32_fused,0,14,100,29.7213721275,164,0.00551791482898,128,2909,true,1,0.0297213721275 +fp16,1,1,1,64,1,control_optimized_transform,1,0,100,9.3327999115,256,0.0274301391252,128,2909,true,1,0.0093327999115 +fp16,1,1,1,64,1,optimized_transform,1,1,100,9.28548574448,256,0.0275699093235,128,2909,true,1,0.00928548574448 +fp16,1,1,1,64,1,control_shuffle32_transform,1,2,100,7.9243427515,256,0.0323055183285,128,2909,true,1,0.0079243427515 +fp16,1,1,1,64,1,shuffle32_transform,1,3,100,8.07474315166,256,0.031703794807,128,2909,true,1,0.00807474315166 +fp16,1,1,1,64,1,baseline_split,1,4,100,69.9234294891,420,0.00600657037374,128,2909,true,1,0.0699234294891 +fp16,1,1,1,64,1,control_optimized_split,1,5,100,64.0991973877,420,0.00655234413404,128,2909,true,1,0.0640991973877 +fp16,1,1,1,64,1,optimized_split,1,6,100,33.7126851082,420,0.0124582185801,128,2909,true,1,0.0337126851082 +fp16,1,1,1,64,1,control_shuffle32_split,1,7,100,93.8987445831,420,0.00447290325195,128,2909,true,1,0.0938987445831 +fp16,1,1,1,64,1,shuffle32_split,1,8,100,34.8082280159,420,0.0120661126389,128,2909,true,1,0.0348082280159 +fp16,1,1,1,64,1,baseline_fused,1,9,100,65.4511976242,164,0.00250568371478,128,2909,true,1,0.0654511976242 +fp16,1,1,1,64,1,control_optimized_fused,1,10,100,58.6774873734,164,0.0027949390361,128,2909,true,1,0.0586774873734 +fp16,1,1,1,64,1,optimized_fused,1,11,100,28.5858297348,164,0.00573710826383,128,2909,true,1,0.0285858297348 +fp16,1,1,1,64,1,control_shuffle32_fused,1,12,100,89.2507457733,164,0.00183751965968,128,2909,true,1,0.0892507457733 +fp16,1,1,1,64,1,shuffle32_fused,1,13,100,29.7661709785,164,0.00550961022559,128,2909,true,1,0.0297661709785 +fp16,1,1,1,64,1,baseline_transform,1,14,100,9.02239978313,256,0.0283738258283,128,2909,true,1,0.00902239978313 +fp16,1,1,1,64,1,optimized_transform,2,0,100,9.28685724735,256,0.0275658377405,128,2909,true,1,0.00928685724735 +fp16,1,1,1,64,1,control_shuffle32_transform,2,1,100,7.93417155743,256,0.0322654984389,128,2909,true,1,0.00793417155743 +fp16,1,1,1,64,1,shuffle32_transform,2,2,100,7.90925681591,256,0.0323671371354,128,2909,true,1,0.00790925681591 +fp16,1,1,1,64,1,baseline_split,2,3,100,69.8790836334,420,0.00601038219395,128,2909,true,1,0.0698790836334 +fp16,1,1,1,64,1,control_optimized_split,2,4,100,64.1101694107,420,0.00655122274454,128,2909,true,1,0.0641101694107 +fp16,1,1,1,64,1,optimized_split,2,5,100,33.6482286453,420,0.0124820835125,128,2909,true,1,0.0336482286453 +fp16,1,1,1,64,1,control_shuffle32_split,2,6,100,94.5771408081,420,0.00444081938206,128,2909,true,1,0.0945771408081 +fp16,1,1,1,64,1,shuffle32_split,2,7,100,34.7947430611,420,0.0120707889483,128,2909,true,1,0.0347947430611 +fp16,1,1,1,64,1,baseline_fused,2,8,100,65.4507446289,164,0.00250570105703,128,2909,true,1,0.0654507446289 +fp16,1,1,1,64,1,control_optimized_fused,2,9,100,58.6621713638,164,0.00279566876212,128,2909,true,1,0.0586621713638 +fp16,1,1,1,64,1,optimized_fused,2,10,100,28.5229706764,164,0.0057497517303,128,2909,true,1,0.0285229706764 +fp16,1,1,1,64,1,control_shuffle32_fused,2,11,100,89.2402362823,164,0.00183773605755,128,2909,true,1,0.0892402362823 +fp16,1,1,1,64,1,shuffle32_fused,2,12,100,29.751086235,164,0.00551240377257,128,2909,true,1,0.029751086235 +fp16,1,1,1,64,1,baseline_transform,2,13,100,9.22285735607,256,0.0277571245132,128,2909,true,1,0.00922285735607 +fp16,1,1,1,64,1,control_optimized_transform,2,14,100,9.21074271202,256,0.0277936327182,128,2909,true,1,0.00921074271202 +fp16,1,1,1,64,1,control_shuffle32_transform,3,0,100,7.90057122707,256,0.0324027203404,128,2909,true,1,0.00790057122707 +fp16,1,1,1,64,1,shuffle32_transform,3,1,100,7.94857144356,256,0.0322070452304,128,2909,true,1,0.00794857144356 +fp16,1,1,1,64,1,baseline_split,3,2,100,69.8886871338,420,0.00600955629909,128,2909,true,1,0.0698886871338 +fp16,1,1,1,64,1,control_optimized_split,3,3,100,64.1362285614,420,0.00654856092135,128,2909,true,1,0.0641362285614 +fp16,1,1,1,64,1,optimized_split,3,4,100,33.7225151062,420,0.0124545870519,128,2909,true,1,0.0337225151062 +fp16,1,1,1,64,1,control_shuffle32_split,3,5,100,94.6111965179,420,0.00443922088989,128,2909,true,1,0.0946111965179 +fp16,1,1,1,64,1,shuffle32_split,3,6,100,34.8644566536,420,0.0120466526748,128,2909,true,1,0.0348644566536 +fp16,1,1,1,64,1,baseline_fused,3,7,100,65.4454851151,164,0.00250590242721,128,2909,true,1,0.0654454851151 +fp16,1,1,1,64,1,control_optimized_fused,3,8,100,58.6589717865,164,0.002795821253,128,2909,true,1,0.0586589717865 +fp16,1,1,1,64,1,optimized_fused,3,9,100,28.5357737541,164,0.00574717200287,128,2909,true,1,0.0285357737541 +fp16,1,1,1,64,1,control_shuffle32_fused,3,10,100,89.245262146,164,0.0018376325651,128,2909,true,1,0.089245262146 +fp16,1,1,1,64,1,shuffle32_fused,3,11,100,29.721827507,164,0.00551783028689,128,2909,true,1,0.029721827507 +fp16,1,1,1,64,1,baseline_transform,3,12,100,9.01051461697,256,0.028411251841,128,2909,true,1,0.00901051461697 +fp16,1,1,1,64,1,control_optimized_transform,3,13,100,9.28251445293,256,0.0275787343287,128,2909,true,1,0.00928251445293 +fp16,1,1,1,64,1,optimized_transform,3,14,100,9.30331408978,256,0.0275170759075,128,2909,true,1,0.00930331408978 +fp16,1,1,1,64,1,shuffle32_transform,4,0,100,7.87131428719,256,0.0325231582249,128,2909,true,1,0.00787131428719 +fp16,1,1,1,64,1,baseline_split,4,1,100,69.8950862885,420,0.0060090061019,128,2909,true,1,0.0698950862885 +fp16,1,1,1,64,1,control_optimized_split,4,2,100,64.0390872955,420,0.00655849447169,128,2909,true,1,0.0640390872955 +fp16,1,1,1,64,1,optimized_split,4,3,100,33.7533712387,420,0.012443201511,128,2909,true,1,0.0337533712387 +fp16,1,1,1,64,1,control_shuffle32_split,4,4,100,93.9236545563,420,0.00447171697039,128,2909,true,1,0.0939236545563 +fp16,1,1,1,64,1,shuffle32_split,4,5,100,34.8235440254,420,0.0120608057495,128,2909,true,1,0.0348235440254 +fp16,1,1,1,64,1,baseline_fused,4,6,100,65.4473161697,164,0.00250583231824,128,2909,true,1,0.0654473161697 +fp16,1,1,1,64,1,control_optimized_fused,4,7,100,58.6116600037,164,0.00279807806143,128,2909,true,1,0.0586116600037 +fp16,1,1,1,64,1,optimized_fused,4,8,100,28.5428571701,164,0.00574574574026,128,2909,true,1,0.0285428571701 +fp16,1,1,1,64,1,control_shuffle32_fused,4,9,100,89.1981697083,164,0.00183860274865,128,2909,true,1,0.0891981697083 +fp16,1,1,1,64,1,shuffle32_fused,4,10,100,29.7458291054,164,0.00551337800735,128,2909,true,1,0.0297458291054 +fp16,1,1,1,64,1,baseline_transform,4,11,100,9.01005685329,256,0.0284126952991,128,2909,true,1,0.00901005685329 +fp16,1,1,1,64,1,control_optimized_transform,4,12,100,9.15634274483,256,0.0279587611707,128,2909,true,1,0.00915634274483 +fp16,1,1,1,64,1,optimized_transform,4,13,100,9.42491412163,256,0.0271620512077,128,2909,true,1,0.00942491412163 +fp16,1,1,1,64,1,control_shuffle32_transform,4,14,100,7.89005696774,256,0.0324459000799,128,2909,true,1,0.00789005696774 +fp16,1,17,1,64,17,baseline_transform,0,0,100,9.22765672207,4352,0.4716256934,2176,2909,true,1,0.00922765672207 +fp16,1,17,1,64,17,control_optimized_transform,0,1,100,9.42765712738,4352,0.461620521536,2176,2909,true,1,0.00942765712738 +fp16,1,17,1,64,17,optimized_transform,0,2,100,9.54354286194,4352,0.456015136408,2176,2909,true,1,0.00954354286194 +fp16,1,17,1,64,17,control_shuffle32_transform,0,3,100,7.92982876301,4352,0.548813868504,2176,2909,true,1,0.00792982876301 +fp16,1,17,1,64,17,shuffle32_transform,0,4,100,7.87634313107,4352,0.552540681326,2176,2909,true,1,0.00787634313107 +fp16,1,17,1,64,17,baseline_split,0,5,100,70.408911705,7140,0.101407617688,2176,2909,true,1,0.070408911705 +fp16,1,17,1,64,17,control_optimized_split,0,6,100,64.8982858658,7140,0.110018314116,2176,2909,true,1,0.0648982858658 +fp16,1,17,1,64,17,optimized_split,0,7,100,34.381942749,7140,0.207667148192,2176,2909,true,1,0.034381942749 +fp16,1,17,1,64,17,control_shuffle32_split,0,8,100,93.7814807892,7140,0.0761344344311,2176,2909,true,1,0.0937814807892 +fp16,1,17,1,64,17,shuffle32_split,0,9,100,34.7935986519,7140,0.205210161543,2176,2909,true,1,0.0347935986519 +fp16,1,17,1,64,17,baseline_fused,0,10,100,65.7357692719,2788,0.0424122213961,2176,2909,true,1,0.0657357692719 +fp16,1,17,1,64,17,control_optimized_fused,0,11,100,58.9124584198,2788,0.0473244552134,2176,2909,true,1,0.0589124584198 +fp16,1,17,1,64,17,optimized_fused,0,12,100,29.0630841255,2788,0.0959292547191,2176,2909,true,1,0.0290630841255 +fp16,1,17,1,64,17,control_shuffle32_fused,0,13,100,89.0834236145,2788,0.031296507104,2176,2909,true,1,0.0890834236145 +fp16,1,17,1,64,17,shuffle32_fused,0,14,100,29.5929145813,2788,0.0942117408659,2176,2909,true,1,0.0295929145813 +fp16,1,17,1,64,17,control_optimized_transform,1,0,100,9.32731449604,4352,0.46658660452,2176,2909,true,1,0.00932731449604 +fp16,1,17,1,64,17,optimized_transform,1,1,100,9.44411456585,4352,0.460816095533,2176,2909,true,1,0.00944411456585 +fp16,1,17,1,64,17,control_shuffle32_transform,1,2,100,7.89759993553,4352,0.551053489101,2176,2909,true,1,0.00789759993553 +fp16,1,17,1,64,17,shuffle32_transform,1,3,100,7.96937167645,4352,0.546090730448,2176,2909,true,1,0.00796937167645 +fp16,1,17,1,64,17,baseline_split,1,4,100,70.3017139435,7140,0.101562246487,2176,2909,true,1,0.0703017139435 +fp16,1,17,1,64,17,control_optimized_split,1,5,100,64.4404602051,7140,0.110799953589,2176,2909,true,1,0.0644404602051 +fp16,1,17,1,64,17,optimized_split,1,6,100,34.2953133583,7140,0.208191711952,2176,2909,true,1,0.0342953133583 +fp16,1,17,1,64,17,control_shuffle32_split,1,7,100,94.38331604,7140,0.0756489631809,2176,2909,true,1,0.09438331604 +fp16,1,17,1,64,17,shuffle32_split,1,8,100,34.8477721214,7140,0.20489114699,2176,2909,true,1,0.0348477721214 +fp16,1,17,1,64,17,baseline_fused,1,9,100,65.7398843765,2788,0.0424095665279,2176,2909,true,1,0.0657398843765 +fp16,1,17,1,64,17,control_optimized_fused,1,10,100,58.9366865158,2788,0.0473050007528,2176,2909,true,1,0.0589366865158 +fp16,1,17,1,64,17,optimized_fused,1,11,100,29.1152000427,2788,0.0957575423115,2176,2909,true,1,0.0291152000427 +fp16,1,17,1,64,17,control_shuffle32_fused,1,12,100,89.0912055969,2788,0.0312937734013,2176,2909,true,1,0.0890912055969 +fp16,1,17,1,64,17,shuffle32_fused,1,13,100,29.6016001701,2788,0.0941840976155,2176,2909,true,1,0.0296016001701 +fp16,1,17,1,64,17,baseline_transform,1,14,100,9.37920033932,4352,0.464005442101,2176,2909,true,1,0.00937920033932 +fp16,1,17,1,64,17,optimized_transform,2,0,100,9.42559957504,4352,0.46172129055,2176,2909,true,1,0.00942559957504 +fp16,1,17,1,64,17,control_shuffle32_transform,2,1,100,7.96525716782,4352,0.546372817388,2176,2909,true,1,0.00796525716782 +fp16,1,17,1,64,17,shuffle32_transform,2,2,100,8.04662883282,4352,0.540847613382,2176,2909,true,1,0.00804662883282 +fp16,1,17,1,64,17,baseline_split,2,3,100,70.2779436111,7140,0.101596598209,2176,2909,true,1,0.0702779436111 +fp16,1,17,1,64,17,control_optimized_split,2,4,100,64.4564580917,7140,0.110772453395,2176,2909,true,1,0.0644564580917 +fp16,1,17,1,64,17,optimized_split,2,5,100,35.222401619,7140,0.202711901285,2176,2909,true,1,0.035222401619 +fp16,1,17,1,64,17,control_shuffle32_split,2,6,100,94.4301700592,7140,0.0756114279528,2176,2909,true,1,0.0944301700592 +fp16,1,17,1,64,17,shuffle32_split,2,7,100,34.8118853569,7140,0.205102364517,2176,2909,true,1,0.0348118853569 +fp16,1,17,1,64,17,baseline_fused,2,8,100,65.7394313812,2788,0.0424098587624,2176,2909,true,1,0.0657394313812 +fp16,1,17,1,64,17,control_optimized_fused,2,9,100,58.9067459106,2788,0.0473290445245,2176,2909,true,1,0.0589067459106 +fp16,1,17,1,64,17,optimized_fused,2,10,100,29.0242290497,2788,0.0960576763375,2176,2909,true,1,0.0290242290497 +fp16,1,17,1,64,17,control_shuffle32_fused,2,11,100,89.0918827057,2788,0.031293535565,2176,2909,true,1,0.0890918827057 +fp16,1,17,1,64,17,shuffle32_fused,2,12,100,29.5844578743,2788,0.0942386712593,2176,2909,true,1,0.0295844578743 +fp16,1,17,1,64,17,baseline_transform,2,13,100,9.33577120304,4352,0.46616395211,2176,2909,true,1,0.00933577120304 +fp16,1,17,1,64,17,control_optimized_transform,2,14,100,9.46834266186,4352,0.459636934934,2176,2909,true,1,0.00946834266186 +fp16,1,17,1,64,17,control_shuffle32_transform,3,0,100,8.04137170315,4352,0.541201198086,2176,2909,true,1,0.00804137170315 +fp16,1,17,1,64,17,shuffle32_transform,3,1,100,8.01051437855,4352,0.543285960719,2176,2909,true,1,0.00801051437855 +fp16,1,17,1,64,17,baseline_split,3,2,100,70.333943367,7140,0.101515707185,2176,2909,true,1,0.070333943367 +fp16,1,17,1,64,17,control_optimized_split,3,3,100,64.5469713211,7140,0.11061711888,2176,2909,true,1,0.0645469713211 +fp16,1,17,1,64,17,optimized_split,3,4,100,34.3485713005,7140,0.207868907779,2176,2909,true,1,0.0343485713005 +fp16,1,17,1,64,17,control_shuffle32_split,3,5,100,93.7805652618,7140,0.0761351776892,2176,2909,true,1,0.0937805652618 +fp16,1,17,1,64,17,shuffle32_split,3,6,100,34.8075437546,7140,0.205127947273,2176,2909,true,1,0.0348075437546 +fp16,1,17,1,64,17,baseline_fused,3,7,100,65.7312011719,2788,0.0424151689045,2176,2909,true,1,0.0657312011719 +fp16,1,17,1,64,17,control_optimized_fused,3,8,100,58.9227437973,2788,0.0473161943984,2176,2909,true,1,0.0589227437973 +fp16,1,17,1,64,17,optimized_fused,3,9,100,29.0804576874,2788,0.0958719436252,2176,2909,true,1,0.0290804576874 +fp16,1,17,1,64,17,control_shuffle32_fused,3,10,100,89.0843391418,2788,0.0312961854671,2176,2909,true,1,0.0890843391418 +fp16,1,17,1,64,17,shuffle32_fused,3,11,100,29.6230840683,2788,0.0941157913731,2176,2909,true,1,0.0296230840683 +fp16,1,17,1,64,17,baseline_transform,3,12,100,9.23977136612,4352,0.471007325566,2176,2909,true,1,0.00923977136612 +fp16,1,17,1,64,17,control_optimized_transform,3,13,100,9.47108566761,4352,0.459503815374,2176,2909,true,1,0.00947108566761 +fp16,1,17,1,64,17,optimized_transform,3,14,100,9.4544005394,4352,0.460314747811,2176,2909,true,1,0.0094544005394 +fp16,1,17,1,64,17,shuffle32_transform,4,0,100,7.99040019512,4352,0.544653571001,2176,2909,true,1,0.00799040019512 +fp16,1,17,1,64,17,baseline_split,4,1,100,70.2882289886,7140,0.101581731433,2176,2909,true,1,0.0702882289886 +fp16,1,17,1,64,17,control_optimized_split,4,2,100,64.4162273407,7140,0.110841635637,2176,2909,true,1,0.0644162273407 +fp16,1,17,1,64,17,optimized_split,4,3,100,34.2893719673,7140,0.208227785764,2176,2909,true,1,0.0342893719673 +fp16,1,17,1,64,17,control_shuffle32_split,4,4,100,93.7620544434,7140,0.0761502085507,2176,2909,true,1,0.0937620544434 +fp16,1,17,1,64,17,shuffle32_split,4,5,100,34.7942852974,7140,0.205206111836,2176,2909,true,1,0.0347942852974 +fp16,1,17,1,64,17,baseline_fused,4,6,100,65.7327985764,2788,0.042414138153,2176,2909,true,1,0.0657327985764 +fp16,1,17,1,64,17,control_optimized_fused,4,7,100,58.9229726791,2788,0.0473160106022,2176,2909,true,1,0.0589229726791 +fp16,1,17,1,64,17,optimized_fused,4,8,100,29.1083431244,2788,0.0957800994748,2176,2909,true,1,0.0291083431244 +fp16,1,17,1,64,17,control_shuffle32_fused,4,9,100,89.0425109863,2788,0.0313108870035,2176,2909,true,1,0.0890425109863 +fp16,1,17,1,64,17,shuffle32_fused,4,10,100,29.6075439453,2788,0.0941651899647,2176,2909,true,1,0.0296075439453 +fp16,1,17,1,64,17,baseline_transform,4,11,100,9.24251437187,4352,0.470867539384,2176,2909,true,1,0.00924251437187 +fp16,1,17,1,64,17,control_optimized_transform,4,12,100,9.502171278,4352,0.458000584569,2176,2909,true,1,0.009502171278 +fp16,1,17,1,64,17,optimized_transform,4,13,100,9.39337134361,4352,0.463305435376,2176,2909,true,1,0.00939337134361 +fp16,1,17,1,64,17,control_shuffle32_transform,4,14,100,8.0601143837,4352,0.539942709598,2176,2909,true,1,0.0080601143837 +fp16,1,257,1,64,257,baseline_transform,0,0,100,9.56914246082,65792,6.87543322397,32896,2909,true,1,0.00956914246082 +fp16,1,257,1,64,257,control_optimized_transform,0,1,100,9.62697148323,65792,6.83413263606,32896,2909,true,1,0.00962697148323 +fp16,1,257,1,64,257,optimized_transform,0,2,100,9.61508572102,65792,6.84258070172,32896,2909,true,1,0.00961508572102 +fp16,1,257,1,64,257,control_shuffle32_transform,0,3,100,8.88297140598,65792,7.40653065208,32896,2909,true,1,0.00888297140598 +fp16,1,257,1,64,257,shuffle32_transform,0,4,100,8.80640029907,65792,7.47092997884,32896,2909,true,1,0.00880640029907 +fp16,1,257,1,64,257,baseline_split,0,5,100,71.0850286484,107940,1.51846319897,32896,2909,true,1,0.0710850286484 +fp16,1,257,1,64,257,control_optimized_split,0,6,100,64.4795417786,107940,1.67401934044,32896,2909,true,1,0.0644795417786 +fp16,1,257,1,64,257,optimized_split,0,7,100,34.7506284714,107940,3.10613087441,32896,2909,true,1,0.0347506284714 +fp16,1,257,1,64,257,control_shuffle32_split,0,8,100,102.767763138,107940,1.05032937085,32896,2909,true,1,0.102767763138 +fp16,1,257,1,64,257,shuffle32_split,0,9,100,39.063770771,107940,2.76317410914,32896,2909,true,1,0.039063770771 +fp16,1,257,1,64,257,baseline_fused,0,10,100,65.99588871,42148,0.638645843307,32896,2909,true,1,0.06599588871 +fp16,1,257,1,64,257,control_optimized_fused,0,11,100,59.3291425705,42148,0.710409727394,32896,2909,true,1,0.0593291425705 +fp16,1,257,1,64,257,optimized_fused,0,12,100,29.377143383,42148,1.43472084574,32896,2909,true,1,0.029377143383 +fp16,1,257,1,64,257,control_shuffle32_fused,0,13,100,96.6690254211,42148,0.43600315423,32896,2909,true,1,0.0966690254211 +fp16,1,257,1,64,257,shuffle32_fused,0,14,100,33.2642269135,42148,1.26706687366,32896,2909,true,1,0.0332642269135 +fp16,1,257,1,64,257,control_optimized_transform,1,0,100,9.71245706081,65792,6.77398104188,32896,2909,true,1,0.00971245706081 +fp16,1,257,1,64,257,optimized_transform,1,1,100,9.67885732651,65792,6.79749662389,32896,2909,true,1,0.00967885732651 +fp16,1,257,1,64,257,control_shuffle32_transform,1,2,100,8.89691412449,65792,7.39492357456,32896,2909,true,1,0.00889691412449 +fp16,1,257,1,64,257,shuffle32_transform,1,3,100,8.88114273548,65792,7.40805569278,32896,2909,true,1,0.00888114273548 +fp16,1,257,1,64,257,baseline_split,1,4,100,71.0847997665,107940,1.51846808818,32896,2909,true,1,0.0710847997665 +fp16,1,257,1,64,257,control_optimized_split,1,5,100,64.5149755478,107940,1.67309991337,32896,2909,true,1,0.0645149755478 +fp16,1,257,1,64,257,optimized_split,1,6,100,34.8164558411,107940,3.10025812199,32896,2909,true,1,0.0348164558411 +fp16,1,257,1,64,257,control_shuffle32_split,1,7,100,102.141942978,107940,1.05676470266,32896,2909,true,1,0.102141942978 +fp16,1,257,1,64,257,shuffle32_split,1,8,100,39.104912281,107940,2.76026702794,32896,2909,true,1,0.039104912281 +fp16,1,257,1,64,257,baseline_fused,1,9,100,65.9983968735,42148,0.638621572594,32896,2909,true,1,0.0659983968735 +fp16,1,257,1,64,257,control_optimized_fused,1,10,100,59.3291425705,42148,0.710409727394,32896,2909,true,1,0.0593291425705 +fp16,1,257,1,64,257,optimized_fused,1,11,100,29.3412566185,42148,1.43647562707,32896,2909,true,1,0.0293412566185 +fp16,1,257,1,64,257,control_shuffle32_fused,1,12,100,96.7202281952,42148,0.435772338284,32896,2909,true,1,0.0967202281952 +fp16,1,257,1,64,257,shuffle32_fused,1,13,100,33.2242298126,42148,1.26859223638,32896,2909,true,1,0.0332242298126 +fp16,1,257,1,64,257,baseline_transform,1,14,100,9.55085754395,65792,6.88859609698,32896,2909,true,1,0.00955085754395 +fp16,1,257,1,64,257,optimized_transform,2,0,100,9.72160041332,65792,6.76760998218,32896,2909,true,1,0.00972160041332 +fp16,1,257,1,64,257,control_shuffle32_transform,2,1,100,8.85988593102,65792,7.42582923892,32896,2909,true,1,0.00885988593102 +fp16,1,257,1,64,257,shuffle32_transform,2,2,100,8.85851442814,65792,7.426978929,32896,2909,true,1,0.00885851442814 +fp16,1,257,1,64,257,baseline_split,2,3,100,71.070857048,107940,1.51876598205,32896,2909,true,1,0.071070857048 +fp16,1,257,1,64,257,control_optimized_split,2,4,100,64.555888176,107940,1.67203957764,32896,2909,true,1,0.064555888176 +fp16,1,257,1,64,257,optimized_split,2,5,100,34.8523426056,107940,3.09706584781,32896,2909,true,1,0.0348523426056 +fp16,1,257,1,64,257,control_shuffle32_split,2,6,100,102.126169205,107940,1.05692792396,32896,2909,true,1,0.102126169205 +fp16,1,257,1,64,257,shuffle32_split,2,7,100,39.1355443001,107940,2.75810652261,32896,2909,true,1,0.0391355443001 +fp16,1,257,1,64,257,baseline_fused,2,8,100,65.9421682358,42148,0.639166122796,32896,2909,true,1,0.0659421682358 +fp16,1,257,1,64,257,control_optimized_fused,2,9,100,59.3318843842,42148,0.710376898315,32896,2909,true,1,0.0593318843842 +fp16,1,257,1,64,257,optimized_fused,2,10,100,29.3471980095,42148,1.43618481009,32896,2909,true,1,0.0293471980095 +fp16,1,257,1,64,257,control_shuffle32_fused,2,11,100,96.7167949677,42148,0.435787807217,32896,2909,true,1,0.0967167949677 +fp16,1,257,1,64,257,shuffle32_fused,2,12,100,33.2244586945,42148,1.2685834971,32896,2909,true,1,0.0332244586945 +fp16,1,257,1,64,257,baseline_transform,2,13,100,9.62331414223,65792,6.8367299485,32896,2909,true,1,0.00962331414223 +fp16,1,257,1,64,257,control_optimized_transform,2,14,100,9.60160017014,65792,6.85219118003,32896,2909,true,1,0.00960160017014 +fp16,1,257,1,64,257,control_shuffle32_transform,3,0,100,8.828343153,65792,7.45236097644,32896,2909,true,1,0.008828343153 +fp16,1,257,1,64,257,shuffle32_transform,3,1,100,8.87565732002,65792,7.41263408757,32896,2909,true,1,0.00887565732002 +fp16,1,257,1,64,257,baseline_split,3,2,100,71.0960006714,107940,1.5182288593,32896,2909,true,1,0.0710960006714 +fp16,1,257,1,64,257,control_optimized_split,3,3,100,64.5158863068,107940,1.67307629452,32896,2909,true,1,0.0645158863068 +fp16,1,257,1,64,257,optimized_split,3,4,100,34.7583985329,107940,3.1054365148,32896,2909,true,1,0.0347583985329 +fp16,1,257,1,64,257,control_shuffle32_split,3,5,100,102.130289078,107940,1.05688528814,32896,2909,true,1,0.102130289078 +fp16,1,257,1,64,257,shuffle32_split,3,6,100,39.6253728867,107940,2.72401222088,32896,2909,true,1,0.0396253728867 +fp16,1,257,1,64,257,baseline_fused,3,7,100,65.9910869598,42148,0.638692313488,32896,2909,true,1,0.0659910869598 +fp16,1,257,1,64,257,control_optimized_fused,3,8,100,59.354057312,42148,0.710111522426,32896,2909,true,1,0.059354057312 +fp16,1,257,1,64,257,optimized_fused,3,9,100,29.3547415733,42148,1.43581574018,32896,2909,true,1,0.0293547415733 +fp16,1,257,1,64,257,control_shuffle32_fused,3,10,100,96.713142395,42148,0.435804265648,32896,2909,true,1,0.096713142395 +fp16,1,257,1,64,257,shuffle32_fused,3,11,100,33.2205724716,42148,1.26873189907,32896,2909,true,1,0.0332205724716 +fp16,1,257,1,64,257,baseline_transform,3,12,100,9.64274287224,65792,6.82295492804,32896,2909,true,1,0.00964274287224 +fp16,1,257,1,64,257,control_optimized_transform,3,13,100,9.67634260654,65792,6.79926317982,32896,2909,true,1,0.00967634260654 +fp16,1,257,1,64,257,optimized_transform,3,14,100,9.65394318104,65792,6.81503907432,32896,2909,true,1,0.00965394318104 +fp16,1,257,1,64,257,shuffle32_transform,4,0,100,8.8676571846,65792,7.41932154462,32896,2909,true,1,0.0088676571846 +fp16,1,257,1,64,257,baseline_split,4,1,100,71.0918855667,107940,1.51831674093,32896,2909,true,1,0.0710918855667 +fp16,1,257,1,64,257,control_optimized_split,4,2,100,64.4281148911,107940,1.67535555219,32896,2909,true,1,0.0644281148911 +fp16,1,257,1,64,257,optimized_split,4,3,100,34.7755432129,107940,3.10390550449,32896,2909,true,1,0.0347755432129 +fp16,1,257,1,64,257,control_shuffle32_split,4,4,100,102.085485458,107940,1.05734913749,32896,2909,true,1,0.102085485458 +fp16,1,257,1,64,257,shuffle32_split,4,5,100,39.0598845482,107940,2.76344902829,32896,2909,true,1,0.0390598845482 +fp16,1,257,1,64,257,baseline_fused,4,6,100,65.9897136688,42148,0.638705605112,32896,2909,true,1,0.0659897136688 +fp16,1,257,1,64,257,control_optimized_fused,4,7,100,59.3245744705,42148,0.710464430233,32896,2909,true,1,0.0593245744705 +fp16,1,257,1,64,257,optimized_fused,4,8,100,29.3828558922,42148,1.43444191248,32896,2909,true,1,0.0293828558922 +fp16,1,257,1,64,257,control_shuffle32_fused,4,9,100,96.7161178589,42148,0.435790858164,32896,2909,true,1,0.0967161178589 +fp16,1,257,1,64,257,shuffle32_fused,4,10,100,33.2185125351,42148,1.26881057529,32896,2909,true,1,0.0332185125351 +fp16,1,257,1,64,257,baseline_transform,4,11,100,9.53942835331,65792,6.89684932506,32896,2909,true,1,0.00953942835331 +fp16,1,257,1,64,257,control_optimized_transform,4,12,100,9.51657116413,65792,6.91341438689,32896,2909,true,1,0.00951657116413 +fp16,1,257,1,64,257,optimized_transform,4,13,100,9.60297107697,65792,6.85121297072,32896,2909,true,1,0.00960297107697 +fp16,1,257,1,64,257,control_shuffle32_transform,4,14,100,8.83474290371,65792,7.44696260175,32896,2909,true,1,0.00883474290371 +fp16,4,64,16,64,4096,baseline_transform,0,0,100,23.1675434113,1048576,45.2605604913,524288,2909,true,1,0.0231675434113 +fp16,4,64,16,64,4096,control_optimized_transform,0,1,100,23.9076590538,1048576,43.8594175047,524288,2909,true,1,0.0239076590538 +fp16,4,64,16,64,4096,optimized_transform,0,2,100,23.9613723755,1048576,43.7610994716,524288,2909,true,1,0.0239613723755 +fp16,4,64,16,64,4096,control_shuffle32_transform,0,3,100,11.0994291306,1048576,94.4711649281,524288,2909,true,1,0.0110994291306 +fp16,4,64,16,64,4096,shuffle32_transform,0,4,100,10.9092581272,1048576,96.1179933386,524288,2909,true,1,0.0109092581272 +fp16,4,64,16,64,4096,baseline_split,0,5,100,436.142654419,1720320,3.94439750978,524288,2909,true,1,0.436142654419 +fp16,4,64,16,64,4096,control_optimized_split,0,6,100,380.986747742,1720320,4.51543265008,524288,2909,true,1,0.380986747742 +fp16,4,64,16,64,4096,optimized_split,0,7,100,161.863327026,1720320,10.6282258718,524288,2909,true,1,0.161863327026 +fp16,4,64,16,64,4096,control_shuffle32_split,0,8,100,247.547416687,1720320,6.94945648403,524288,2909,true,1,0.247547416687 +fp16,4,64,16,64,4096,shuffle32_split,0,9,100,71.3734817505,1720320,24.1030696249,524288,2909,true,1,0.0713734817505 +fp16,4,64,16,64,4096,baseline_fused,0,10,100,434.089584351,671744,1.54747781153,524288,2909,true,1,0.434089584351 +fp16,4,64,16,64,4096,control_optimized_fused,0,11,100,371.561393738,671744,1.80789503786,524288,2909,true,1,0.371561393738 +fp16,4,64,16,64,4096,optimized_fused,0,12,100,147.914514542,671744,4.54143396327,524288,2909,true,1,0.147914514542 +fp16,4,64,16,64,4096,control_shuffle32_fused,0,13,100,247.985839844,671744,2.70879982673,524288,2909,true,1,0.247985839844 +fp16,4,64,16,64,4096,shuffle32_fused,0,14,100,69.9979448318,671744,9.59662460968,524288,2909,true,1,0.0699979448318 +fp16,4,64,16,64,4096,control_optimized_transform,1,0,100,23.8960003853,1048576,43.8808161656,524288,2909,true,1,0.0238960003853 +fp16,4,64,16,64,4096,optimized_transform,1,1,100,24.0283441544,1048576,43.6391285752,524288,2909,true,1,0.0240283441544 +fp16,4,64,16,64,4096,control_shuffle32_transform,1,2,100,11.3554286957,1048576,92.3413838527,524288,2909,true,1,0.0113554286957 +fp16,4,64,16,64,4096,shuffle32_transform,1,3,100,11.4164566994,1048576,91.8477621921,524288,2909,true,1,0.0114164566994 +fp16,4,64,16,64,4096,baseline_split,1,4,100,436.997718811,1720320,3.93667958881,524288,2909,true,1,0.436997718811 +fp16,4,64,16,64,4096,control_optimized_split,1,5,100,381.986274719,1720320,4.5036173126,524288,2909,true,1,0.381986274719 +fp16,4,64,16,64,4096,optimized_split,1,6,100,162.167549133,1720320,10.6082875963,524288,2909,true,1,0.162167549133 +fp16,4,64,16,64,4096,control_shuffle32_split,1,7,100,248.310623169,1720320,6.92809666395,524288,2909,true,1,0.248310623169 +fp16,4,64,16,64,4096,shuffle32_split,1,8,100,70.1499462128,1720320,24.5234685538,524288,2909,true,1,0.0701499462128 +fp16,4,64,16,64,4096,baseline_fused,1,9,100,433.858299255,671744,1.54830275496,524288,2909,true,1,0.433858299255 +fp16,4,64,16,64,4096,control_optimized_fused,1,10,100,371.569137573,671744,1.8078573597,524288,2909,true,1,0.371569137573 +fp16,4,64,16,64,4096,optimized_fused,1,11,100,147.505598068,671744,4.55402377128,524288,2909,true,1,0.147505598068 +fp16,4,64,16,64,4096,control_shuffle32_fused,1,12,100,248.164577484,671744,2.70684884527,524288,2909,true,1,0.248164577484 +fp16,4,64,16,64,4096,shuffle32_fused,1,13,100,71.0866260529,671744,9.44965371546,524288,2909,true,1,0.0710866260529 +fp16,4,64,16,64,4096,baseline_transform,1,14,100,22.7997708321,1048576,45.9906377009,524288,2909,true,1,0.0227997708321 +fp16,4,64,16,64,4096,optimized_transform,2,0,100,24.0173697472,1048576,43.6590688755,524288,2909,true,1,0.0240173697472 +fp16,4,64,16,64,4096,control_shuffle32_transform,2,1,100,11.2861704826,1048576,92.9080418919,524288,2909,true,1,0.0112861704826 +fp16,4,64,16,64,4096,shuffle32_transform,2,2,100,11.1149716377,1048576,94.3390621386,524288,2909,true,1,0.0111149716377 +fp16,4,64,16,64,4096,baseline_split,2,3,100,437.058944702,1720320,3.93612811465,524288,2909,true,1,0.437058944702 +fp16,4,64,16,64,4096,control_optimized_split,2,4,100,381.461257935,1720320,4.50981577871,524288,2909,true,1,0.381461257935 +fp16,4,64,16,64,4096,optimized_split,2,5,100,161.758861542,1720320,10.635089686,524288,2909,true,1,0.161758861542 +fp16,4,64,16,64,4096,control_shuffle32_split,2,6,100,249.615097046,1720320,6.89189083657,524288,2909,true,1,0.249615097046 +fp16,4,64,16,64,4096,shuffle32_split,2,7,100,70.3844547272,1720320,24.4417607079,524288,2909,true,1,0.0703844547272 +fp16,4,64,16,64,4096,baseline_fused,2,8,100,433.858299255,671744,1.54830275496,524288,2909,true,1,0.433858299255 +fp16,4,64,16,64,4096,control_optimized_fused,2,9,100,371.953353882,671744,1.80598989897,524288,2909,true,1,0.371953353882 +fp16,4,64,16,64,4096,optimized_fused,2,10,100,147.517719269,671744,4.55364957735,524288,2909,true,1,0.147517719269 +fp16,4,64,16,64,4096,control_shuffle32_fused,2,11,100,248.690509796,671744,2.70112438368,524288,2909,true,1,0.248690509796 +fp16,4,64,16,64,4096,shuffle32_fused,2,12,100,68.8095998764,671744,9.76235875818,524288,2909,true,1,0.0688095998764 +fp16,4,64,16,64,4096,baseline_transform,2,13,100,22.7206850052,1048576,46.1507212375,524288,2909,true,1,0.0227206850052 +fp16,4,64,16,64,4096,control_optimized_transform,2,14,100,23.9462852478,1048576,43.7886707332,524288,2909,true,1,0.0239462852478 +fp16,4,64,16,64,4096,control_shuffle32_transform,3,0,100,11.178971529,1048576,93.798968651,524288,2909,true,1,0.011178971529 +fp16,4,64,16,64,4096,shuffle32_transform,3,1,100,11.5517711639,1048576,90.7718812223,524288,2909,true,1,0.0115517711639 +fp16,4,64,16,64,4096,baseline_split,3,2,100,437.85987854,1720320,3.92892814417,524288,2909,true,1,0.43785987854 +fp16,4,64,16,64,4096,control_optimized_split,3,3,100,381.47819519,1720320,4.50961554733,524288,2909,true,1,0.38147819519 +fp16,4,64,16,64,4096,optimized_split,3,4,100,161.959762573,1720320,10.6218975174,524288,2909,true,1,0.161959762573 +fp16,4,64,16,64,4096,control_shuffle32_split,3,5,100,247.902412415,1720320,6.93950487712,524288,2909,true,1,0.247902412415 +fp16,4,64,16,64,4096,shuffle32_split,3,6,100,71.0420608521,1720320,24.2155137304,524288,2909,true,1,0.0710420608521 +fp16,4,64,16,64,4096,baseline_fused,3,7,100,433.966636658,671744,1.54791622963,524288,2909,true,1,0.433966636658 +fp16,4,64,16,64,4096,control_optimized_fused,3,8,100,372.403678894,671744,1.80380602575,524288,2909,true,1,0.372403678894 +fp16,4,64,16,64,4096,optimized_fused,3,9,100,147.678174973,671744,4.54870193327,524288,2909,true,1,0.147678174973 +fp16,4,64,16,64,4096,control_shuffle32_fused,3,10,100,248.658504486,671744,2.70147205055,524288,2909,true,1,0.248658504486 +fp16,4,64,16,64,4096,shuffle32_fused,3,11,100,68.4701681137,671744,9.81075435487,524288,2909,true,1,0.0684701681137 +fp16,4,64,16,64,4096,baseline_transform,3,12,100,23.2683420181,1048576,45.0644914529,524288,2909,true,1,0.0232683420181 +fp16,4,64,16,64,4096,control_optimized_transform,3,13,100,23.9636564255,1048576,43.7569284663,524288,2909,true,1,0.0239636564255 +fp16,4,64,16,64,4096,optimized_transform,3,14,100,23.98239851,1048576,43.7227327185,524288,2909,true,1,0.02398239851 +fp16,4,64,16,64,4096,shuffle32_transform,4,0,100,11.3821721077,1048576,92.1244196695,524288,2909,true,1,0.0113821721077 +fp16,4,64,16,64,4096,baseline_split,4,1,100,437.400245667,1720320,3.9330567759,524288,2909,true,1,0.437400245667 +fp16,4,64,16,64,4096,control_optimized_split,4,2,100,382.256469727,1720320,4.50043396579,524288,2909,true,1,0.382256469727 +fp16,4,64,16,64,4096,optimized_split,4,3,100,162.344913483,1720320,10.5966978767,524288,2909,true,1,0.162344913483 +fp16,4,64,16,64,4096,control_shuffle32_split,4,4,100,248.770065308,1720320,6.91530147678,524288,2909,true,1,0.248770065308 +fp16,4,64,16,64,4096,shuffle32_split,4,5,100,72.215089798,1720320,23.822167982,524288,2909,true,1,0.072215089798 +fp16,4,64,16,64,4096,baseline_fused,4,6,100,434.094619751,671744,1.54745986114,524288,2909,true,1,0.434094619751 +fp16,4,64,16,64,4096,control_optimized_fused,4,7,100,371.535758972,671744,1.80801977677,524288,2909,true,1,0.371535758972 +fp16,4,64,16,64,4096,optimized_fused,4,8,100,147.847537994,671744,4.54349128239,524288,2909,true,1,0.147847537994 +fp16,4,64,16,64,4096,control_shuffle32_fused,4,9,100,248.673591614,671744,2.70130815114,524288,2909,true,1,0.248673591614 +fp16,4,64,16,64,4096,shuffle32_fused,4,10,100,70.0399971008,671744,9.59086276136,524288,2909,true,1,0.0700399971008 +fp16,4,64,16,64,4096,baseline_transform,4,11,100,23.3030843735,1048576,44.9973052148,524288,2909,true,1,0.0233030843735 +fp16,4,64,16,64,4096,control_optimized_transform,4,12,100,23.9810276031,1048576,43.7252321857,524288,2909,true,1,0.0239810276031 +fp16,4,64,16,64,4096,optimized_transform,4,13,100,23.9718866348,1048576,43.7419055068,524288,2909,true,1,0.0239718866348 +fp16,4,64,16,64,4096,control_shuffle32_transform,4,14,100,11.3131427765,1048576,92.6865346541,524288,2909,true,1,0.0113131427765 +fp16,16,64,16,64,16384,baseline_transform,0,0,100,68.2715415955,4194304,61.4356128774,2097152,2909,true,1,0.0682715415955 +fp16,16,64,16,64,16384,control_optimized_transform,0,1,100,73.9302873611,4194304,56.7332300429,2097152,2909,true,1,0.0739302873611 +fp16,16,64,16,64,16384,optimized_transform,0,2,100,74.0404558182,4194304,56.6488138633,2097152,2909,true,1,0.0740404558182 +fp16,16,64,16,64,16384,control_shuffle32_transform,0,3,100,21.3103985786,4194304,196.819594177,2097152,2909,true,1,0.0213103985786 +fp16,16,64,16,64,16384,shuffle32_transform,0,4,100,21.2987422943,4194304,196.92730876,2097152,2909,true,1,0.0212987422943 +fp16,16,64,16,64,16384,baseline_split,0,5,100,1548.35006714,6881280,4.44426628451,2097152,2909,true,1,1.54835006714 +fp16,16,64,16,64,16384,control_optimized_split,0,6,100,1361.63635254,6881280,5.05368411116,2097152,2909,true,1,1.36163635254 +fp16,16,64,16,64,16384,optimized_split,0,7,100,576.891174316,6881280,11.9282116045,2097152,2909,true,1,0.576891174316 +fp16,16,64,16,64,16384,control_shuffle32_split,0,8,100,851.743087769,6881280,8.07905587826,2097152,2909,true,1,0.851743087769 +fp16,16,64,16,64,16384,shuffle32_split,0,9,100,228.59703064,6881280,30.1022282781,2097152,2909,true,1,0.22859703064 +fp16,16,64,16,64,16384,baseline_fused,0,10,100,1526.20849609,2686976,1.7605563112,2097152,2909,true,1,1.52620849609 +fp16,16,64,16,64,16384,control_optimized_fused,0,11,100,1333.48403931,2686976,2.01500424512,2097152,2909,true,1,1.33348403931 +fp16,16,64,16,64,16384,optimized_fused,0,12,100,544.487075806,2686976,4.93487562772,2097152,2909,true,1,0.544487075806 +fp16,16,64,16,64,16384,control_shuffle32_fused,0,13,100,866.483917236,2686976,3.10101081688,2097152,2909,true,1,0.866483917236 +fp16,16,64,16,64,16384,shuffle32_fused,0,14,100,211.490745544,2686976,12.7049341714,2097152,2909,true,1,0.211490745544 +fp16,16,64,16,64,16384,control_optimized_transform,1,0,100,73.9961147308,4194304,56.6827598349,2097152,2909,true,1,0.0739961147308 +fp16,16,64,16,64,16384,optimized_transform,1,1,100,74.053940773,4194304,56.6384983192,2097152,2909,true,1,0.074053940773 +fp16,16,64,16,64,16384,control_shuffle32_transform,1,2,100,21.1613726616,4194304,198.205667802,2097152,2909,true,1,0.0211613726616 +fp16,16,64,16,64,16384,shuffle32_transform,1,3,100,21.3291430473,4194304,196.646625263,2097152,2909,true,1,0.0213291430473 +fp16,16,64,16,64,16384,baseline_split,1,4,100,1548.62335205,6881280,4.4434820067,2097152,2909,true,1,1.54862335205 +fp16,16,64,16,64,16384,control_optimized_split,1,5,100,1361.69418335,6881280,5.05346948246,2097152,2909,true,1,1.36169418335 +fp16,16,64,16,64,16384,optimized_split,1,6,100,576.804580688,6881280,11.9300023446,2097152,2909,true,1,0.576804580688 +fp16,16,64,16,64,16384,control_shuffle32_split,1,7,100,851.620101929,6881280,8.08022260679,2097152,2909,true,1,0.851620101929 +fp16,16,64,16,64,16384,shuffle32_split,1,8,100,228.509502411,6881280,30.1137586288,2097152,2909,true,1,0.228509502411 +fp16,16,64,16,64,16384,baseline_fused,1,9,100,1526.13204956,2686976,1.76064450044,2097152,2909,true,1,1.52613204956 +fp16,16,64,16,64,16384,control_optimized_fused,1,10,100,1333.359375,2686976,2.01519264077,2097152,2909,true,1,1.333359375 +fp16,16,64,16,64,16384,optimized_fused,1,11,100,544.571876526,2686976,4.934107169,2097152,2909,true,1,0.544571876526 +fp16,16,64,16,64,16384,control_shuffle32_fused,1,12,100,866.494598389,2686976,3.10097259117,2097152,2909,true,1,0.866494598389 +fp16,16,64,16,64,16384,shuffle32_fused,1,13,100,211.51473999,2686976,12.7034929108,2097152,2909,true,1,0.21151473999 +fp16,16,64,16,64,16384,baseline_transform,1,14,100,68.2685709,4194304,61.4382862378,2097152,2909,true,1,0.0682685709 +fp16,16,64,16,64,16384,optimized_transform,2,0,100,74.0733718872,4194304,56.6236407651,2097152,2909,true,1,0.0740733718872 +fp16,16,64,16,64,16384,control_shuffle32_transform,2,1,100,21.3652563095,4194304,196.314237435,2097152,2909,true,1,0.0213652563095 +fp16,16,64,16,64,16384,shuffle32_transform,2,2,100,21.2671995163,4194304,197.219384564,2097152,2909,true,1,0.0212671995163 +fp16,16,64,16,64,16384,baseline_split,2,3,100,1548.61846924,6881280,4.44349601706,2097152,2909,true,1,1.54861846924 +fp16,16,64,16,64,16384,control_optimized_split,2,4,100,1361.67404175,6881280,5.05354423234,2097152,2909,true,1,1.36167404175 +fp16,16,64,16,64,16384,optimized_split,2,5,100,576.861495972,6881280,11.9288252866,2097152,2909,true,1,0.576861495972 +fp16,16,64,16,64,16384,control_shuffle32_split,2,6,100,851.665344238,6881280,8.07979336785,2097152,2909,true,1,0.851665344238 +fp16,16,64,16,64,16384,shuffle32_split,2,7,100,228.984451294,6881280,30.0512980734,2097152,2909,true,1,0.228984451294 +fp16,16,64,16,64,16384,baseline_fused,2,8,100,1526.17279053,2686976,1.76059750028,2097152,2909,true,1,1.52617279053 +fp16,16,64,16,64,16384,control_optimized_fused,2,9,100,1333.33251953,2686976,2.01523323,2097152,2909,true,1,1.33333251953 +fp16,16,64,16,64,16384,optimized_fused,2,10,100,544.486656189,2686976,4.93487943085,2097152,2909,true,1,0.544486656189 +fp16,16,64,16,64,16384,control_shuffle32_fused,2,11,100,866.493530273,2686976,3.1009764137,2097152,2909,true,1,0.866493530273 +fp16,16,64,16,64,16384,shuffle32_fused,2,12,100,211.561126709,2686976,12.7007075534,2097152,2909,true,1,0.211561126709 +fp16,16,64,16,64,16384,baseline_transform,2,13,100,68.2829713821,4194304,61.42532926,2097152,2909,true,1,0.0682829713821 +fp16,16,64,16,64,16384,control_optimized_transform,2,14,100,73.9888000488,4194304,56.6883636068,2097152,2909,true,1,0.0739888000488 +fp16,16,64,16,64,16384,control_shuffle32_transform,3,0,100,21.3298296928,4194304,196.640294855,2097152,2909,true,1,0.0213298296928 +fp16,16,64,16,64,16384,shuffle32_transform,3,1,100,21.328458786,4194304,196.652934095,2097152,2909,true,1,0.021328458786 +fp16,16,64,16,64,16384,baseline_split,3,2,100,1548.57437134,6881280,4.44362255205,2097152,2909,true,1,1.54857437134 +fp16,16,64,16,64,16384,control_optimized_split,3,3,100,1361.69036865,6881280,5.05348363946,2097152,2909,true,1,1.36169036865 +fp16,16,64,16,64,16384,optimized_split,3,4,100,576.767311096,6881280,11.9307732384,2097152,2909,true,1,0.576767311096 +fp16,16,64,16,64,16384,control_shuffle32_split,3,5,100,851.82975769,6881280,8.07823386994,2097152,2909,true,1,0.85182975769 +fp16,16,64,16,64,16384,shuffle32_split,3,6,100,228.550167084,6881280,30.1084006536,2097152,2909,true,1,0.228550167084 +fp16,16,64,16,64,16384,baseline_fused,3,7,100,1526.21765137,2686976,1.7605457502,2097152,2909,true,1,1.52621765137 +fp16,16,64,16,64,16384,control_optimized_fused,3,8,100,1333.45565796,2686976,2.01504713259,2097152,2909,true,1,1.33345565796 +fp16,16,64,16,64,16384,optimized_fused,3,9,100,544.562034607,2686976,4.93419634356,2097152,2909,true,1,0.544562034607 +fp16,16,64,16,64,16384,control_shuffle32_fused,3,10,100,866.471557617,2686976,3.10105505066,2097152,2909,true,1,0.866471557617 +fp16,16,64,16,64,16384,shuffle32_fused,3,11,100,211.503772736,2686976,12.704151634,2097152,2909,true,1,0.211503772736 +fp16,16,64,16,64,16384,baseline_transform,3,12,100,68.2898283005,4194304,61.4191615997,2097152,2909,true,1,0.0682898283005 +fp16,16,64,16,64,16384,control_optimized_transform,3,13,100,73.9968013763,4194304,56.6822338532,2097152,2909,true,1,0.0739968013763 +fp16,16,64,16,64,16384,optimized_transform,3,14,100,74.051656723,4194304,56.6402452775,2097152,2909,true,1,0.074051656723 +fp16,16,64,16,64,16384,shuffle32_transform,4,0,100,21.3293719292,4194304,196.644515081,2097152,2909,true,1,0.0213293719292 +fp16,16,64,16,64,16384,baseline_split,4,1,100,1548.47213745,6881280,4.44391593079,2097152,2909,true,1,1.54847213745 +fp16,16,64,16,64,16384,control_optimized_split,4,2,100,1361.6583252,6881280,5.05360256143,2097152,2909,true,1,1.3616583252 +fp16,16,64,16,64,16384,optimized_split,4,3,100,576.643867493,6881280,11.933327289,2097152,2909,true,1,0.576643867493 +fp16,16,64,16,64,16384,control_shuffle32_split,4,4,100,851.64390564,6881280,8.07999676206,2097152,2909,true,1,0.85164390564 +fp16,16,64,16,64,16384,shuffle32_split,4,5,100,228.414402008,6881280,30.126296501,2097152,2909,true,1,0.228414402008 +fp16,16,64,16,64,16384,baseline_fused,4,6,100,1526.04812622,2686976,1.76074132515,2097152,2909,true,1,1.52604812622 +fp16,16,64,16,64,16384,control_optimized_fused,4,7,100,1333.49090576,2686976,2.01499386939,2097152,2909,true,1,1.33349090576 +fp16,16,64,16,64,16384,optimized_fused,4,8,100,544.475212097,2686976,4.93498315497,2097152,2909,true,1,0.544475212097 +fp16,16,64,16,64,16384,control_shuffle32_fused,4,9,100,866.47819519,2686976,3.10103129532,2097152,2909,true,1,0.86647819519 +fp16,16,64,16,64,16384,shuffle32_fused,4,10,100,211.378288269,2686976,12.7116934384,2097152,2909,true,1,0.211378288269 +fp16,16,64,16,64,16384,baseline_transform,4,11,100,68.3039999008,4194304,61.4064184541,2097152,2909,true,1,0.0683039999008 +fp16,16,64,16,64,16384,control_optimized_transform,4,12,100,73.9380598068,4194304,56.7272661868,2097152,2909,true,1,0.0739380598068 +fp16,16,64,16,64,16384,optimized_transform,4,13,100,74.0130281448,4194304,56.6698067237,2097152,2909,true,1,0.0740130281448 +fp16,16,64,16,64,16384,control_shuffle32_transform,4,14,100,21.1856007576,4194304,197.978997527,2097152,2909,true,1,0.0211856007576 +fp16,1,1,1,128,1,baseline_transform,0,0,100,9.40937161446,512,0.0544138355864,256,2909,true,1,0.00940937161446 +fp16,1,1,1,128,1,control_optimized_transform,0,1,100,9.54171419144,512,0.053659121383,256,2909,true,1,0.00954171419144 +fp16,1,1,1,128,1,optimized_transform,0,2,100,9.69782888889,512,0.0527953221145,256,2909,true,1,0.00969782888889 +fp16,1,1,1,128,1,control_shuffle32_transform,0,3,100,9.07519996166,512,0.0564174896601,256,2909,true,1,0.00907519996166 +fp16,1,1,1,128,1,shuffle32_transform,0,4,100,9.23862874508,512,0.0554194798955,256,2909,true,1,0.00923862874508 +fp16,1,1,1,128,1,baseline_split,0,5,100,80.2224063873,836,0.0104210287082,256,2909,true,1,0.0802224063873 +fp16,1,1,1,128,1,control_optimized_split,0,6,100,64.7371435165,836,0.0129137610124,256,2909,true,1,0.0647371435165 +fp16,1,1,1,128,1,optimized_split,0,7,100,34.1188573837,836,0.0245025790459,256,2909,true,1,0.0341188573837 +fp16,1,1,1,128,1,control_shuffle32_split,0,8,100,164.729824066,836,0.00507497658508,256,2909,true,1,0.164729824066 +fp16,1,1,1,128,1,shuffle32_split,0,9,100,46.1266279221,836,0.0181240215828,256,2909,true,1,0.0461266279221 +fp16,1,1,1,128,1,baseline_fused,0,10,100,75.781712532,324,0.00427543782232,256,2909,true,1,0.075781712532 +fp16,1,1,1,128,1,control_optimized_fused,0,11,100,59.2516565323,324,0.00546820154848,256,2909,true,1,0.0592516565323 +fp16,1,1,1,128,1,optimized_fused,0,12,100,28.8925719261,324,0.0112139549511,256,2909,true,1,0.0288925719261 +fp16,1,1,1,128,1,control_shuffle32_fused,0,13,100,159.355201721,324,0.00203319374894,256,2909,true,1,0.159355201721 +fp16,1,1,1,128,1,shuffle32_fused,0,14,100,40.3753137589,324,0.00802470544093,256,2909,true,1,0.0403753137589 +fp16,1,1,1,128,1,control_optimized_transform,1,0,100,9.60731387138,512,0.0532927316474,256,2909,true,1,0.00960731387138 +fp16,1,1,1,128,1,optimized_transform,1,1,100,9.60068523884,512,0.0533295267226,256,2909,true,1,0.00960068523884 +fp16,1,1,1,128,1,control_shuffle32_transform,1,2,100,9.1615998745,512,0.0558854356241,256,2909,true,1,0.0091615998745 +fp16,1,1,1,128,1,shuffle32_transform,1,3,100,9.3524569273,512,0.0547449727895,256,2909,true,1,0.0093524569273 +fp16,1,1,1,128,1,baseline_split,1,4,100,80.2882289886,836,0.0104124852488,256,2909,true,1,0.0802882289886 +fp16,1,1,1,128,1,control_optimized_split,1,5,100,64.7590875626,836,0.0129093850989,256,2909,true,1,0.0647590875626 +fp16,1,1,1,128,1,optimized_split,1,6,100,34.1154289246,836,0.0245050414535,256,2909,true,1,0.0341154289246 +fp16,1,1,1,128,1,control_shuffle32_split,1,7,100,165.396118164,836,0.00505453216968,256,2909,true,1,0.165396118164 +fp16,1,1,1,128,1,shuffle32_split,1,8,100,46.1894893646,836,0.0180993557517,256,2909,true,1,0.0461894893646 +fp16,1,1,1,128,1,baseline_fused,1,9,100,75.7771444321,324,0.00427569556003,256,2909,true,1,0.0757771444321 +fp16,1,1,1,128,1,control_optimized_fused,1,10,100,59.2676544189,324,0.00546672553818,256,2909,true,1,0.0592676544189 +fp16,1,1,1,128,1,optimized_fused,1,11,100,28.9115428925,324,0.0112065966595,256,2909,true,1,0.0289115428925 +fp16,1,1,1,128,1,control_shuffle32_fused,1,12,100,159.361829758,324,0.00203310918614,256,2909,true,1,0.159361829758 +fp16,1,1,1,128,1,shuffle32_fused,1,13,100,40.3794288635,324,0.00802388763583,256,2909,true,1,0.0403794288635 +fp16,1,1,1,128,1,baseline_transform,1,14,100,9.40251469612,512,0.0544535176543,256,2909,true,1,0.00940251469612 +fp16,1,1,1,128,1,optimized_transform,2,0,100,9.62080001831,512,0.0532180275056,256,2909,true,1,0.00962080001831 +fp16,1,1,1,128,1,control_shuffle32_transform,2,1,100,9.30057168007,512,0.0550503794404,256,2909,true,1,0.00930057168007 +fp16,1,1,1,128,1,shuffle32_transform,2,2,100,9.20594274998,512,0.055616248537,256,2909,true,1,0.00920594274998 +fp16,1,1,1,128,1,baseline_split,2,3,100,80.2292537689,836,0.0104201392974,256,2909,true,1,0.0802292537689 +fp16,1,1,1,128,1,control_optimized_split,2,4,100,64.725484848,836,0.0129160871017,256,2909,true,1,0.064725484848 +fp16,1,1,1,128,1,optimized_split,2,5,100,34.2324566841,836,0.0244212680298,256,2909,true,1,0.0342324566841 +fp16,1,1,1,128,1,control_shuffle32_split,2,6,100,165.407772064,836,0.00505417604969,256,2909,true,1,0.165407772064 +fp16,1,1,1,128,1,shuffle32_split,2,7,100,46.2233114243,836,0.0180861122719,256,2909,true,1,0.0462233114243 +fp16,1,1,1,128,1,baseline_fused,2,8,100,75.7801151276,324,0.00427552794628,256,2909,true,1,0.0757801151276 +fp16,1,1,1,128,1,control_optimized_fused,2,9,100,59.2573738098,324,0.00546767396476,256,2909,true,1,0.0592573738098 +fp16,1,1,1,128,1,optimized_fused,2,10,100,28.9762282372,324,0.0111815795123,256,2909,true,1,0.0289762282372 +fp16,1,1,1,128,1,control_shuffle32_fused,2,11,100,159.358177185,324,0.00203315578606,256,2909,true,1,0.159358177185 +fp16,1,1,1,128,1,shuffle32_fused,2,12,100,40.3903961182,324,0.00802170889961,256,2909,true,1,0.0403903961182 +fp16,1,1,1,128,1,baseline_transform,2,13,100,9.39771413803,512,0.0544813337031,256,2909,true,1,0.00939771413803 +fp16,1,1,1,128,1,control_optimized_transform,2,14,100,9.64800000191,512,0.053067993356,256,2909,true,1,0.00964800000191 +fp16,1,1,1,128,1,control_shuffle32_transform,3,0,100,9.21280026436,512,0.0555748507846,256,2909,true,1,0.00921280026436 +fp16,1,1,1,128,1,shuffle32_transform,3,1,100,9.2139428854,512,0.0555679589475,256,2909,true,1,0.0092139428854 +fp16,1,1,1,128,1,baseline_split,3,2,100,80.2768039703,836,0.0104139671568,256,2909,true,1,0.0802768039703 +fp16,1,1,1,128,1,control_optimized_split,3,3,100,64.6518850327,836,0.0129307907972,256,2909,true,1,0.0646518850327 +fp16,1,1,1,128,1,optimized_split,3,4,100,34.2635440826,836,0.0243991105527,256,2909,true,1,0.0342635440826 +fp16,1,1,1,128,1,control_shuffle32_split,3,5,100,164.750404358,836,0.00507434262913,256,2909,true,1,0.164750404358 +fp16,1,1,1,128,1,shuffle32_split,3,6,100,46.1958837509,836,0.0180968504577,256,2909,true,1,0.0461958837509 +fp16,1,1,1,128,1,baseline_fused,3,7,100,75.723657608,324,0.00427871566475,256,2909,true,1,0.075723657608 +fp16,1,1,1,128,1,control_optimized_fused,3,8,100,59.2633152008,324,0.00546712580797,256,2909,true,1,0.0592633152008 +fp16,1,1,1,128,1,optimized_fused,3,9,100,28.9126849174,324,0.011206154009,256,2909,true,1,0.0289126849174 +fp16,1,1,1,128,1,control_shuffle32_fused,3,10,100,159.351549149,324,0.00203324035274,256,2909,true,1,0.159351549149 +fp16,1,1,1,128,1,shuffle32_fused,3,11,100,40.4230880737,324,0.00801522138559,256,2909,true,1,0.0404230880737 +fp16,1,1,1,128,1,baseline_transform,3,12,100,9.42308545113,512,0.0543346447037,256,2909,true,1,0.00942308545113 +fp16,1,1,1,128,1,control_optimized_transform,3,13,100,9.63017165661,512,0.053166238179,256,2909,true,1,0.00963017165661 +fp16,1,1,1,128,1,optimized_transform,3,14,100,9.63451445103,512,0.0531422732928,256,2909,true,1,0.00963451445103 +fp16,1,1,1,128,1,shuffle32_transform,4,0,100,9.24045681953,512,0.0554085160506,256,2909,true,1,0.00924045681953 +fp16,1,1,1,128,1,baseline_split,4,1,100,80.1421737671,836,0.0104314614978,256,2909,true,1,0.0801421737671 +fp16,1,1,1,128,1,control_optimized_split,4,2,100,64.7056007385,836,0.012920056231,256,2909,true,1,0.0647056007385 +fp16,1,1,1,128,1,optimized_split,4,3,100,34.2694854736,836,0.0243948804146,256,2909,true,1,0.0342694854736 +fp16,1,1,1,128,1,control_shuffle32_split,4,4,100,164.768009186,836,0.00507380045514,256,2909,true,1,0.164768009186 +fp16,1,1,1,128,1,shuffle32_split,4,5,100,46.1913156509,836,0.0180986401495,256,2909,true,1,0.0461913156509 +fp16,1,1,1,128,1,baseline_fused,4,6,100,75.781712532,324,0.00427543782232,256,2909,true,1,0.075781712532 +fp16,1,1,1,128,1,control_optimized_fused,4,7,100,59.2621707916,324,0.00546723138339,256,2909,true,1,0.0592621707916 +fp16,1,1,1,128,1,optimized_fused,4,8,100,28.9490294456,324,0.0111920850614,256,2909,true,1,0.0289490294456 +fp16,1,1,1,128,1,control_shuffle32_fused,4,9,100,159.353151321,324,0.00203321991008,256,2909,true,1,0.159353151321 +fp16,1,1,1,128,1,shuffle32_fused,4,10,100,40.3766822815,324,0.00802443345249,256,2909,true,1,0.0403766822815 +fp16,1,1,1,128,1,baseline_transform,4,11,100,9.44525718689,512,0.0542070999094,256,2909,true,1,0.00944525718689 +fp16,1,1,1,128,1,control_optimized_transform,4,12,100,9.72000002861,512,0.0526748969643,256,2909,true,1,0.00972000002861 +fp16,1,1,1,128,1,optimized_transform,4,13,100,9.64182853699,512,0.0531019606951,256,2909,true,1,0.00964182853699 +fp16,1,1,1,128,1,control_shuffle32_transform,4,14,100,9.27222847939,512,0.0552186565655,256,2909,true,1,0.00927222847939 +fp16,1,17,1,128,17,baseline_transform,0,0,100,8.83565664291,8704,0.985099393488,4352,2909,true,1,0.00883565664291 +fp16,1,17,1,128,17,control_optimized_transform,0,1,100,9.75542902946,8704,0.892221138989,4352,2909,true,1,0.00975542902946 +fp16,1,17,1,128,17,optimized_transform,0,2,100,9.80617105961,8704,0.887604340888,4352,2909,true,1,0.00980617105961 +fp16,1,17,1,128,17,control_shuffle32_transform,0,3,100,9.15657162666,8704,0.950574118227,4352,2909,true,1,0.00915657162666 +fp16,1,17,1,128,17,shuffle32_transform,0,4,100,9.24091458321,8704,0.9418981121,4352,2909,true,1,0.00924091458321 +fp16,1,17,1,128,17,baseline_split,0,5,100,80.4194259644,14212,0.176723469853,4352,2909,true,1,0.0804194259644 +fp16,1,17,1,128,17,control_optimized_split,0,6,100,64.4665145874,14212,0.220455535575,4352,2909,true,1,0.0644665145874 +fp16,1,17,1,128,17,optimized_split,0,7,100,34.6946287155,14212,0.40963113099,4352,2909,true,1,0.0346946287155 +fp16,1,17,1,128,17,control_shuffle32_split,0,8,100,163.414402008,14212,0.0869690787676,4352,2909,true,1,0.163414402008 +fp16,1,17,1,128,17,shuffle32_split,0,9,100,45.6745147705,14212,0.311158204338,4352,2909,true,1,0.0456745147705 +fp16,1,17,1,128,17,baseline_fused,0,10,100,75.73097229,5508,0.0727311406871,4352,2909,true,1,0.07573097229 +fp16,1,17,1,128,17,control_optimized_fused,0,11,100,59.3401145935,5508,0.0928208520953,4352,2909,true,1,0.0593401145935 +fp16,1,17,1,128,17,optimized_fused,0,12,100,29.3711996078,5508,0.187530644766,4352,2909,true,1,0.0293711996078 +fp16,1,17,1,128,17,control_shuffle32_fused,0,13,100,157.948560715,5508,0.0348721126364,4352,2909,true,1,0.157948560715 +fp16,1,17,1,128,17,shuffle32_fused,0,14,100,39.9348568916,5508,0.137924620963,4352,2909,true,1,0.0399348568916 +fp16,1,17,1,128,17,control_optimized_transform,1,0,100,9.76228535175,8704,0.891594507472,4352,2909,true,1,0.00976228535175 +fp16,1,17,1,128,17,optimized_transform,1,1,100,9.78171467781,8704,0.889823541852,4352,2909,true,1,0.00978171467781 +fp16,1,17,1,128,17,control_shuffle32_transform,1,2,100,9.27382826805,8704,0.938555227509,4352,2909,true,1,0.00927382826805 +fp16,1,17,1,128,17,shuffle32_transform,1,3,100,9.40160036087,8704,0.925799828317,4352,2909,true,1,0.00940160036087 +fp16,1,17,1,128,17,baseline_split,1,4,100,80.366859436,14212,0.17683906152,4352,2909,true,1,0.080366859436 +fp16,1,17,1,128,17,control_optimized_split,1,5,100,64.4927978516,14212,0.220365691572,4352,2909,true,1,0.0644927978516 +fp16,1,17,1,128,17,optimized_split,1,6,100,35.388572216,14212,0.401598570105,4352,2909,true,1,0.035388572216 +fp16,1,17,1,128,17,control_shuffle32_split,1,7,100,163.380794525,14212,0.0869869683356,4352,2909,true,1,0.163380794525 +fp16,1,17,1,128,17,shuffle32_split,1,8,100,45.6610298157,14212,0.311250097892,4352,2909,true,1,0.0456610298157 +fp16,1,17,1,128,17,baseline_fused,1,9,100,75.7284545898,5508,0.0727335587374,4352,2909,true,1,0.0757284545898 +fp16,1,17,1,128,17,control_optimized_fused,1,10,100,59.3622875214,5508,0.092786181766,4352,2909,true,1,0.0593622875214 +fp16,1,17,1,128,17,optimized_fused,1,11,100,29.3835449219,5508,0.187451854929,4352,2909,true,1,0.0293835449219 +fp16,1,17,1,128,17,control_shuffle32_fused,1,12,100,157.988576889,5508,0.034863280045,4352,2909,true,1,0.157988576889 +fp16,1,17,1,128,17,shuffle32_fused,1,13,100,39.9508571625,5508,0.137869382316,4352,2909,true,1,0.0399508571625 +fp16,1,17,1,128,17,baseline_transform,1,14,100,9.68822836876,8704,0.898409871104,4352,2909,true,1,0.00968822836876 +fp16,1,17,1,128,17,optimized_transform,2,0,100,9.78034317493,8704,0.889948322295,4352,2909,true,1,0.00978034317493 +fp16,1,17,1,128,17,control_shuffle32_transform,2,1,100,9.28137183189,8704,0.937792403715,4352,2909,true,1,0.00928137183189 +fp16,1,17,1,128,17,shuffle32_transform,2,2,100,9.37417149544,8704,0.928508722529,4352,2909,true,1,0.00937417149544 +fp16,1,17,1,128,17,baseline_split,2,3,100,80.3305149078,14212,0.176919070123,4352,2909,true,1,0.0803305149078 +fp16,1,17,1,128,17,control_optimized_split,2,4,100,64.4895982742,14212,0.220376624763,4352,2909,true,1,0.0644895982742 +fp16,1,17,1,128,17,optimized_split,2,5,100,34.7017145157,14212,0.409547487735,4352,2909,true,1,0.0347017145157 +fp16,1,17,1,128,17,control_shuffle32_split,2,6,100,163.313827515,14212,0.0870226374354,4352,2909,true,1,0.163313827515 +fp16,1,17,1,128,17,shuffle32_split,2,7,100,45.6429672241,14212,0.31137327094,4352,2909,true,1,0.0456429672241 +fp16,1,17,1,128,17,baseline_fused,2,8,100,75.7311964035,5508,0.0727309254518,4352,2909,true,1,0.0757311964035 +fp16,1,17,1,128,17,control_optimized_fused,2,9,100,59.3364572525,5508,0.0928265733251,4352,2909,true,1,0.0593364572525 +fp16,1,17,1,128,17,optimized_fused,2,10,100,29.3275427818,5508,0.187809801898,4352,2909,true,1,0.0293275427818 +fp16,1,17,1,128,17,control_shuffle32_fused,2,11,100,158.006401062,5508,0.0348593472352,4352,2909,true,1,0.158006401062 +fp16,1,17,1,128,17,shuffle32_fused,2,12,100,39.9419426918,5508,0.137900152792,4352,2909,true,1,0.0399419426918 +fp16,1,17,1,128,17,baseline_transform,2,13,100,9.59680020809,8704,0.906968969997,4352,2909,true,1,0.00959680020809 +fp16,1,17,1,128,17,control_optimized_transform,2,14,100,9.7922283411,8704,0.88886816124,4352,2909,true,1,0.0097922283411 +fp16,1,17,1,128,17,control_shuffle32_transform,3,0,100,9.26285684109,8704,0.939666902914,4352,2909,true,1,0.00926285684109 +fp16,1,17,1,128,17,shuffle32_transform,3,1,100,9.33120012283,8704,0.932784624209,4352,2909,true,1,0.00933120012283 +fp16,1,17,1,128,17,baseline_split,3,2,100,80.3385162354,14212,0.17690144984,4352,2909,true,1,0.0803385162354 +fp16,1,17,1,128,17,control_optimized_split,3,3,100,64.3949699402,14212,0.220700467959,4352,2909,true,1,0.0643949699402 +fp16,1,17,1,128,17,optimized_split,3,4,100,34.7019433975,14212,0.409544786504,4352,2909,true,1,0.0347019433975 +fp16,1,17,1,128,17,control_shuffle32_split,3,5,100,164.017829895,14212,0.0866491161912,4352,2909,true,1,0.164017829895 +fp16,1,17,1,128,17,shuffle32_split,3,6,100,45.7615995407,14212,0.310566067241,4352,2909,true,1,0.0457615995407 +fp16,1,17,1,128,17,baseline_fused,3,7,100,75.7307434082,5508,0.0727313605032,4352,2909,true,1,0.0757307434082 +fp16,1,17,1,128,17,control_optimized_fused,3,8,100,59.339427948,5508,0.0928219261707,4352,2909,true,1,0.059339427948 +fp16,1,17,1,128,17,optimized_fused,3,9,100,29.4333720207,5508,0.187134521866,4352,2909,true,1,0.0294333720207 +fp16,1,17,1,128,17,control_shuffle32_fused,3,10,100,157.994737625,5508,0.0348619206107,4352,2909,true,1,0.157994737625 +fp16,1,17,1,128,17,shuffle32_fused,3,11,100,39.9410295486,5508,0.137903305504,4352,2909,true,1,0.0399410295486 +fp16,1,17,1,128,17,baseline_transform,3,12,100,9.55702841282,8704,0.910743342389,4352,2909,true,1,0.00955702841282 +fp16,1,17,1,128,17,control_optimized_transform,3,13,100,9.67108607292,8704,0.900002330077,4352,2909,true,1,0.00967108607292 +fp16,1,17,1,128,17,optimized_transform,3,14,100,9.86377120018,8704,0.88242111697,4352,2909,true,1,0.00986377120018 +fp16,1,17,1,128,17,shuffle32_transform,4,0,100,9.29965674877,8704,0.93594852317,4352,2909,true,1,0.00929965674877 +fp16,1,17,1,128,17,baseline_split,4,1,100,80.3385162354,14212,0.17690144984,4352,2909,true,1,0.0803385162354 +fp16,1,17,1,128,17,control_optimized_split,4,2,100,64.4395399094,14212,0.220547819242,4352,2909,true,1,0.0644395399094 +fp16,1,17,1,128,17,optimized_split,4,3,100,34.6500563622,14212,0.41015806299,4352,2909,true,1,0.0346500563622 +fp16,1,17,1,128,17,control_shuffle32_split,4,4,100,163.365936279,14212,0.086994879861,4352,2909,true,1,0.163365936279 +fp16,1,17,1,128,17,shuffle32_split,4,5,100,45.6399965286,14212,0.311393538146,4352,2909,true,1,0.0456399965286 +fp16,1,17,1,128,17,baseline_fused,4,6,100,75.7339429855,5508,0.0727282877778,4352,2909,true,1,0.0757339429855 +fp16,1,17,1,128,17,control_optimized_fused,4,7,100,59.3405723572,5508,0.0928201360588,4352,2909,true,1,0.0593405723572 +fp16,1,17,1,128,17,optimized_fused,4,8,100,29.392914772,5508,0.187392099175,4352,2909,true,1,0.029392914772 +fp16,1,17,1,128,17,control_shuffle32_fused,4,9,100,157.943534851,5508,0.0348732222892,4352,2909,true,1,0.157943534851 +fp16,1,17,1,128,17,shuffle32_fused,4,10,100,39.8854875565,5508,0.138095340873,4352,2909,true,1,0.0398854875565 +fp16,1,17,1,128,17,baseline_transform,4,11,100,9.51245725155,8704,0.915010682291,4352,2909,true,1,0.00951245725155 +fp16,1,17,1,128,17,control_optimized_transform,4,12,100,9.79497075081,8704,0.888619294681,4352,2909,true,1,0.00979497075081 +fp16,1,17,1,128,17,optimized_transform,4,13,100,9.77485656738,8704,0.890447848518,4352,2909,true,1,0.00977485656738 +fp16,1,17,1,128,17,control_shuffle32_transform,4,14,100,9.19177114964,8704,0.946933932352,4352,2909,true,1,0.00919177114964 +fp16,1,257,1,128,257,baseline_transform,0,0,100,10.0226283073,131584,13.1286919923,65792,2909,true,1,0.0100226283073 +fp16,1,257,1,128,257,control_optimized_transform,0,1,100,9.9746286869,131584,13.1918695052,65792,2909,true,1,0.0099746286869 +fp16,1,257,1,128,257,optimized_transform,0,2,100,9.92160081863,131584,13.2623759417,65792,2909,true,1,0.00992160081863 +fp16,1,257,1,128,257,control_shuffle32_transform,0,3,100,10.570743084,131584,12.4479423022,65792,2909,true,1,0.010570743084 +fp16,1,257,1,128,257,shuffle32_transform,0,4,100,10.5622851849,131584,12.4579101678,65792,2909,true,1,0.0105622851849 +fp16,1,257,1,128,257,baseline_split,0,5,100,81.7037773132,214852,2.62964586296,65792,2909,true,1,0.0817037773132 +fp16,1,257,1,128,257,control_optimized_split,0,6,100,65.6845712662,214852,3.27096601011,65792,2909,true,1,0.0656845712662 +fp16,1,257,1,128,257,optimized_split,0,7,100,35.1421713829,214852,6.11379409824,65792,2909,true,1,0.0351421713829 +fp16,1,257,1,128,257,control_shuffle32_split,0,8,100,177.914047241,214852,1.20761684269,65792,2909,true,1,0.177914047241 +fp16,1,257,1,128,257,shuffle32_split,0,9,100,51.5556573868,214852,4.16737969973,65792,2909,true,1,0.0515556573868 +fp16,1,257,1,128,257,baseline_fused,0,10,100,76.627202034,83268,1.08666371458,65792,2909,true,1,0.076627202034 +fp16,1,257,1,128,257,control_optimized_fused,0,11,100,60.2514314651,83268,1.38200865897,65792,2909,true,1,0.0602514314651 +fp16,1,257,1,128,257,optimized_fused,0,12,100,29.7019410133,83268,2.8034531468,65792,2909,true,1,0.0297019410133 +fp16,1,257,1,128,257,control_shuffle32_fused,0,13,100,170.360012054,83268,0.488776673562,65792,2909,true,1,0.170360012054 +fp16,1,257,1,128,257,shuffle32_fused,0,14,100,44.9167966843,83268,1.85382765795,65792,2909,true,1,0.0449167966843 +fp16,1,257,1,128,257,control_optimized_transform,1,0,100,10.0637722015,131584,13.0750177334,65792,2909,true,1,0.0100637722015 +fp16,1,257,1,128,257,optimized_transform,1,1,100,10.1648008823,131584,12.9450641998,65792,2909,true,1,0.0101648008823 +fp16,1,257,1,128,257,control_shuffle32_transform,1,2,100,10.5065143108,131584,12.5240394775,65792,2909,true,1,0.0105065143108 +fp16,1,257,1,128,257,shuffle32_transform,1,3,100,10.5814862251,131584,12.4353041908,65792,2909,true,1,0.0105814862251 +fp16,1,257,1,128,257,baseline_split,1,4,100,81.6893672943,214852,2.63010973296,65792,2909,true,1,0.0816893672943 +fp16,1,257,1,128,257,control_optimized_split,1,5,100,65.7165718079,214852,3.26937322032,65792,2909,true,1,0.0657165718079 +fp16,1,257,1,128,257,optimized_split,1,6,100,35.1839995384,214852,6.10652577361,65792,2909,true,1,0.0351839995384 +fp16,1,257,1,128,257,control_shuffle32_split,1,7,100,177.899646759,214852,1.20771459592,65792,2909,true,1,0.177899646759 +fp16,1,257,1,128,257,shuffle32_split,1,8,100,51.6020536423,214852,4.16363274007,65792,2909,true,1,0.0516020536423 +fp16,1,257,1,128,257,baseline_fused,1,9,100,76.6726875305,83268,1.08601905948,65792,2909,true,1,0.0766726875305 +fp16,1,257,1,128,257,control_optimized_fused,1,10,100,60.2667427063,83268,1.38165754877,65792,2909,true,1,0.0602667427063 +fp16,1,257,1,128,257,optimized_fused,1,11,100,29.6802282333,83268,2.8055040327,65792,2909,true,1,0.0296802282333 +fp16,1,257,1,128,257,control_shuffle32_fused,1,12,100,170.361824036,83268,0.488771474897,65792,2909,true,1,0.170361824036 +fp16,1,257,1,128,257,shuffle32_fused,1,13,100,44.9072027206,83268,1.85422370923,65792,2909,true,1,0.0449072027206 +fp16,1,257,1,128,257,baseline_transform,1,14,100,9.98880028725,131584,13.1731535536,65792,2909,true,1,0.00998880028725 +fp16,1,257,1,128,257,optimized_transform,2,0,100,10.0409150124,131584,13.1047817692,65792,2909,true,1,0.0100409150124 +fp16,1,257,1,128,257,control_shuffle32_transform,2,1,100,10.5284571648,131584,12.4979375364,65792,2909,true,1,0.0105284571648 +fp16,1,257,1,128,257,shuffle32_transform,2,2,100,10.6674277782,131584,12.3351198373,65792,2909,true,1,0.0106674277782 +fp16,1,257,1,128,257,baseline_split,2,3,100,81.6731452942,214852,2.63063212793,65792,2909,true,1,0.0816731452942 +fp16,1,257,1,128,257,control_optimized_split,2,4,100,65.7257175446,214852,3.26891828688,65792,2909,true,1,0.0657257175446 +fp16,1,257,1,128,257,optimized_split,2,5,100,35.2349710464,214852,6.09769196963,65792,2909,true,1,0.0352349710464 +fp16,1,257,1,128,257,control_shuffle32_split,2,6,100,177.282752991,214852,1.21191710065,65792,2909,true,1,0.177282752991 +fp16,1,257,1,128,257,shuffle32_split,2,7,100,51.5371417999,214852,4.16887690113,65792,2909,true,1,0.0515371417999 +fp16,1,257,1,128,257,baseline_fused,2,8,100,76.6843414307,83268,1.08585401461,65792,2909,true,1,0.0766843414307 +fp16,1,257,1,128,257,control_optimized_fused,2,9,100,60.2596569061,83268,1.38182001484,65792,2909,true,1,0.0602596569061 +fp16,1,257,1,128,257,optimized_fused,2,10,100,29.7693729401,83268,2.79710292077,65792,2909,true,1,0.0297693729401 +fp16,1,257,1,128,257,control_shuffle32_fused,2,11,100,170.305843353,83268,0.488932137386,65792,2909,true,1,0.170305843353 +fp16,1,257,1,128,257,shuffle32_fused,2,12,100,44.9165725708,83268,1.85383690772,65792,2909,true,1,0.0449165725708 +fp16,1,257,1,128,257,baseline_transform,2,13,100,10.0463998318,131584,13.09762723,65792,2909,true,1,0.0100463998318 +fp16,1,257,1,128,257,control_optimized_transform,2,14,100,9.92457151413,131584,13.2584061501,65792,2909,true,1,0.00992457151413 +fp16,1,257,1,128,257,control_shuffle32_transform,3,0,100,10.4763424397,131584,12.5601087171,65792,2909,true,1,0.0104763424397 +fp16,1,257,1,128,257,shuffle32_transform,3,1,100,10.5275428295,131584,12.4990230038,65792,2909,true,1,0.0105275428295 +fp16,1,257,1,128,257,baseline_split,3,2,100,81.709022522,214852,2.62947705613,65792,2909,true,1,0.081709022522 +fp16,1,257,1,128,257,control_optimized_split,3,3,100,65.747885704,214852,3.26781610845,65792,2909,true,1,0.065747885704 +fp16,1,257,1,128,257,optimized_split,3,4,100,35.2438855171,214852,6.09614963979,65792,2909,true,1,0.0352438855171 +fp16,1,257,1,128,257,control_shuffle32_split,3,5,100,177.920455933,214852,1.20757334436,65792,2909,true,1,0.177920455933 +fp16,1,257,1,128,257,shuffle32_split,3,6,100,51.5755414963,214852,4.1657730344,65792,2909,true,1,0.0515755414963 +fp16,1,257,1,128,257,baseline_fused,3,7,100,76.6868543625,83268,1.08581843254,65792,2909,true,1,0.0766868543625 +fp16,1,257,1,128,257,control_optimized_fused,3,8,100,60.2596569061,83268,1.38182001484,65792,2909,true,1,0.0602596569061 +fp16,1,257,1,128,257,optimized_fused,3,9,100,29.6365714073,83268,2.80963674426,65792,2909,true,1,0.0296365714073 +fp16,1,257,1,128,257,control_shuffle32_fused,3,10,100,170.358409882,83268,0.488781270369,65792,2909,true,1,0.170358409882 +fp16,1,257,1,128,257,shuffle32_fused,3,11,100,44.9698257446,83268,1.85164159792,65792,2909,true,1,0.0449698257446 +fp16,1,257,1,128,257,baseline_transform,3,12,100,9.99039947987,131584,13.1710448882,65792,2909,true,1,0.00999039947987 +fp16,1,257,1,128,257,control_optimized_transform,3,13,100,10.1083421707,131584,13.01736702,65792,2909,true,1,0.0101083421707 +fp16,1,257,1,128,257,optimized_transform,3,14,100,10.015771389,131584,13.1376800537,65792,2909,true,1,0.010015771389 +fp16,1,257,1,128,257,shuffle32_transform,4,0,100,10.6118857861,131584,12.3996811361,65792,2909,true,1,0.0106118857861 +fp16,1,257,1,128,257,baseline_split,4,1,100,81.7026329041,214852,2.62968269642,65792,2909,true,1,0.0817026329041 +fp16,1,257,1,128,257,control_optimized_split,4,2,100,65.6527996063,214852,3.27254894366,65792,2909,true,1,0.0656527996063 +fp16,1,257,1,128,257,optimized_split,4,3,100,35.2201151848,214852,6.10026397906,65792,2909,true,1,0.0352201151848 +fp16,1,257,1,128,257,control_shuffle32_split,4,4,100,177.278633118,214852,1.21194526504,65792,2909,true,1,0.177278633118 +fp16,1,257,1,128,257,shuffle32_split,4,5,100,51.5414857864,214852,4.16852554252,65792,2909,true,1,0.0515414857864 +fp16,1,257,1,128,257,baseline_fused,4,6,100,76.6930246353,83268,1.08573107393,65792,2909,true,1,0.0766930246353 +fp16,1,257,1,128,257,control_optimized_fused,4,7,100,60.2585124969,83268,1.38184625789,65792,2909,true,1,0.0602585124969 +fp16,1,257,1,128,257,optimized_fused,4,8,100,29.6763420105,83268,2.80587142346,65792,2909,true,1,0.0296763420105 +fp16,1,257,1,128,257,control_shuffle32_fused,4,9,100,170.370750427,83268,0.488745866243,65792,2909,true,1,0.170370750427 +fp16,1,257,1,128,257,shuffle32_fused,4,10,100,44.9076557159,83268,1.85420500519,65792,2909,true,1,0.0449076557159 +fp16,1,257,1,128,257,baseline_transform,4,11,100,9.92114305496,131584,13.2629878706,65792,2909,true,1,0.00992114305496 +fp16,1,257,1,128,257,control_optimized_transform,4,12,100,10.0902855396,131584,13.0406616823,65792,2909,true,1,0.0100902855396 +fp16,1,257,1,128,257,optimized_transform,4,13,100,9.92274284363,131584,13.2608495528,65792,2909,true,1,0.00992274284363 +fp16,1,257,1,128,257,control_shuffle32_transform,4,14,100,10.4587435722,131584,12.5812435396,65792,2909,true,1,0.0104587435722 +fp16,4,64,16,128,4096,baseline_transform,0,0,100,25.8589720726,2097152,81.0995887274,1048576,2909,true,1,0.0258589720726 +fp16,4,64,16,128,4096,control_optimized_transform,0,1,100,27.7503991127,2097152,75.5719581359,1048576,2909,true,1,0.0277503991127 +fp16,4,64,16,128,4096,optimized_transform,0,2,100,27.8566837311,2097152,75.2836202703,1048576,2909,true,1,0.0278566837311 +fp16,4,64,16,128,4096,control_shuffle32_transform,0,3,100,15.9828579426,2097152,131.212578347,1048576,2909,true,1,0.0159828579426 +fp16,4,64,16,128,4096,shuffle32_transform,0,4,100,15.7522284985,2097152,133.133670592,1048576,2909,true,1,0.0157522284985 +fp16,4,64,16,128,4096,baseline_split,0,5,100,527.073822021,3424256,6.49672940854,1048576,2909,true,1,0.527073822021 +fp16,4,64,16,128,4096,control_optimized_split,0,6,100,405.66947937,3424256,8.44099981423,1048576,2909,true,1,0.40566947937 +fp16,4,64,16,128,4096,optimized_split,0,7,100,168.860359192,3424256,20.2786255838,1048576,2909,true,1,0.168860359192 +fp16,4,64,16,128,4096,control_shuffle32_split,0,8,100,453.523445129,3424256,7.55033953983,1048576,2909,true,1,0.453523445129 +fp16,4,64,16,128,4096,shuffle32_split,0,9,100,105.785827637,3424256,32.3697046807,1048576,2909,true,1,0.105785827637 +fp16,4,64,16,128,4096,baseline_fused,0,10,100,528.91292572,1327104,2.50911621831,1048576,2909,true,1,0.52891292572 +fp16,4,64,16,128,4096,control_optimized_fused,0,11,100,394.402313232,1327104,3.36484841867,1048576,2909,true,1,0.394402313232 +fp16,4,64,16,128,4096,optimized_fused,0,12,100,153.630399704,1327104,8.63829035502,1048576,2909,true,1,0.153630399704 +fp16,4,64,16,128,4096,control_shuffle32_fused,0,13,100,441.381492615,1327104,3.00670513423,1048576,2909,true,1,0.441381492615 +fp16,4,64,16,128,4096,shuffle32_fused,0,14,100,92.8765678406,1327104,14.2889001053,1048576,2909,true,1,0.0928765678406 +fp16,4,64,16,128,4096,control_optimized_transform,1,0,100,27.7307415009,2097152,75.62552916,1048576,2909,true,1,0.0277307415009 +fp16,4,64,16,128,4096,optimized_transform,1,1,100,27.8706288338,2097152,75.2459520202,1048576,2909,true,1,0.0278706288338 +fp16,4,64,16,128,4096,control_shuffle32_transform,1,2,100,15.7618284225,2097152,133.052583988,1048576,2909,true,1,0.0157618284225 +fp16,4,64,16,128,4096,shuffle32_transform,1,3,100,15.7026278973,2097152,133.554205941,1048576,2909,true,1,0.0157026278973 +fp16,4,64,16,128,4096,baseline_split,1,4,100,522.950172424,3424256,6.54795844913,1048576,2909,true,1,0.522950172424 +fp16,4,64,16,128,4096,control_optimized_split,1,5,100,402.373695374,3424256,8.51013880721,1048576,2909,true,1,0.402373695374 +fp16,4,64,16,128,4096,optimized_split,1,6,100,168.744010925,3424256,20.2926076086,1048576,2909,true,1,0.168744010925 +fp16,4,64,16,128,4096,control_shuffle32_split,1,7,100,453.677711487,3424256,7.54777215918,1048576,2909,true,1,0.453677711487 +fp16,4,64,16,128,4096,shuffle32_split,1,8,100,105.981712341,3424256,32.3098761508,1048576,2909,true,1,0.105981712341 +fp16,4,64,16,128,4096,baseline_fused,1,9,100,528.95565033,1327104,2.50891355291,1048576,2909,true,1,0.52895565033 +fp16,4,64,16,128,4096,control_optimized_fused,1,10,100,392.10811615,1327104,3.38453591074,1048576,2909,true,1,0.39210811615 +fp16,4,64,16,128,4096,optimized_fused,1,11,100,153.760690689,1327104,8.63097059497,1048576,2909,true,1,0.153760690689 +fp16,4,64,16,128,4096,control_shuffle32_fused,1,12,100,442.118148804,1327104,3.00169536942,1048576,2909,true,1,0.442118148804 +fp16,4,64,16,128,4096,shuffle32_fused,1,13,100,92.9167938232,1327104,14.2827140864,1048576,2909,true,1,0.0929167938232 +fp16,4,64,16,128,4096,baseline_transform,1,14,100,25.8639979362,2097152,81.0838295444,1048576,2909,true,1,0.0258639979362 +fp16,4,64,16,128,4096,optimized_transform,2,0,100,27.8514266014,2097152,75.297830521,1048576,2909,true,1,0.0278514266014 +fp16,4,64,16,128,4096,control_shuffle32_transform,2,1,100,15.7798862457,2097152,132.900324333,1048576,2909,true,1,0.0157798862457 +fp16,4,64,16,128,4096,shuffle32_transform,2,2,100,15.5757713318,2097152,134.641935563,1048576,2909,true,1,0.0155757713318 +fp16,4,64,16,128,4096,baseline_split,2,3,100,525.725021362,3424256,6.51339742424,1048576,2909,true,1,0.525725021362 +fp16,4,64,16,128,4096,control_optimized_split,2,4,100,403.02947998,3424256,8.49629163645,1048576,2909,true,1,0.40302947998 +fp16,4,64,16,128,4096,optimized_split,2,5,100,168.751087189,3424256,20.2917566757,1048576,2909,true,1,0.168751087189 +fp16,4,64,16,128,4096,control_shuffle32_split,2,6,100,454.682731628,3424256,7.53108873903,1048576,2909,true,1,0.454682731628 +fp16,4,64,16,128,4096,shuffle32_split,2,7,100,105.945596695,3424256,32.3208902193,1048576,2909,true,1,0.105945596695 +fp16,4,64,16,128,4096,baseline_fused,2,8,100,528.973503113,1327104,2.50882887742,1048576,2909,true,1,0.528973503113 +fp16,4,64,16,128,4096,control_optimized_fused,2,9,100,395.210266113,1327104,3.35796945017,1048576,2909,true,1,0.395210266113 +fp16,4,64,16,128,4096,optimized_fused,2,10,100,153.649377823,1327104,8.63722339006,1048576,2909,true,1,0.153649377823 +fp16,4,64,16,128,4096,control_shuffle32_fused,2,11,100,443.393135071,1327104,2.99306393138,1048576,2909,true,1,0.443393135071 +fp16,4,64,16,128,4096,shuffle32_fused,2,12,100,92.8379440308,1327104,14.2948447842,1048576,2909,true,1,0.0928379440308 +fp16,4,64,16,128,4096,baseline_transform,2,13,100,25.8448004723,2097152,81.1440584442,1048576,2909,true,1,0.0258448004723 +fp16,4,64,16,128,4096,control_optimized_transform,2,14,100,27.718398571,2097152,75.6592050088,1048576,2909,true,1,0.027718398571 +fp16,4,64,16,128,4096,control_shuffle32_transform,3,0,100,15.9773719311,2097152,131.257631671,1048576,2909,true,1,0.0159773719311 +fp16,4,64,16,128,4096,shuffle32_transform,3,1,100,15.7476568222,2097152,133.172320408,1048576,2909,true,1,0.0157476568222 +fp16,4,64,16,128,4096,baseline_split,3,2,100,525.402069092,3424256,6.51740105615,1048576,2909,true,1,0.525402069092 +fp16,4,64,16,128,4096,control_optimized_split,3,3,100,402.566871643,3424256,8.50605512079,1048576,2909,true,1,0.402566871643 +fp16,4,64,16,128,4096,optimized_split,3,4,100,168.979434967,3424256,20.2643357203,1048576,2909,true,1,0.168979434967 +fp16,4,64,16,128,4096,control_shuffle32_split,3,5,100,455.47542572,3424256,7.51798188582,1048576,2909,true,1,0.45547542572 +fp16,4,64,16,128,4096,shuffle32_split,3,6,100,105.602741241,3424256,32.4258249336,1048576,2909,true,1,0.105602741241 +fp16,4,64,16,128,4096,baseline_fused,3,7,100,529.013710022,1327104,2.50863819757,1048576,2909,true,1,0.529013710022 +fp16,4,64,16,128,4096,control_optimized_fused,3,8,100,392.721138,1327104,3.37925278674,1048576,2909,true,1,0.392721138 +fp16,4,64,16,128,4096,optimized_fused,3,9,100,153.322973251,1327104,8.65561090982,1048576,2909,true,1,0.153322973251 +fp16,4,64,16,128,4096,control_shuffle32_fused,3,10,100,439.327087402,1327104,3.02076525226,1048576,2909,true,1,0.439327087402 +fp16,4,64,16,128,4096,shuffle32_fused,3,11,100,92.9314231873,1327104,14.2804656863,1048576,2909,true,1,0.0929314231873 +fp16,4,64,16,128,4096,baseline_transform,3,12,100,25.9161138535,2097152,80.9207743051,1048576,2909,true,1,0.0259161138535 +fp16,4,64,16,128,4096,control_optimized_transform,3,13,100,27.7602267265,2097152,75.5452043191,1048576,2909,true,1,0.0277602267265 +fp16,4,64,16,128,4096,optimized_transform,3,14,100,27.8429698944,2097152,75.3207006276,1048576,2909,true,1,0.0278429698944 +fp16,4,64,16,128,4096,shuffle32_transform,4,0,100,15.6434285641,2097152,134.059614324,1048576,2909,true,1,0.0156434285641 +fp16,4,64,16,128,4096,baseline_split,4,1,100,525.461463928,3424256,6.5166643704,1048576,2909,true,1,0.525461463928 +fp16,4,64,16,128,4096,control_optimized_split,4,2,100,402.413253784,3424256,8.50930223545,1048576,2909,true,1,0.402413253784 +fp16,4,64,16,128,4096,optimized_split,4,3,100,168.891429901,3424256,20.2748949547,1048576,2909,true,1,0.168891429901 +fp16,4,64,16,128,4096,control_shuffle32_split,4,4,100,452.616462708,3424256,7.5654694032,1048576,2909,true,1,0.452616462708 +fp16,4,64,16,128,4096,shuffle32_split,4,5,100,105.7508564,3424256,32.3804091672,1048576,2909,true,1,0.1057508564 +fp16,4,64,16,128,4096,baseline_fused,4,6,100,528.882522583,1327104,2.5092604564,1048576,2909,true,1,0.528882522583 +fp16,4,64,16,128,4096,control_optimized_fused,4,7,100,394.329147339,1327104,3.36547275025,1048576,2909,true,1,0.394329147339 +fp16,4,64,16,128,4096,optimized_fused,4,8,100,153.86879921,1327104,8.62490645808,1048576,2909,true,1,0.15386879921 +fp16,4,64,16,128,4096,control_shuffle32_fused,4,9,100,441.643676758,1327104,3.00492018757,1048576,2909,true,1,0.441643676758 +fp16,4,64,16,128,4096,shuffle32_fused,4,10,100,92.8672027588,1327104,14.2903410523,1048576,2909,true,1,0.0928672027588 +fp16,4,64,16,128,4096,baseline_transform,4,11,100,25.9001135826,2097152,80.9707645996,1048576,2909,true,1,0.0259001135826 +fp16,4,64,16,128,4096,control_optimized_transform,4,12,100,27.7647995949,2097152,75.532762008,1048576,2909,true,1,0.0277647995949 +fp16,4,64,16,128,4096,optimized_transform,4,13,100,27.7860569954,2097152,75.4749765448,1048576,2909,true,1,0.0277860569954 +fp16,4,64,16,128,4096,control_shuffle32_transform,4,14,100,15.8253717422,2097152,132.518340432,1048576,2909,true,1,0.0158253717422 +fp16,16,64,16,128,16384,baseline_transform,0,0,100,78.8592004776,8388608,106.374499731,4194304,2909,true,1,0.0788592004776 +fp16,16,64,16,128,16384,control_optimized_transform,0,1,100,87.1334838867,8388608,96.2730700738,4194304,2909,true,1,0.0871334838867 +fp16,16,64,16,128,16384,optimized_transform,0,2,100,87.1570301056,8388608,96.2470610786,4194304,2909,true,1,0.0871570301056 +fp16,16,64,16,128,16384,control_shuffle32_transform,0,3,100,26.6921138763,8388608,314.272898687,4194304,2909,true,1,0.0266921138763 +fp16,16,64,16,128,16384,shuffle32_transform,0,4,100,26.6847991943,8388608,314.359045347,4194304,2909,true,1,0.0266847991943 +fp16,16,64,16,128,16384,baseline_split,0,5,100,1884.21234131,13697024,7.26936327701,4194304,2909,true,1,1.88421234131 +fp16,16,64,16,128,16384,control_optimized_split,0,6,100,1453.8420105,13697024,9.42126028901,4194304,2909,true,1,1.4538420105 +fp16,16,64,16,128,16384,optimized_split,0,7,100,601.851196289,13697024,22.7581569738,4194304,2909,true,1,0.601851196289 +fp16,16,64,16,128,16384,control_shuffle32_split,0,8,100,1601.88140869,13697024,8.55058553379,4194304,2909,true,1,1.60188140869 +fp16,16,64,16,128,16384,shuffle32_split,0,9,100,288.943996429,13697024,47.4037327969,4194304,2909,true,1,0.288943996429 +fp16,16,64,16,128,16384,baseline_fused,0,10,100,1865.11291504,5308416,2.84616333799,4194304,2909,true,1,1.86511291504 +fp16,16,64,16,128,16384,control_optimized_fused,0,11,100,1421.56295776,5308416,3.73421097603,4194304,2909,true,1,1.42156295776 +fp16,16,64,16,128,16384,optimized_fused,0,12,100,566.201591492,5308416,9.37548759977,4194304,2909,true,1,0.566201591492 +fp16,16,64,16,128,16384,control_shuffle32_fused,0,13,100,1574.77828979,5308416,3.37089737292,4194304,2909,true,1,1.57477828979 +fp16,16,64,16,128,16384,shuffle32_fused,0,14,100,265.24433136,5308416,20.0133061196,4194304,2909,true,1,0.26524433136 +fp16,16,64,16,128,16384,control_optimized_transform,1,0,100,87.1341705322,8388608,96.2723114108,4194304,2909,true,1,0.0871341705322 +fp16,16,64,16,128,16384,optimized_transform,1,1,100,87.1826267242,8388608,96.2188031628,4194304,2909,true,1,0.0871826267242 +fp16,16,64,16,128,16384,control_shuffle32_transform,1,2,100,26.7129135132,8388608,314.028194486,4194304,2909,true,1,0.0267129135132 +fp16,16,64,16,128,16384,shuffle32_transform,1,3,100,26.7206859589,8388608,313.936850757,4194304,2909,true,1,0.0267206859589 +fp16,16,64,16,128,16384,baseline_split,1,4,100,1884.31259155,13697024,7.26897652831,4194304,2909,true,1,1.88431259155 +fp16,16,64,16,128,16384,control_optimized_split,1,5,100,1453.82919312,13697024,9.42134334959,4194304,2909,true,1,1.45382919312 +fp16,16,64,16,128,16384,optimized_split,1,6,100,601.900558472,13697024,22.7562905653,4194304,2909,true,1,0.601900558472 +fp16,16,64,16,128,16384,control_shuffle32_split,1,7,100,1601.6633606,13697024,8.55174959793,4194304,2909,true,1,1.6016633606 +fp16,16,64,16,128,16384,shuffle32_split,1,8,100,288.751087189,13697024,47.4354023507,4194304,2909,true,1,0.288751087189 +fp16,16,64,16,128,16384,baseline_fused,1,9,100,1865.16906738,5308416,2.84607765206,4194304,2909,true,1,1.86516906738 +fp16,16,64,16,128,16384,control_optimized_fused,1,10,100,1421.66885376,5308416,3.73393282547,4194304,2909,true,1,1.42166885376 +fp16,16,64,16,128,16384,optimized_fused,1,11,100,566.214408875,5308416,9.37527536707,4194304,2909,true,1,0.566214408875 +fp16,16,64,16,128,16384,control_shuffle32_fused,1,12,100,1574.80895996,5308416,3.37083172306,4194304,2909,true,1,1.57480895996 +fp16,16,64,16,128,16384,shuffle32_fused,1,13,100,265.247325897,5308416,20.0130801773,4194304,2909,true,1,0.265247325897 +fp16,16,64,16,128,16384,baseline_transform,1,14,100,78.8427400589,8388608,106.396708102,4194304,2909,true,1,0.0788427400589 +fp16,16,64,16,128,16384,optimized_transform,2,0,100,87.1632003784,8388608,96.2402477603,4194304,2909,true,1,0.0871632003784 +fp16,16,64,16,128,16384,control_shuffle32_transform,2,1,100,26.7666268349,8388608,313.398025525,4194304,2909,true,1,0.0267666268349 +fp16,16,64,16,128,16384,shuffle32_transform,2,2,100,26.7307448387,8388608,313.818715139,4194304,2909,true,1,0.0267307448387 +fp16,16,64,16,128,16384,baseline_split,2,3,100,1884.32800293,13697024,7.26891707744,4194304,2909,true,1,1.88432800293 +fp16,16,64,16,128,16384,control_optimized_split,2,4,100,1453.55865479,13697024,9.42309686294,4194304,2909,true,1,1.45355865479 +fp16,16,64,16,128,16384,optimized_split,2,5,100,601.814880371,13697024,22.759530292,4194304,2909,true,1,0.601814880371 +fp16,16,64,16,128,16384,control_shuffle32_split,2,6,100,1602.13363647,13697024,8.54923939437,4194304,2909,true,1,1.60213363647 +fp16,16,64,16,128,16384,shuffle32_split,2,7,100,288.663082123,13697024,47.4498640397,4194304,2909,true,1,0.288663082123 +fp16,16,64,16,128,16384,baseline_fused,2,8,100,1865.14846802,5308416,2.84610908516,4194304,2909,true,1,1.86514846802 +fp16,16,64,16,128,16384,control_optimized_fused,2,9,100,1421.34567261,5308416,3.73478183549,4194304,2909,true,1,1.42134567261 +fp16,16,64,16,128,16384,optimized_fused,2,10,100,566.136245728,5308416,9.37656975695,4194304,2909,true,1,0.566136245728 +fp16,16,64,16,128,16384,control_shuffle32_fused,2,11,100,1574.30969238,5308416,3.37190072937,4194304,2909,true,1,1.57430969238 +fp16,16,64,16,128,16384,shuffle32_fused,2,12,100,265.213718414,5308416,20.0156162047,4194304,2909,true,1,0.265213718414 +fp16,16,64,16,128,16384,baseline_transform,2,13,100,78.8653755188,8388608,106.366170767,4194304,2909,true,1,0.0788653755188 +fp16,16,64,16,128,16384,control_optimized_transform,2,14,100,87.0676517487,8388608,96.345862459,4194304,2909,true,1,0.0870676517487 +fp16,16,64,16,128,16384,control_shuffle32_transform,3,0,100,26.7225122452,8388608,313.915395493,4194304,2909,true,1,0.0267225122452 +fp16,16,64,16,128,16384,shuffle32_transform,3,1,100,26.7092585564,8388608,314.071166831,4194304,2909,true,1,0.0267092585564 +fp16,16,64,16,128,16384,baseline_split,3,2,100,1884.4670105,13697024,7.26838088632,4194304,2909,true,1,1.8844670105 +fp16,16,64,16,128,16384,control_optimized_split,3,3,100,1453.86383057,13697024,9.42111889163,4194304,2909,true,1,1.45386383057 +fp16,16,64,16,128,16384,optimized_split,3,4,100,601.821250916,13697024,22.7592893723,4194304,2909,true,1,0.601821250916 +fp16,16,64,16,128,16384,control_shuffle32_split,3,5,100,1602.46154785,13697024,8.54748996527,4194304,2909,true,1,1.60246154785 +fp16,16,64,16,128,16384,shuffle32_split,3,6,100,288.799781799,13697024,47.4274042545,4194304,2909,true,1,0.288799781799 +fp16,16,64,16,128,16384,baseline_fused,3,7,100,1865.22537231,5308416,2.84599173847,4194304,2909,true,1,1.86522537231 +fp16,16,64,16,128,16384,control_optimized_fused,3,8,100,1421.56234741,5308416,3.73421257932,4194304,2909,true,1,1.42156234741 +fp16,16,64,16,128,16384,optimized_fused,3,9,100,566.181945801,5308416,9.37581291557,4194304,2909,true,1,0.566181945801 +fp16,16,64,16,128,16384,control_shuffle32_fused,3,10,100,1574.93530273,5308416,3.37056131181,4194304,2909,true,1,1.57493530273 +fp16,16,64,16,128,16384,shuffle32_fused,3,11,100,265.224914551,5308416,20.0147712706,4194304,2909,true,1,0.265224914551 +fp16,16,64,16,128,16384,baseline_transform,3,12,100,78.8443422318,8388608,106.394546045,4194304,2909,true,1,0.0788443422318 +fp16,16,64,16,128,16384,control_optimized_transform,3,13,100,87.1179485321,8388608,96.2902380203,4194304,2909,true,1,0.0871179485321 +fp16,16,64,16,128,16384,optimized_transform,3,14,100,87.1648025513,8388608,96.238478772,4194304,2909,true,1,0.0871648025513 +fp16,16,64,16,128,16384,shuffle32_transform,4,0,100,26.7599987984,8388608,313.475649353,4194304,2909,true,1,0.0267599987984 +fp16,16,64,16,128,16384,baseline_split,4,1,100,1884.14825439,13697024,7.26961053519,4194304,2909,true,1,1.88414825439 +fp16,16,64,16,128,16384,control_optimized_split,4,2,100,1453.55529785,13697024,9.42311862524,4194304,2909,true,1,1.45355529785 +fp16,16,64,16,128,16384,optimized_split,4,3,100,601.788368225,13697024,22.7605329767,4194304,2909,true,1,0.601788368225 +fp16,16,64,16,128,16384,control_shuffle32_split,4,4,100,1600.97335815,13697024,8.5554353108,4194304,2909,true,1,1.60097335815 +fp16,16,64,16,128,16384,shuffle32_split,4,5,100,288.683185577,13697024,47.4465597039,4194304,2909,true,1,0.288683185577 +fp16,16,64,16,128,16384,baseline_fused,4,6,100,1865.24780273,5308416,2.84595751418,4194304,2909,true,1,1.86524780273 +fp16,16,64,16,128,16384,control_optimized_fused,4,7,100,1421.43722534,5308416,3.73454128354,4194304,2909,true,1,1.42143722534 +fp16,16,64,16,128,16384,optimized_fused,4,8,100,566.126174927,5308416,9.37673655645,4194304,2909,true,1,0.566126174927 +fp16,16,64,16,128,16384,control_shuffle32_fused,4,9,100,1574.01916504,5308416,3.37252310385,4194304,2909,true,1,1.57401916504 +fp16,16,64,16,128,16384,shuffle32_fused,4,10,100,265.200004578,5308416,20.0166512382,4194304,2909,true,1,0.265200004578 +fp16,16,64,16,128,16384,baseline_transform,4,11,100,78.8596534729,8388608,106.373888682,4194304,2909,true,1,0.0788596534729 +fp16,16,64,16,128,16384,control_optimized_transform,4,12,100,87.0651435852,8388608,96.3486379804,4194304,2909,true,1,0.0870651435852 +fp16,16,64,16,128,16384,optimized_transform,4,13,100,87.1547412872,8388608,96.2495886753,4194304,2909,true,1,0.0871547412872 +fp16,16,64,16,128,16384,control_shuffle32_transform,4,14,100,26.6671991348,8388608,314.566518875,4194304,2909,true,1,0.0266671991348 +fp16,1,1,1,256,1,baseline_transform,0,0,100,9.78468596935,1024,0.104653333097,512,2909,true,1,0.00978468596935 +fp16,1,1,1,256,1,control_optimized_transform,0,1,100,10.6020569801,1024,0.0965850308029,512,2909,true,1,0.0106020569801 +fp16,1,1,1,256,1,optimized_transform,0,2,100,10.6368005276,1024,0.0962695499785,512,2909,true,1,0.0106368005276 +fp16,1,1,1,256,1,control_shuffle32_transform,0,3,100,11.5890288353,1024,0.0883594315411,512,2909,true,1,0.0115890288353 +fp16,1,1,1,256,1,shuffle32_transform,0,4,100,11.6105139256,1024,0.0881959236745,512,2909,true,1,0.0116105139256 +fp16,1,1,1,256,1,baseline_split,0,5,100,100.570287704,1668,0.0165854154152,512,2909,true,1,0.100570287704 +fp16,1,1,1,256,1,control_optimized_split,0,6,100,66.9398880005,1668,0.0249178785598,512,2909,true,1,0.0669398880005 +fp16,1,1,1,256,1,optimized_split,0,7,100,35.4431986809,1668,0.047061215186,512,2909,true,1,0.0354431986809 +fp16,1,1,1,256,1,control_shuffle32_split,0,8,100,303.177585602,1668,0.00550172598244,512,2909,true,1,0.303177585602 +fp16,1,1,1,256,1,shuffle32_split,0,9,100,67.7542877197,1668,0.0246183681673,512,2909,true,1,0.0677542877197 +fp16,1,1,1,256,1,baseline_fused,0,10,100,96.0912036896,644,0.00670196620786,512,2909,true,1,0.0960912036896 +fp16,1,1,1,256,1,control_optimized_fused,0,11,100,61.2857151031,644,0.010508158368,512,2909,true,1,0.0612857151031 +fp16,1,1,1,256,1,optimized_fused,0,12,100,29.7549724579,644,0.0216434413075,512,2909,true,1,0.0297549724579 +fp16,1,1,1,256,1,control_shuffle32_fused,0,13,100,299.391326904,644,0.00215103091549,512,2909,true,1,0.299391326904 +fp16,1,1,1,256,1,shuffle32_fused,0,14,100,61.6774845123,644,0.0104414115636,512,2909,true,1,0.0616774845123 +fp16,1,1,1,256,1,control_optimized_transform,1,0,100,10.5476570129,1024,0.0970831720015,512,2909,true,1,0.0105476570129 +fp16,1,1,1,256,1,optimized_transform,1,1,100,10.5673146248,1024,0.0969025751914,512,2909,true,1,0.0105673146248 +fp16,1,1,1,256,1,control_shuffle32_transform,1,2,100,11.4932572842,1024,0.0890957171394,512,2909,true,1,0.0114932572842 +fp16,1,1,1,256,1,shuffle32_transform,1,3,100,11.6843426228,1024,0.0876386488364,512,2909,true,1,0.0116843426228 +fp16,1,1,1,256,1,baseline_split,1,4,100,100.607309341,1668,0.0165793122877,512,2909,true,1,0.100607309341 +fp16,1,1,1,256,1,control_optimized_split,1,5,100,66.9001150131,1668,0.0249326925622,512,2909,true,1,0.0669001150131 +fp16,1,1,1,256,1,optimized_split,1,6,100,35.690972805,1668,0.0467345064847,512,2909,true,1,0.035690972805 +fp16,1,1,1,256,1,control_shuffle32_split,1,7,100,302.667655945,1668,0.00551099520295,512,2909,true,1,0.302667655945 +fp16,1,1,1,256,1,shuffle32_split,1,8,100,67.7796602249,1668,0.0246091525756,512,2909,true,1,0.0677796602249 +fp16,1,1,1,256,1,baseline_fused,1,9,100,96.1572551727,644,0.00669736255307,512,2909,true,1,0.0961572551727 +fp16,1,1,1,256,1,control_optimized_fused,1,10,100,61.298289299,644,0.0105060028161,512,2909,true,1,0.061298289299 +fp16,1,1,1,256,1,optimized_fused,1,11,100,29.7597718239,644,0.0216399508642,512,2909,true,1,0.0297597718239 +fp16,1,1,1,256,1,control_shuffle32_fused,1,12,100,299.444103241,644,0.00215065180122,512,2909,true,1,0.299444103241 +fp16,1,1,1,256,1,shuffle32_fused,1,13,100,61.6758871078,644,0.010441681996,512,2909,true,1,0.0616758871078 +fp16,1,1,1,256,1,baseline_transform,1,14,100,9.93462860584,1024,0.103073807852,512,2909,true,1,0.00993462860584 +fp16,1,1,1,256,1,optimized_transform,2,0,100,10.626513958,1024,0.0963627398458,512,2909,true,1,0.010626513958 +fp16,1,1,1,256,1,control_shuffle32_transform,2,1,100,11.454628706,1024,0.089396175667,512,2909,true,1,0.011454628706 +fp16,1,1,1,256,1,shuffle32_transform,2,2,100,11.613714695,1024,0.0881716166528,512,2909,true,1,0.011613714695 +fp16,1,1,1,256,1,baseline_split,2,3,100,100.593824387,1668,0.0165815348027,512,2909,true,1,0.100593824387 +fp16,1,1,1,256,1,control_optimized_split,2,4,100,66.9563436508,1668,0.024911754571,512,2909,true,1,0.0669563436508 +fp16,1,1,1,256,1,optimized_split,2,5,100,35.5423998833,1668,0.0469298642038,512,2909,true,1,0.0355423998833 +fp16,1,1,1,256,1,control_shuffle32_split,2,6,100,302.625598907,1668,0.00551176108704,512,2909,true,1,0.302625598907 +fp16,1,1,1,256,1,shuffle32_split,2,7,100,68.3968019485,1668,0.0243871051348,512,2909,true,1,0.0683968019485 +fp16,1,1,1,256,1,baseline_fused,2,8,100,96.1385154724,644,0.00669866802951,512,2909,true,1,0.0961385154724 +fp16,1,1,1,256,1,control_optimized_fused,2,9,100,61.2827444077,644,0.0105086677535,512,2909,true,1,0.0612827444077 +fp16,1,1,1,256,1,optimized_fused,2,10,100,29.7609138489,644,0.0216391204675,512,2909,true,1,0.0297609138489 +fp16,1,1,1,256,1,control_shuffle32_fused,2,11,100,299.445953369,644,0.00215063851341,512,2909,true,1,0.299445953369 +fp16,1,1,1,256,1,shuffle32_fused,2,12,100,61.6724538803,644,0.0104422632712,512,2909,true,1,0.0616724538803 +fp16,1,1,1,256,1,baseline_transform,2,13,100,9.76068556309,1024,0.10491066364,512,2909,true,1,0.00976068556309 +fp16,1,1,1,256,1,control_optimized_transform,2,14,100,10.5584001541,1024,0.0969843901589,512,2909,true,1,0.0105584001541 +fp16,1,1,1,256,1,control_shuffle32_transform,3,0,100,11.566400528,1024,0.0885322964154,512,2909,true,1,0.011566400528 +fp16,1,1,1,256,1,shuffle32_transform,3,1,100,11.5936005116,1024,0.0883245889816,512,2909,true,1,0.0115936005116 +fp16,1,1,1,256,1,baseline_split,3,2,100,100.600004196,1668,0.016580516207,512,2909,true,1,0.100600004196 +fp16,1,1,1,256,1,control_optimized_split,3,3,100,66.982626915,1668,0.0249019794658,512,2909,true,1,0.066982626915 +fp16,1,1,1,256,1,optimized_split,3,4,100,35.4745149612,1668,0.0470196703696,512,2909,true,1,0.0354745149612 +fp16,1,1,1,256,1,control_shuffle32_split,3,5,100,303.309020996,1668,0.00549934187424,512,2909,true,1,0.303309020996 +fp16,1,1,1,256,1,shuffle32_split,3,6,100,67.7885723114,1668,0.0246059172383,512,2909,true,1,0.0677885723114 +fp16,1,1,1,256,1,baseline_fused,3,7,100,96.1460590363,644,0.00669814245592,512,2909,true,1,0.0961460590363 +fp16,1,1,1,256,1,control_optimized_fused,3,8,100,61.2971401215,644,0.010506199779,512,2909,true,1,0.0612971401215 +fp16,1,1,1,256,1,optimized_fused,3,9,100,29.7746300697,644,0.0216291520161,512,2909,true,1,0.0297746300697 +fp16,1,1,1,256,1,control_shuffle32_fused,3,10,100,299.447784424,644,0.00215062536275,512,2909,true,1,0.299447784424 +fp16,1,1,1,256,1,shuffle32_fused,3,11,100,61.6747426987,644,0.0104418757472,512,2909,true,1,0.0616747426987 +fp16,1,1,1,256,1,baseline_transform,3,12,100,9.90651428699,1024,0.103366327483,512,2909,true,1,0.00990651428699 +fp16,1,1,1,256,1,control_optimized_transform,3,13,100,10.5872011185,1024,0.0967205580154,512,2909,true,1,0.0105872011185 +fp16,1,1,1,256,1,optimized_transform,3,14,100,10.5762279034,1024,0.0968209090572,512,2909,true,1,0.0105762279034 +fp16,1,1,1,256,1,shuffle32_transform,4,0,100,11.1074280739,1024,0.0921905587134,512,2909,true,1,0.0111074280739 +fp16,1,1,1,256,1,baseline_split,4,1,100,100.596570969,1668,0.016581082078,512,2909,true,1,0.100596570969 +fp16,1,1,1,256,1,control_optimized_split,4,2,100,66.9101715088,1668,0.024928945217,512,2909,true,1,0.0669101715088 +fp16,1,1,1,256,1,optimized_split,4,3,100,35.4717707634,1668,0.0470233079461,512,2909,true,1,0.0354717707634 +fp16,1,1,1,256,1,control_shuffle32_split,4,4,100,302.605476379,1668,0.00551212760574,512,2909,true,1,0.302605476379 +fp16,1,1,1,256,1,shuffle32_split,4,5,100,67.7771425247,1668,0.0246100667255,512,2909,true,1,0.0677771425247 +fp16,1,1,1,256,1,baseline_fused,4,6,100,96.1437702179,644,0.00669830191327,512,2909,true,1,0.0961437702179 +fp16,1,1,1,256,1,control_optimized_fused,4,7,100,61.2459421158,644,0.0105149823442,512,2909,true,1,0.0612459421158 +fp16,1,1,1,256,1,optimized_fused,4,8,100,29.7247982025,644,0.021665412011,512,2909,true,1,0.0297247982025 +fp16,1,1,1,256,1,control_shuffle32_fused,4,9,100,299.402503967,644,0.00215095061486,512,2909,true,1,0.299402503967 +fp16,1,1,1,256,1,shuffle32_fused,4,10,100,61.6283416748,644,0.0104497376126,512,2909,true,1,0.0616283416748 +fp16,1,1,1,256,1,baseline_transform,4,11,100,9.76662874222,1024,0.104846823508,512,2909,true,1,0.00976662874222 +fp16,1,1,1,256,1,control_optimized_transform,4,12,100,10.7266283035,1024,0.0954633619274,512,2909,true,1,0.0107266283035 +fp16,1,1,1,256,1,optimized_transform,4,13,100,10.6651425362,1024,0.0960137191348,512,2909,true,1,0.0106651425362 +fp16,1,1,1,256,1,control_shuffle32_transform,4,14,100,11.5183997154,1024,0.0889012384792,512,2909,true,1,0.0115183997154 +fp16,1,17,1,256,17,baseline_transform,0,0,100,10.1003420353,17408,1.72350599011,8704,2909,true,1,0.0101003420353 +fp16,1,17,1,256,17,control_optimized_transform,0,1,100,10.6457138062,17408,1.63521209728,8704,2909,true,1,0.0106457138062 +fp16,1,17,1,256,17,optimized_transform,0,2,100,10.780800581,17408,1.61472238256,8704,2909,true,1,0.010780800581 +fp16,1,17,1,256,17,control_shuffle32_transform,0,3,100,11.5259432793,17408,1.51033191629,8704,2909,true,1,0.0115259432793 +fp16,1,17,1,256,17,shuffle32_transform,0,4,100,10.8575999737,17408,1.60330091753,8704,2909,true,1,0.0108575999737 +fp16,1,17,1,256,17,baseline_split,0,5,100,101.166391373,28356,0.280290713302,8704,2909,true,1,0.101166391373 +fp16,1,17,1,256,17,control_optimized_split,0,6,100,67.3056030273,28356,0.421302220388,8704,2909,true,1,0.0673056030273 +fp16,1,17,1,256,17,optimized_split,0,7,100,36.271314621,28356,0.781774807346,8704,2909,true,1,0.036271314621 +fp16,1,17,1,256,17,control_shuffle32_split,0,8,100,303.201141357,28356,0.0935220753888,8704,2909,true,1,0.303201141357 +fp16,1,17,1,256,17,shuffle32_split,0,9,100,68.0070829391,28356,0.416956569441,8704,2909,true,1,0.0680070829391 +fp16,1,17,1,256,17,baseline_fused,0,10,100,96.5999984741,10948,0.113333335124,8704,2909,true,1,0.0965999984741 +fp16,1,17,1,256,17,control_optimized_fused,0,11,100,61.6877698898,10948,0.177474400834,8704,2909,true,1,0.0616877698898 +fp16,1,17,1,256,17,optimized_fused,0,12,100,30.5163407326,10948,0.358758610541,8704,2909,true,1,0.0305163407326 +fp16,1,17,1,256,17,control_shuffle32_fused,0,13,100,299.925937653,10948,0.0365023448311,8704,2909,true,1,0.299925937653 +fp16,1,17,1,256,17,shuffle32_fused,0,14,100,61.6237688065,10948,0.177658721822,8704,2909,true,1,0.0616237688065 +fp16,1,17,1,256,17,control_optimized_transform,1,0,100,10.7888007164,17408,1.61352503004,8704,2909,true,1,0.0107888007164 +fp16,1,17,1,256,17,optimized_transform,1,1,100,10.7410287857,17408,1.62070136365,8704,2909,true,1,0.0107410287857 +fp16,1,17,1,256,17,control_shuffle32_transform,1,2,100,11.7490291595,17408,1.48165433617,8704,2909,true,1,0.0117490291595 +fp16,1,17,1,256,17,shuffle32_transform,1,3,100,11.8358850479,17408,1.4707814354,8704,2909,true,1,0.0118358850479 +fp16,1,17,1,256,17,baseline_split,1,4,100,101.147661209,28356,0.280342616537,8704,2909,true,1,0.101147661209 +fp16,1,17,1,256,17,control_optimized_split,1,5,100,67.4153184891,28356,0.42061656958,8704,2909,true,1,0.0674153184891 +fp16,1,17,1,256,17,optimized_split,1,6,100,36.2740564346,28356,0.781715716055,8704,2909,true,1,0.0362740564346 +fp16,1,17,1,256,17,control_shuffle32_split,1,7,100,303.8463974,28356,0.0933234694986,8704,2909,true,1,0.3038463974 +fp16,1,17,1,256,17,shuffle32_split,1,8,100,68.1172609329,28356,0.416282152448,8704,2909,true,1,0.0681172609329 +fp16,1,17,1,256,17,baseline_fused,1,9,100,96.6059398651,10948,0.113326364976,8704,2909,true,1,0.0966059398651 +fp16,1,17,1,256,17,control_optimized_fused,1,10,100,61.6864013672,10948,0.177478338132,8704,2909,true,1,0.0616864013672 +fp16,1,17,1,256,17,optimized_fused,1,11,100,30.4923439026,10948,0.359040945982,8704,2909,true,1,0.0304923439026 +fp16,1,17,1,256,17,control_shuffle32_fused,1,12,100,299.882965088,10948,0.0365075755363,8704,2909,true,1,0.299882965088 +fp16,1,17,1,256,17,shuffle32_fused,1,13,100,61.6025114059,10948,0.177720027157,8704,2909,true,1,0.0616025114059 +fp16,1,17,1,256,17,baseline_transform,1,14,100,10.031542778,17408,1.73532629878,8704,2909,true,1,0.010031542778 +fp16,1,17,1,256,17,optimized_transform,2,0,100,10.820800066,17408,1.60875350194,8704,2909,true,1,0.010820800066 +fp16,1,17,1,256,17,control_shuffle32_transform,2,1,100,11.6891419888,17408,1.48924531987,8704,2909,true,1,0.0116891419888 +fp16,1,17,1,256,17,shuffle32_transform,2,2,100,11.7485713959,17408,1.48171206638,8704,2909,true,1,0.0117485713959 +fp16,1,17,1,256,17,baseline_split,2,3,100,101.207771301,28356,0.28017611331,8704,2909,true,1,0.101207771301 +fp16,1,17,1,256,17,control_optimized_split,2,4,100,67.2598838806,28356,0.421588595816,8704,2909,true,1,0.0672598838806 +fp16,1,17,1,256,17,optimized_split,2,5,100,36.2614870071,28356,0.781986684507,8704,2909,true,1,0.0362614870071 +fp16,1,17,1,256,17,control_shuffle32_split,2,6,100,303.233833313,28356,0.0935119926764,8704,2909,true,1,0.303233833313 +fp16,1,17,1,256,17,shuffle32_split,2,7,100,68.086400032,28356,0.416470836858,8704,2909,true,1,0.068086400032 +fp16,1,17,1,256,17,baseline_fused,2,8,100,96.6063976288,10948,0.113325827986,8704,2909,true,1,0.0966063976288 +fp16,1,17,1,256,17,control_optimized_fused,2,9,100,61.6937112808,10948,0.177457309225,8704,2909,true,1,0.0616937112808 +fp16,1,17,1,256,17,optimized_fused,2,10,100,30.5190873146,10948,0.358726323862,8704,2909,true,1,0.0305190873146 +fp16,1,17,1,256,17,control_shuffle32_fused,2,11,100,299.921836853,10948,0.0365028439238,8704,2909,true,1,0.299921836853 +fp16,1,17,1,256,17,shuffle32_fused,2,12,100,61.6020584106,10948,0.177721334034,8704,2909,true,1,0.0616020584106 +fp16,1,17,1,256,17,baseline_transform,2,13,100,10.1410281658,17408,1.71659122876,8704,2909,true,1,0.0101410281658 +fp16,1,17,1,256,17,control_optimized_transform,2,14,100,10.6461715698,17408,1.63514178649,8704,2909,true,1,0.0106461715698 +fp16,1,17,1,256,17,control_shuffle32_transform,3,0,100,11.6804563999,17408,1.49035272287,8704,2909,true,1,0.0116804563999 +fp16,1,17,1,256,17,shuffle32_transform,3,1,100,11.8891429901,17408,1.46419300487,8704,2909,true,1,0.0118891429901 +fp16,1,17,1,256,17,baseline_split,3,2,100,101.200914383,28356,0.280195096782,8704,2909,true,1,0.101200914383 +fp16,1,17,1,256,17,control_optimized_split,3,3,100,67.2765731812,28356,0.421484012327,8704,2909,true,1,0.0672765731812 +fp16,1,17,1,256,17,optimized_split,3,4,100,36.2909722328,28356,0.781351345951,8704,2909,true,1,0.0362909722328 +fp16,1,17,1,256,17,control_shuffle32_split,3,5,100,303.802757263,28356,0.093336875068,8704,2909,true,1,0.303802757263 +fp16,1,17,1,256,17,shuffle32_split,3,6,100,68.0758857727,28356,0.416535160404,8704,2909,true,1,0.0680758857727 +fp16,1,17,1,256,17,baseline_fused,3,7,100,96.6022777557,10948,0.113330661081,8704,2909,true,1,0.0966022777557 +fp16,1,17,1,256,17,control_optimized_fused,3,8,100,61.6841173172,10948,0.17748490983,8704,2909,true,1,0.0616841173172 +fp16,1,17,1,256,17,optimized_fused,3,9,100,30.4985141754,10948,0.358968307014,8704,2909,true,1,0.0304985141754 +fp16,1,17,1,256,17,control_shuffle32_fused,3,10,100,299.930744171,10948,0.0365017598654,8704,2909,true,1,0.299930744171 +fp16,1,17,1,256,17,shuffle32_fused,3,11,100,61.6068601608,10948,0.177707482112,8704,2909,true,1,0.0616068601608 +fp16,1,17,1,256,17,baseline_transform,3,12,100,10.1620578766,17408,1.71303885605,8704,2909,true,1,0.0101620578766 +fp16,1,17,1,256,17,control_optimized_transform,3,13,100,10.6692564487,17408,1.63160385952,8704,2909,true,1,0.0106692564487 +fp16,1,17,1,256,17,optimized_transform,3,14,100,10.7428574562,17408,1.62042548465,8704,2909,true,1,0.0107428574562 +fp16,1,17,1,256,17,shuffle32_transform,4,0,100,11.8445706367,17408,1.4697029157,8704,2909,true,1,0.0118445706367 +fp16,1,17,1,256,17,baseline_split,4,1,100,101.216230392,28356,0.280152697745,8704,2909,true,1,0.101216230392 +fp16,1,17,1,256,17,control_optimized_split,4,2,100,67.288684845,28356,0.42140814708,8704,2909,true,1,0.067288684845 +fp16,1,17,1,256,17,optimized_split,4,3,100,36.2315416336,28356,0.782632996596,8704,2909,true,1,0.0362315416336 +fp16,1,17,1,256,17,control_shuffle32_split,4,4,100,303.231525421,28356,0.0935127043952,8704,2909,true,1,0.303231525421 +fp16,1,17,1,256,17,shuffle32_split,4,5,100,68.0198860168,28356,0.416878087578,8704,2909,true,1,0.0680198860168 +fp16,1,17,1,256,17,baseline_fused,4,6,100,96.5922355652,10948,0.113342443478,8704,2909,true,1,0.0965922355652 +fp16,1,17,1,256,17,control_optimized_fused,4,7,100,61.6838884354,10948,0.177485568399,8704,2909,true,1,0.0616838884354 +fp16,1,17,1,256,17,optimized_fused,4,8,100,30.4603433609,10948,0.359418141493,8704,2909,true,1,0.0304603433609 +fp16,1,17,1,256,17,control_shuffle32_fused,4,9,100,299.869728088,10948,0.036509187072,8704,2909,true,1,0.299869728088 +fp16,1,17,1,256,17,shuffle32_fused,4,10,100,61.5599966049,10948,0.177842764844,8704,2909,true,1,0.0615599966049 +fp16,1,17,1,256,17,baseline_transform,4,11,100,9.9401140213,17408,1.75128775814,8704,2909,true,1,0.0099401140213 +fp16,1,17,1,256,17,control_optimized_transform,4,12,100,10.670170784,17408,1.63146404612,8704,2909,true,1,0.010670170784 +fp16,1,17,1,256,17,optimized_transform,4,13,100,10.7129144669,17408,1.62495463339,8704,2909,true,1,0.0107129144669 +fp16,1,17,1,256,17,control_shuffle32_transform,4,14,100,11.6370284557,17408,1.49591453404,8704,2909,true,1,0.0116370284557 +fp16,1,257,1,256,257,baseline_transform,0,0,100,10.5613720417,263168,24.9179745738,131584,2909,true,1,0.0105613720417 +fp16,1,257,1,256,257,control_optimized_transform,0,1,100,10.9625136852,263168,24.0061729961,131584,2909,true,1,0.0109625136852 +fp16,1,257,1,256,257,optimized_transform,0,2,100,10.917943716,263168,24.1041726212,131584,2909,true,1,0.010917943716 +fp16,1,257,1,256,257,control_shuffle32_transform,0,3,100,13.7695991993,263168,19.1122483807,131584,2909,true,1,0.0137695991993 +fp16,1,257,1,256,257,shuffle32_transform,0,4,100,13.8639998436,263168,18.9821121587,131584,2909,true,1,0.0138639998436 +fp16,1,257,1,256,257,baseline_split,0,5,100,103.906974792,428676,4.1255748313,131584,2909,true,1,0.103906974792 +fp16,1,257,1,256,257,control_optimized_split,0,6,100,69.7524547577,428676,6.14567618429,131584,2909,true,1,0.0697524547577 +fp16,1,257,1,256,257,optimized_split,0,7,100,37.0480012894,428676,11.5708266325,131584,2909,true,1,0.0370480012894 +fp16,1,257,1,256,257,control_shuffle32_split,0,8,100,326.249389648,428676,1.31395188344,131584,2909,true,1,0.326249389648 +fp16,1,257,1,256,257,shuffle32_split,0,9,100,76.6569137573,428676,5.59213747317,131584,2909,true,1,0.0766569137573 +fp16,1,257,1,256,257,baseline_fused,0,10,100,98.9545154572,165508,1.67256642343,131584,2909,true,1,0.0989545154572 +fp16,1,257,1,256,257,control_optimized_fused,0,11,100,64.0011453629,165508,2.58601621989,131584,2909,true,1,0.0640011453629 +fp16,1,257,1,256,257,optimized_fused,0,12,100,30.8699440956,165508,5.36146095657,131584,2909,true,1,0.0308699440956 +fp16,1,257,1,256,257,control_shuffle32_fused,0,13,100,320.788574219,165508,0.515941069295,131584,2909,true,1,0.320788574219 +fp16,1,257,1,256,257,shuffle32_fused,0,14,100,68.0482292175,165508,2.43221611941,131584,2909,true,1,0.0680482292175 +fp16,1,257,1,256,257,control_optimized_transform,1,0,100,11.0557711124,263168,23.8036765888,131584,2909,true,1,0.0110557711124 +fp16,1,257,1,256,257,optimized_transform,1,1,100,11.0399997234,263168,23.8376817566,131584,2909,true,1,0.0110399997234 +fp16,1,257,1,256,257,control_shuffle32_transform,1,2,100,13.9220571518,263168,18.9029535744,131584,2909,true,1,0.0139220571518 +fp16,1,257,1,256,257,shuffle32_transform,1,3,100,13.9003431797,263168,18.9324822127,131584,2909,true,1,0.0139003431797 +fp16,1,257,1,256,257,baseline_split,1,4,100,103.94194603,428676,4.12418678286,131584,2909,true,1,0.10394194603 +fp16,1,257,1,256,257,control_optimized_split,1,5,100,69.6498298645,428676,6.15473147363,131584,2909,true,1,0.0696498298645 +fp16,1,257,1,256,257,optimized_split,1,6,100,37.5236558914,428676,11.4241533725,131584,2909,true,1,0.0375236558914 +fp16,1,257,1,256,257,control_shuffle32_split,1,7,100,326.259002686,428676,1.31391316859,131584,2909,true,1,0.326259002686 +fp16,1,257,1,256,257,shuffle32_split,1,8,100,76.6557741165,428676,5.59222061144,131584,2909,true,1,0.0766557741165 +fp16,1,257,1,256,257,baseline_fused,1,9,100,98.9654827118,165508,1.67238107131,131584,2909,true,1,0.0989654827118 +fp16,1,257,1,256,257,control_optimized_fused,1,10,100,63.946056366,165508,2.5882440514,131584,2909,true,1,0.063946056366 +fp16,1,257,1,256,257,optimized_fused,1,11,100,30.9115433693,165508,5.35424575935,131584,2909,true,1,0.0309115433693 +fp16,1,257,1,256,257,control_shuffle32_fused,1,12,100,320.838851929,165508,0.515860217692,131584,2909,true,1,0.320838851929 +fp16,1,257,1,256,257,shuffle32_fused,1,13,100,68.0404567719,165508,2.43249395804,131584,2909,true,1,0.0680404567719 +fp16,1,257,1,256,257,baseline_transform,1,14,100,10.494171381,263168,25.0775397547,131584,2909,true,1,0.010494171381 +fp16,1,257,1,256,257,optimized_transform,2,0,100,11.1606860161,263168,23.579912527,131584,2909,true,1,0.0111606860161 +fp16,1,257,1,256,257,control_shuffle32_transform,2,1,100,13.6925709248,263168,19.2197653345,131584,2909,true,1,0.0136925709248 +fp16,1,257,1,256,257,shuffle32_transform,2,2,100,13.9954292774,263168,18.8038533712,131584,2909,true,1,0.0139954292774 +fp16,1,257,1,256,257,baseline_split,2,3,100,103.938970566,428676,4.12430484607,131584,2909,true,1,0.103938970566 +fp16,1,257,1,256,257,control_optimized_split,2,4,100,69.708571434,428676,6.1495450442,131584,2909,true,1,0.069708571434 +fp16,1,257,1,256,257,optimized_split,2,5,100,36.9750857353,428676,11.593644517,131584,2909,true,1,0.0369750857353 +fp16,1,257,1,256,257,control_shuffle32_split,2,6,100,325.73802948,428676,1.31601459211,131584,2909,true,1,0.32573802948 +fp16,1,257,1,256,257,shuffle32_split,2,7,100,76.6461706161,428676,5.59292129736,131584,2909,true,1,0.0766461706161 +fp16,1,257,1,256,257,baseline_fused,2,8,100,98.9558792114,165508,1.67254337306,131584,2909,true,1,0.0989558792114 +fp16,1,257,1,256,257,control_optimized_fused,2,9,100,63.990855217,165508,2.58643206813,131584,2909,true,1,0.063990855217 +fp16,1,257,1,256,257,optimized_fused,2,10,100,30.8500576019,165508,5.36491704928,131584,2909,true,1,0.0308500576019 +fp16,1,257,1,256,257,control_shuffle32_fused,2,11,100,320.834083557,165508,0.515867884624,131584,2909,true,1,0.320834083557 +fp16,1,257,1,256,257,shuffle32_fused,2,12,100,67.984457016,165508,2.43449763762,131584,2909,true,1,0.067984457016 +fp16,1,257,1,256,257,baseline_transform,2,13,100,10.537828207,263168,24.9736468303,131584,2909,true,1,0.010537828207 +fp16,1,257,1,256,257,control_optimized_transform,2,14,100,11.0859417915,263168,23.7388942635,131584,2909,true,1,0.0110859417915 +fp16,1,257,1,256,257,control_shuffle32_transform,3,0,100,13.7245714664,263168,19.1749520663,131584,2909,true,1,0.0137245714664 +fp16,1,257,1,256,257,shuffle32_transform,3,1,100,13.8297140598,263168,19.0291714537,131584,2909,true,1,0.0138297140598 +fp16,1,257,1,256,257,baseline_split,3,2,100,103.859424591,428676,4.1274636528,131584,2909,true,1,0.103859424591 +fp16,1,257,1,256,257,control_optimized_split,3,3,100,69.7295999527,428676,6.1476905115,131584,2909,true,1,0.0697295999527 +fp16,1,257,1,256,257,optimized_split,3,4,100,37.0047998428,428676,11.5843350544,131584,2909,true,1,0.0370047998428 +fp16,1,257,1,256,257,control_shuffle32_split,3,5,100,326.327095032,428676,1.31363900371,131584,2909,true,1,0.326327095032 +fp16,1,257,1,256,257,shuffle32_split,3,6,100,76.687541008,428676,5.58990410131,131584,2909,true,1,0.076687541008 +fp16,1,257,1,256,257,baseline_fused,3,7,100,98.9485740662,165508,1.67266685308,131584,2909,true,1,0.0989485740662 +fp16,1,257,1,256,257,control_optimized_fused,3,8,100,63.9901733398,165508,2.58645962906,131584,2909,true,1,0.0639901733398 +fp16,1,257,1,256,257,optimized_fused,3,9,100,30.8761143684,165508,5.36038952392,131584,2909,true,1,0.0308761143684 +fp16,1,257,1,256,257,control_shuffle32_fused,3,10,100,320.822181702,165508,0.515887022282,131584,2909,true,1,0.320822181702 +fp16,1,257,1,256,257,shuffle32_fused,3,11,100,68.0480003357,165508,2.43222430025,131584,2909,true,1,0.0680480003357 +fp16,1,257,1,256,257,baseline_transform,3,12,100,10.534170866,263168,24.9823173886,131584,2909,true,1,0.010534170866 +fp16,1,257,1,256,257,control_optimized_transform,3,13,100,11.1282289028,263168,23.6486868035,131584,2909,true,1,0.0111282289028 +fp16,1,257,1,256,257,optimized_transform,3,14,100,11.0800004005,263168,23.7516236901,131584,2909,true,1,0.0110800004005 +fp16,1,257,1,256,257,shuffle32_transform,4,0,100,13.9794290066,263168,18.8253754768,131584,2909,true,1,0.0139794290066 +fp16,1,257,1,256,257,baseline_split,4,1,100,103.85966301,428676,4.12745417786,131584,2909,true,1,0.10385966301 +fp16,1,257,1,256,257,control_optimized_split,4,2,100,69.6388578415,428676,6.15570118878,131584,2909,true,1,0.0696388578415 +fp16,1,257,1,256,257,optimized_split,4,3,100,36.9307422638,428676,11.6075652349,131584,2909,true,1,0.0369307422638 +fp16,1,257,1,256,257,control_shuffle32_split,4,4,100,325.734176636,428676,1.31603015817,131584,2909,true,1,0.325734176636 +fp16,1,257,1,256,257,shuffle32_split,4,5,100,76.6123390198,428676,5.59539110129,131584,2909,true,1,0.0766123390198 +fp16,1,257,1,256,257,baseline_fused,4,6,100,98.9485740662,165508,1.67266685308,131584,2909,true,1,0.0989485740662 +fp16,1,257,1,256,257,control_optimized_fused,4,7,100,63.9915418625,165508,2.58640431505,131584,2909,true,1,0.0639915418625 +fp16,1,257,1,256,257,optimized_fused,4,8,100,30.8989715576,165508,5.35642423216,131584,2909,true,1,0.0308989715576 +fp16,1,257,1,256,257,control_shuffle32_fused,4,9,100,320.770950317,165508,0.515969416296,131584,2909,true,1,0.320770950317 +fp16,1,257,1,256,257,shuffle32_fused,4,10,100,68.0505180359,165508,2.43213431399,131584,2909,true,1,0.0680505180359 +fp16,1,257,1,256,257,baseline_transform,4,11,100,10.4838860035,263168,25.1021424606,131584,2909,true,1,0.0104838860035 +fp16,1,257,1,256,257,control_optimized_transform,4,12,100,11.0352003574,263168,23.8480491043,131584,2909,true,1,0.0110352003574 +fp16,1,257,1,256,257,optimized_transform,4,13,100,11.1190855503,263168,23.6681333918,131584,2909,true,1,0.0111190855503 +fp16,1,257,1,256,257,control_shuffle32_transform,4,14,100,13.7705147266,263168,19.1109777103,131584,2909,true,1,0.0137705147266 +fp16,4,64,16,256,4096,baseline_transform,0,0,100,49.1506290436,4194304,85.3357135324,2097152,2909,true,1,0.0491506290436 +fp16,4,64,16,256,4096,control_optimized_transform,0,1,100,37.4077701569,4194304,112.123871121,2097152,2909,true,1,0.0374077701569 +fp16,4,64,16,256,4096,optimized_transform,0,2,100,37.4717712402,4194304,111.93236565,2097152,2909,true,1,0.0374717712402 +fp16,4,64,16,256,4096,control_shuffle32_transform,0,3,100,22.3076581955,4194304,188.020811653,2097152,2909,true,1,0.0223076581955 +fp16,4,64,16,256,4096,shuffle32_transform,0,4,100,22.4651408195,4194304,186.702769134,2097152,2909,true,1,0.0224651408195 +fp16,4,64,16,256,4096,baseline_split,0,5,100,1140.87226868,6832128,5.9885126386,2097152,2909,true,1,1.14087226868 +fp16,4,64,16,256,4096,control_optimized_split,0,6,100,503.423118591,6832128,13.5713433644,2097152,2909,true,1,0.503423118591 +fp16,4,64,16,256,4096,optimized_split,0,7,100,189.060115814,6832128,36.1373310842,2097152,2909,true,1,0.189060115814 +fp16,4,64,16,256,4096,control_shuffle32_split,0,8,100,841.548614502,6832128,8.11851850537,2097152,2909,true,1,0.841548614502 +fp16,4,64,16,256,4096,shuffle32_split,0,9,100,149.067878723,6832128,45.832328591,2097152,2909,true,1,0.149067878723 +fp16,4,64,16,256,4096,baseline_fused,0,10,100,1131.46903992,2637824,2.33132671504,2097152,2909,true,1,1.13146903992 +fp16,4,64,16,256,4096,control_optimized_fused,0,11,100,496.45690918,2637824,5.31329900184,2097152,2909,true,1,0.49645690918 +fp16,4,64,16,256,4096,optimized_fused,0,12,100,171.303081512,2637824,15.3985788038,2097152,2909,true,1,0.171303081512 +fp16,4,64,16,256,4096,control_shuffle32_fused,0,13,100,976.572570801,2637824,2.70110392087,2097152,2909,true,1,0.976572570801 +fp16,4,64,16,256,4096,shuffle32_fused,0,14,100,131.657371521,2637824,20.0355207576,2097152,2909,true,1,0.131657371521 +fp16,4,64,16,256,4096,control_optimized_transform,1,0,100,37.4262857437,4194304,112.068401036,2097152,2909,true,1,0.0374262857437 +fp16,4,64,16,256,4096,optimized_transform,1,1,100,37.4475431442,4194304,112.004784502,2097152,2909,true,1,0.0374475431442 +fp16,4,64,16,256,4096,control_shuffle32_transform,1,2,100,23.6507439613,4194304,177.343427626,2097152,2909,true,1,0.0236507439613 +fp16,4,64,16,256,4096,shuffle32_transform,1,3,100,22.3446846008,4194304,187.709250541,2097152,2909,true,1,0.0223446846008 +fp16,4,64,16,256,4096,baseline_split,1,4,100,1140.34446716,6832128,5.99128438532,2097152,2909,true,1,1.14034446716 +fp16,4,64,16,256,4096,control_optimized_split,1,5,100,502.738075256,6832128,13.5898360126,2097152,2909,true,1,0.502738075256 +fp16,4,64,16,256,4096,optimized_split,1,6,100,188.859672546,6832128,36.1756848769,2097152,2909,true,1,0.188859672546 +fp16,4,64,16,256,4096,control_shuffle32_split,1,7,100,843.81942749,6832128,8.09667065894,2097152,2909,true,1,0.84381942749 +fp16,4,64,16,256,4096,shuffle32_split,1,8,100,149.474058151,6832128,45.7077842437,2097152,2909,true,1,0.149474058151 +fp16,4,64,16,256,4096,baseline_fused,1,9,100,1131.60980225,2637824,2.33103671846,2097152,2909,true,1,1.13160980225 +fp16,4,64,16,256,4096,control_optimized_fused,1,10,100,495.686149597,2637824,5.3215608347,2097152,2909,true,1,0.495686149597 +fp16,4,64,16,256,4096,optimized_fused,1,11,100,171.362953186,2637824,15.3931987688,2097152,2909,true,1,0.171362953186 +fp16,4,64,16,256,4096,control_shuffle32_fused,1,12,100,979.061050415,2637824,2.69423852464,2097152,2909,true,1,0.979061050415 +fp16,4,64,16,256,4096,shuffle32_fused,1,13,100,131.344003677,2637824,20.0833226196,2097152,2909,true,1,0.131344003677 +fp16,4,64,16,256,4096,baseline_transform,1,14,100,49.1915416718,4194304,85.2647397796,2097152,2909,true,1,0.0491915416718 +fp16,4,64,16,256,4096,optimized_transform,2,0,100,37.4660563469,4194304,111.949439278,2097152,2909,true,1,0.0374660563469 +fp16,4,64,16,256,4096,control_shuffle32_transform,2,1,100,23.1392002106,4194304,181.264000563,2097152,2909,true,1,0.0231392002106 +fp16,4,64,16,256,4096,shuffle32_transform,2,2,100,22.3981714249,4194304,187.261000929,2097152,2909,true,1,0.0223981714249 +fp16,4,64,16,256,4096,baseline_split,2,3,100,1140.56427002,6832128,5.99012978013,2097152,2909,true,1,1.14056427002 +fp16,4,64,16,256,4096,control_optimized_split,2,4,100,503.74786377,6832128,13.562594487,2097152,2909,true,1,0.50374786377 +fp16,4,64,16,256,4096,optimized_split,2,5,100,188.896007538,6832128,36.1687263222,2097152,2909,true,1,0.188896007538 +fp16,4,64,16,256,4096,control_shuffle32_split,2,6,100,841.07131958,6832128,8.12312563863,2097152,2909,true,1,0.84107131958 +fp16,4,64,16,256,4096,shuffle32_split,2,7,100,149.548578262,6832128,45.6850080381,2097152,2909,true,1,0.149548578262 +fp16,4,64,16,256,4096,baseline_fused,2,8,100,1130.44578552,2637824,2.33343698016,2097152,2909,true,1,1.13044578552 +fp16,4,64,16,256,4096,control_optimized_fused,2,9,100,495.312004089,2637824,5.32558060015,2097152,2909,true,1,0.495312004089 +fp16,4,64,16,256,4096,optimized_fused,2,10,100,171.252803802,2637824,15.4030996365,2097152,2909,true,1,0.171252803802 +fp16,4,64,16,256,4096,control_shuffle32_fused,2,11,100,986.292572021,2637824,2.67448430093,2097152,2909,true,1,0.986292572021 +fp16,4,64,16,256,4096,shuffle32_fused,2,12,100,131.637029648,2637824,20.0386168471,2097152,2909,true,1,0.131637029648 +fp16,4,64,16,256,4096,baseline_transform,2,13,100,49.1979455948,4194304,85.2536411692,2097152,2909,true,1,0.0491979455948 +fp16,4,64,16,256,4096,control_optimized_transform,2,14,100,37.4555420876,4194304,111.980864946,2097152,2909,true,1,0.0374555420876 +fp16,4,64,16,256,4096,control_shuffle32_transform,3,0,100,22.1874284744,4194304,189.039662926,2097152,2909,true,1,0.0221874284744 +fp16,4,64,16,256,4096,shuffle32_transform,3,1,100,23.0555415154,4194304,181.921730062,2097152,2909,true,1,0.0230555415154 +fp16,4,64,16,256,4096,baseline_split,3,2,100,1140.6137085,6832128,5.98987014544,2097152,2909,true,1,1.1406137085 +fp16,4,64,16,256,4096,control_optimized_split,3,3,100,503.375091553,6832128,13.5726382069,2097152,2909,true,1,0.503375091553 +fp16,4,64,16,256,4096,optimized_split,3,4,100,188.962974548,6832128,36.1559084066,2097152,2909,true,1,0.188962974548 +fp16,4,64,16,256,4096,control_shuffle32_split,3,5,100,843.381500244,6832128,8.10087486864,2097152,2909,true,1,0.843381500244 +fp16,4,64,16,256,4096,shuffle32_split,3,6,100,149.141483307,6832128,45.8097093345,2097152,2909,true,1,0.149141483307 +fp16,4,64,16,256,4096,baseline_fused,3,7,100,1130.66192627,2637824,2.3329909133,2097152,2909,true,1,1.13066192627 +fp16,4,64,16,256,4096,control_optimized_fused,3,8,100,494.748802185,2637824,5.3316430244,2097152,2909,true,1,0.494748802185 +fp16,4,64,16,256,4096,optimized_fused,3,9,100,171.619205475,2637824,15.3702144973,2097152,2909,true,1,0.171619205475 +fp16,4,64,16,256,4096,control_shuffle32_fused,3,10,100,964.209365845,2637824,2.7357377904,2097152,2909,true,1,0.964209365845 +fp16,4,64,16,256,4096,shuffle32_fused,3,11,100,131.236572266,2637824,20.099763004,2097152,2909,true,1,0.131236572266 +fp16,4,64,16,256,4096,baseline_transform,3,12,100,49.1430854797,4194304,85.3488127385,2097152,2909,true,1,0.0491430854797 +fp16,4,64,16,256,4096,control_optimized_transform,3,13,100,37.4205708504,4194304,112.085516193,2097152,2909,true,1,0.0374205708504 +fp16,4,64,16,256,4096,optimized_transform,3,14,100,37.4575996399,4194304,111.974713818,2097152,2909,true,1,0.0374575996399 +fp16,4,64,16,256,4096,shuffle32_transform,4,0,100,22.2230863571,4194304,188.736340785,2097152,2909,true,1,0.0222230863571 +fp16,4,64,16,256,4096,baseline_split,4,1,100,1141.64161682,6832128,5.98447700166,2097152,2909,true,1,1.14164161682 +fp16,4,64,16,256,4096,control_optimized_split,4,2,100,503.916320801,6832128,13.5580605707,2097152,2909,true,1,0.503916320801 +fp16,4,64,16,256,4096,optimized_split,4,3,100,188.911323547,6832128,36.1657939382,2097152,2909,true,1,0.188911323547 +fp16,4,64,16,256,4096,control_shuffle32_split,4,4,100,842.629013062,6832128,8.10810913711,2097152,2909,true,1,0.842629013062 +fp16,4,64,16,256,4096,shuffle32_split,4,5,100,149.184913635,6832128,45.7963733297,2097152,2909,true,1,0.149184913635 +fp16,4,64,16,256,4096,baseline_fused,4,6,100,1128.70559692,2637824,2.33703457056,2097152,2909,true,1,1.12870559692 +fp16,4,64,16,256,4096,control_optimized_fused,4,7,100,493.025131226,2637824,5.35028304428,2097152,2909,true,1,0.493025131226 +fp16,4,64,16,256,4096,optimized_fused,4,8,100,171.149253845,2637824,15.4124189311,2097152,2909,true,1,0.171149253845 +fp16,4,64,16,256,4096,control_shuffle32_fused,4,9,100,983.843460083,2637824,2.68114197738,2097152,2909,true,1,0.983843460083 +fp16,4,64,16,256,4096,shuffle32_fused,4,10,100,131.703777313,2637824,20.028461247,2097152,2909,true,1,0.131703777313 +fp16,4,64,16,256,4096,baseline_transform,4,11,100,49.1636562347,4194304,85.3131016126,2097152,2909,true,1,0.0491636562347 +fp16,4,64,16,256,4096,control_optimized_transform,4,12,100,37.3808002472,4194304,112.204767481,2097152,2909,true,1,0.0373808002472 +fp16,4,64,16,256,4096,optimized_transform,4,13,100,37.4564576149,4194304,111.97812786,2097152,2909,true,1,0.0374564576149 +fp16,4,64,16,256,4096,control_shuffle32_transform,4,14,100,22.4381709099,4194304,186.927179441,2097152,2909,true,1,0.0224381709099 +fp16,16,64,16,256,16384,baseline_transform,0,0,100,170.561141968,16777216,98.364819832,8388608,2909,true,1,0.170561141968 +fp16,16,64,16,256,16384,control_optimized_transform,0,1,100,120.800914764,16777216,138.88318671,8388608,2909,true,1,0.120800914764 +fp16,16,64,16,256,16384,optimized_transform,0,2,100,120.864906311,16777216,138.809655441,8388608,2909,true,1,0.120864906311 +fp16,16,64,16,256,16384,control_shuffle32_transform,0,3,100,43.4150886536,16777216,386.437446526,8388608,2909,true,1,0.0434150886536 +fp16,16,64,16,256,16384,shuffle32_transform,0,4,100,43.5385131836,16777216,385.34195987,8388608,2909,true,1,0.0435385131836 +fp16,16,64,16,256,16384,baseline_split,0,5,100,4265.48095703,27328512,6.4069004821,8388608,2909,true,1,4.26548095703 +fp16,16,64,16,256,16384,control_optimized_split,0,6,100,1870.81970215,27328512,14.6077743187,8388608,2909,true,1,1.87081970215 +fp16,16,64,16,256,16384,optimized_split,0,7,100,677.966156006,27328512,40.3095519709,8388608,2909,true,1,0.677966156006 +fp16,16,64,16,256,16384,control_shuffle32_split,0,8,100,3182.34405518,27328512,8.58754161278,8388608,2909,true,1,3.18234405518 +fp16,16,64,16,256,16384,shuffle32_split,0,9,100,436.499633789,27328512,62.6083274407,8388608,2909,true,1,0.436499633789 +fp16,16,64,16,256,16384,baseline_fused,0,10,100,4260.15930176,10551296,2.4767374299,8388608,2909,true,1,4.26015930176 +fp16,16,64,16,256,16384,control_optimized_fused,0,11,100,1824.02694702,10551296,5.78461629486,8388608,2909,true,1,1.82402694702 +fp16,16,64,16,256,16384,optimized_fused,0,12,100,631.309242249,10551296,16.7133558229,8388608,2909,true,1,0.631309242249 +fp16,16,64,16,256,16384,control_shuffle32_fused,0,13,100,2908.30657959,10551296,3.6279861532,8388608,2909,true,1,2.90830657959 +fp16,16,64,16,256,16384,shuffle32_fused,0,14,100,391.983985901,10551296,26.9176710772,8388608,2909,true,1,0.391983985901 +fp16,16,64,16,256,16384,control_optimized_transform,1,0,100,120.789260864,16777216,138.896586335,8388608,2909,true,1,0.120789260864 +fp16,16,64,16,256,16384,optimized_transform,1,1,100,120.907201767,16777216,138.761097394,8388608,2909,true,1,0.120907201767 +fp16,16,64,16,256,16384,control_shuffle32_transform,1,2,100,43.4111976624,16777216,386.47208332,8388608,2909,true,1,0.0434111976624 +fp16,16,64,16,256,16384,shuffle32_transform,1,3,100,43.5019445419,16777216,385.665886357,8388608,2909,true,1,0.0435019445419 +fp16,16,64,16,256,16384,baseline_split,1,4,100,4265.65124512,27328512,6.4066447137,8388608,2909,true,1,4.26565124512 +fp16,16,64,16,256,16384,control_optimized_split,1,5,100,1870.64285278,27328512,14.6091553283,8388608,2909,true,1,1.87064285278 +fp16,16,64,16,256,16384,optimized_split,1,6,100,677.942199707,27328512,40.3109763809,8388608,2909,true,1,0.677942199707 +fp16,16,64,16,256,16384,control_shuffle32_split,1,7,100,3194.33319092,27328512,8.5553104096,8388608,2909,true,1,3.19433319092 +fp16,16,64,16,256,16384,shuffle32_split,1,8,100,436.395454407,27328512,62.6232737395,8388608,2909,true,1,0.436395454407 +fp16,16,64,16,256,16384,baseline_fused,1,9,100,4265.11260986,10551296,2.47386105952,8388608,2909,true,1,4.26511260986 +fp16,16,64,16,256,16384,control_optimized_fused,1,10,100,1824.10354614,10551296,5.78437338292,8388608,2909,true,1,1.82410354614 +fp16,16,64,16,256,16384,optimized_fused,1,11,100,631.402282715,10551296,16.7108930215,8388608,2909,true,1,0.631402282715 +fp16,16,64,16,256,16384,control_shuffle32_fused,1,12,100,2902.18536377,10551296,3.63563820965,8388608,2909,true,1,2.90218536377 +fp16,16,64,16,256,16384,shuffle32_fused,1,13,100,391.898765564,10551296,26.9235244587,8388608,2909,true,1,0.391898765564 +fp16,16,64,16,256,16384,baseline_transform,1,14,100,170.567321777,16777216,98.3612559849,8388608,2909,true,1,0.170567321777 +fp16,16,64,16,256,16384,optimized_transform,2,0,100,120.906057358,16777216,138.762410806,8388608,2909,true,1,0.120906057358 +fp16,16,64,16,256,16384,control_shuffle32_transform,2,1,100,43.4210300446,16777216,386.384569477,8388608,2909,true,1,0.0434210300446 +fp16,16,64,16,256,16384,shuffle32_transform,2,2,100,43.5531425476,16777216,385.212524714,8388608,2909,true,1,0.0435531425476 +fp16,16,64,16,256,16384,baseline_split,2,3,100,4266.28692627,27328512,6.40569011703,8388608,2909,true,1,4.26628692627 +fp16,16,64,16,256,16384,control_optimized_split,2,4,100,1870.71807861,27328512,14.6085678609,8388608,2909,true,1,1.87071807861 +fp16,16,64,16,256,16384,optimized_split,2,5,100,678.063812256,27328512,40.3037465,8388608,2909,true,1,0.678063812256 +fp16,16,64,16,256,16384,control_shuffle32_split,2,6,100,3166.98547363,27328512,8.62918766996,8388608,2909,true,1,3.16698547363 +fp16,16,64,16,256,16384,shuffle32_split,2,7,100,436.331176758,27328512,62.6324990184,8388608,2909,true,1,0.436331176758 +fp16,16,64,16,256,16384,baseline_fused,2,8,100,4260.47607422,10551296,2.47655328095,8388608,2909,true,1,4.26047607422 +fp16,16,64,16,256,16384,control_optimized_fused,2,9,100,1824.02450562,10551296,5.78462403741,8388608,2909,true,1,1.82402450562 +fp16,16,64,16,256,16384,optimized_fused,2,10,100,631.386947632,10551296,16.7112988946,8388608,2909,true,1,0.631386947632 +fp16,16,64,16,256,16384,control_shuffle32_fused,2,11,100,2904.2678833,10551296,3.63303125744,8388608,2909,true,1,2.9042678833 +fp16,16,64,16,256,16384,shuffle32_fused,2,12,100,391.729354858,10551296,26.9351680418,8388608,2909,true,1,0.391729354858 +fp16,16,64,16,256,16384,baseline_transform,2,13,100,170.539646149,16777216,98.3772183119,8388608,2909,true,1,0.170539646149 +fp16,16,64,16,256,16384,control_optimized_transform,2,14,100,120.787878036,16777216,138.898176479,8388608,2909,true,1,0.120787878036 +fp16,16,64,16,256,16384,control_shuffle32_transform,3,0,100,43.358631134,16777216,386.940628917,8388608,2909,true,1,0.043358631134 +fp16,16,64,16,256,16384,shuffle32_transform,3,1,100,43.4573698044,16777216,386.061468412,8388608,2909,true,1,0.0434573698044 +fp16,16,64,16,256,16384,baseline_split,3,2,100,4264.91821289,27328512,6.40774585487,8388608,2909,true,1,4.26491821289 +fp16,16,64,16,256,16384,control_optimized_split,3,3,100,1870.50857544,27328512,14.610204069,8388608,2909,true,1,1.87050857544 +fp16,16,64,16,256,16384,optimized_split,3,4,100,677.902145386,27328512,40.3133581831,8388608,2909,true,1,0.677902145386 +fp16,16,64,16,256,16384,control_shuffle32_split,3,5,100,3184.65148926,27328512,8.5813195234,8388608,2909,true,1,3.18465148926 +fp16,16,64,16,256,16384,shuffle32_split,3,6,100,436.304206848,27328512,62.6363706127,8388608,2909,true,1,0.436304206848 +fp16,16,64,16,256,16384,baseline_fused,3,7,100,4261.17462158,10551296,2.47614729201,8388608,2909,true,1,4.26117462158 +fp16,16,64,16,256,16384,control_optimized_fused,3,8,100,1824.04922485,10551296,5.78454564506,8388608,2909,true,1,1.82404922485 +fp16,16,64,16,256,16384,optimized_fused,3,9,100,631.384239197,10551296,16.7113705807,8388608,2909,true,1,0.631384239197 +fp16,16,64,16,256,16384,control_shuffle32_fused,3,10,100,2907.04986572,10551296,3.62955452688,8388608,2909,true,1,2.90704986572 +fp16,16,64,16,256,16384,shuffle32_fused,3,11,100,391.836128235,10551296,26.9278283438,8388608,2909,true,1,0.391836128235 +fp16,16,64,16,256,16384,baseline_transform,3,12,100,170.581016541,16777216,98.3533592439,8388608,2909,true,1,0.170581016541 +fp16,16,64,16,256,16384,control_optimized_transform,3,13,100,120.779428482,16777216,138.907893595,8388608,2909,true,1,0.120779428482 +fp16,16,64,16,256,16384,optimized_transform,3,14,100,120.886631012,16777216,138.784709769,8388608,2909,true,1,0.120886631012 +fp16,16,64,16,256,16384,shuffle32_transform,4,0,100,43.5421705246,16777216,385.309592927,8388608,2909,true,1,0.0435421705246 +fp16,16,64,16,256,16384,baseline_split,4,1,100,4264.97406006,27328512,6.40766194944,8388608,2909,true,1,4.26497406006 +fp16,16,64,16,256,16384,control_optimized_split,4,2,100,1870.68069458,27328512,14.6088598012,8388608,2909,true,1,1.87068069458 +fp16,16,64,16,256,16384,optimized_split,4,3,100,677.967758179,27328512,40.3094567114,8388608,2909,true,1,0.677967758179 +fp16,16,64,16,256,16384,control_shuffle32_split,4,4,100,3182.0791626,27328512,8.58825648375,8388608,2909,true,1,3.1820791626 +fp16,16,64,16,256,16384,shuffle32_split,4,5,100,436.286849976,27328512,62.6388624858,8388608,2909,true,1,0.436286849976 +fp16,16,64,16,256,16384,baseline_fused,4,6,100,4261.5423584,10551296,2.47593362042,8388608,2909,true,1,4.2615423584 +fp16,16,64,16,256,16384,control_optimized_fused,4,7,100,1824.11270142,10551296,5.78434435099,8388608,2909,true,1,1.82411270142 +fp16,16,64,16,256,16384,optimized_fused,4,8,100,631.250953674,10551296,16.714899104,8388608,2909,true,1,0.631250953674 +fp16,16,64,16,256,16384,control_shuffle32_fused,4,9,100,2905.88012695,10551296,3.63101557498,8388608,2909,true,1,2.90588012695 +fp16,16,64,16,256,16384,shuffle32_fused,4,10,100,392.497138977,10551296,26.8824787551,8388608,2909,true,1,0.392497138977 +fp16,16,64,16,256,16384,baseline_transform,4,11,100,170.535316467,16777216,98.3797159881,8388608,2909,true,1,0.170535316467 +fp16,16,64,16,256,16384,control_optimized_transform,4,12,100,120.748109818,16777216,138.943922396,8388608,2909,true,1,0.120748109818 +fp16,16,64,16,256,16384,optimized_transform,4,13,100,120.841140747,16777216,138.836954834,8388608,2909,true,1,0.120841140747 +fp16,16,64,16,256,16384,control_shuffle32_transform,4,14,100,43.4617137909,16777216,386.022881673,8388608,2909,true,1,0.0434617137909 +bf16,1,1,1,64,1,baseline_transform,0,0,100,8.97097170353,256,0.028536485061,128,2909,true,1,0.00897097170353 +bf16,1,1,1,64,1,control_optimized_transform,0,1,100,9.41211402416,256,0.0271989905076,128,2909,true,1,0.00941211402416 +bf16,1,1,1,64,1,optimized_transform,0,2,100,9.34354245663,256,0.0273986018888,128,2909,true,1,0.00934354245663 +bf16,1,1,1,64,1,control_shuffle32_transform,0,3,100,7.88937151432,256,0.0324487190818,128,2909,true,1,0.00788937151432 +bf16,1,1,1,64,1,shuffle32_transform,0,4,100,8.01919996738,256,0.0319233840085,128,2909,true,1,0.00801919996738 +bf16,1,1,1,64,1,baseline_split,0,5,100,69.5709705353,420,0.00603700073132,128,2909,true,1,0.0695709705353 +bf16,1,1,1,64,1,control_optimized_split,0,6,100,64.1277694702,420,0.00654942474173,128,2909,true,1,0.0641277694702 +bf16,1,1,1,64,1,optimized_split,0,7,100,33.738515377,420,0.0124486805453,128,2909,true,1,0.033738515377 +bf16,1,1,1,64,1,control_shuffle32_split,0,8,100,93.8484573364,420,0.00447529998809,128,2909,true,1,0.0938484573364 +bf16,1,1,1,64,1,shuffle32_split,0,9,100,34.7501707077,420,0.012086271562,128,2909,true,1,0.0347501707077 +bf16,1,1,1,64,1,baseline_fused,0,10,100,65.0852584839,164,0.00251977181654,128,2909,true,1,0.0650852584839 +bf16,1,1,1,64,1,control_optimized_fused,0,11,100,58.5778284073,164,0.00279969409005,128,2909,true,1,0.0585778284073 +bf16,1,1,1,64,1,optimized_fused,0,12,100,28.7453722954,164,0.00570526616649,128,2909,true,1,0.0287453722954 +bf16,1,1,1,64,1,control_shuffle32_fused,0,13,100,89.3796634674,164,0.00183486929395,128,2909,true,1,0.0893796634674 +bf16,1,1,1,64,1,shuffle32_fused,0,14,100,29.6633124352,164,0.00552871498618,128,2909,true,1,0.0296633124352 +bf16,1,1,1,64,1,control_optimized_transform,1,0,100,9.34834301472,256,0.0273845321676,128,2909,true,1,0.00934834301472 +bf16,1,1,1,64,1,optimized_transform,1,1,100,9.30034279823,256,0.0275258671163,128,2909,true,1,0.00930034279823 +bf16,1,1,1,64,1,control_shuffle32_transform,1,2,100,7.93211460114,256,0.0322738655293,128,2909,true,1,0.00793211460114 +bf16,1,1,1,64,1,shuffle32_transform,1,3,100,7.88022875786,256,0.0324863665594,128,2909,true,1,0.00788022875786 +bf16,1,1,1,64,1,baseline_split,1,4,100,69.6630859375,420,0.0060290180136,128,2909,true,1,0.0696630859375 +bf16,1,1,1,64,1,control_optimized_split,1,5,100,64.235200882,420,0.00653847102886,128,2909,true,1,0.064235200882 +bf16,1,1,1,64,1,optimized_split,1,6,100,33.7266302109,420,0.012453067424,128,2909,true,1,0.0337266302109 +bf16,1,1,1,64,1,control_shuffle32_split,1,7,100,94.5638847351,420,0.00444144190117,128,2909,true,1,0.0945638847351 +bf16,1,1,1,64,1,shuffle32_split,1,8,100,34.7469735146,420,0.012087383663,128,2909,true,1,0.0347469735146 +bf16,1,1,1,64,1,baseline_fused,1,9,100,65.1366853714,164,0.00251778239966,128,2909,true,1,0.0651366853714 +bf16,1,1,1,64,1,control_optimized_fused,1,10,100,58.5714292526,164,0.00279999996744,128,2909,true,1,0.0585714292526 +bf16,1,1,1,64,1,optimized_fused,1,11,100,28.750629425,164,0.00570422294328,128,2909,true,1,0.028750629425 +bf16,1,1,1,64,1,control_shuffle32_fused,1,12,100,89.3716621399,164,0.00183503356739,128,2909,true,1,0.0893716621399 +bf16,1,1,1,64,1,shuffle32_fused,1,13,100,29.6724557877,164,0.00552701135267,128,2909,true,1,0.0296724557877 +bf16,1,1,1,64,1,baseline_transform,1,14,100,9.01074290276,256,0.0284105320463,128,2909,true,1,0.00901074290276 +bf16,1,1,1,64,1,optimized_transform,2,0,100,9.2139428854,256,0.0277839794737,128,2909,true,1,0.0092139428854 +bf16,1,1,1,64,1,control_shuffle32_transform,2,1,100,7.95177102089,256,0.0321940859876,128,2909,true,1,0.00795177102089 +bf16,1,1,1,64,1,shuffle32_transform,2,2,100,7.99885690212,256,0.0320045730449,128,2909,true,1,0.00799885690212 +bf16,1,1,1,64,1,baseline_split,2,3,100,69.6249103546,420,0.00603232374535,128,2909,true,1,0.0696249103546 +bf16,1,1,1,64,1,control_optimized_split,2,4,100,64.1357707977,420,0.00654860766115,128,2909,true,1,0.0641357707977 +bf16,1,1,1,64,1,optimized_split,2,5,100,33.7284564972,420,0.0124523931309,128,2909,true,1,0.0337284564972 +bf16,1,1,1,64,1,control_shuffle32_split,2,6,100,93.9026260376,420,0.004472718365,128,2909,true,1,0.0939026260376 +bf16,1,1,1,64,1,shuffle32_split,2,7,100,34.7627425194,420,0.0120819006086,128,2909,true,1,0.0347627425194 +bf16,1,1,1,64,1,baseline_fused,2,8,100,65.1405715942,164,0.00251763219122,128,2909,true,1,0.0651405715942 +bf16,1,1,1,64,1,control_optimized_fused,2,9,100,58.5689163208,164,0.00280012010299,128,2909,true,1,0.0585689163208 +bf16,1,1,1,64,1,optimized_fused,2,10,100,28.763885498,164,0.00570159410526,128,2909,true,1,0.028763885498 +bf16,1,1,1,64,1,control_shuffle32_fused,2,11,100,89.3670845032,164,0.00183512756304,128,2909,true,1,0.0893670845032 +bf16,1,1,1,64,1,shuffle32_fused,2,12,100,29.6724557877,164,0.00552701135267,128,2909,true,1,0.0296724557877 +bf16,1,1,1,64,1,baseline_transform,2,13,100,9.02811408043,256,0.0283558667646,128,2909,true,1,0.00902811408043 +bf16,1,1,1,64,1,control_optimized_transform,2,14,100,9.27817106247,256,0.0275916447624,128,2909,true,1,0.00927817106247 +bf16,1,1,1,64,1,control_shuffle32_transform,3,0,100,7.88045704365,256,0.032485425475,128,2909,true,1,0.00788045704365 +bf16,1,1,1,64,1,shuffle32_transform,3,1,100,8.0020570755,256,0.0319917738132,128,2909,true,1,0.0080020570755 +bf16,1,1,1,64,1,baseline_split,3,2,100,69.7389698029,420,0.00602245776196,128,2909,true,1,0.0697389698029 +bf16,1,1,1,64,1,control_optimized_split,3,3,100,64.1049146652,420,0.00655175975498,128,2909,true,1,0.0641049146652 +bf16,1,1,1,64,1,optimized_split,3,4,100,33.9058279991,420,0.0123872509473,128,2909,true,1,0.0339058279991 +bf16,1,1,1,64,1,control_shuffle32_split,3,5,100,94.012336731,420,0.00446749878372,128,2909,true,1,0.094012336731 +bf16,1,1,1,64,1,shuffle32_split,3,6,100,34.9360013008,420,0.0120219826071,128,2909,true,1,0.0349360013008 +bf16,1,1,1,64,1,baseline_fused,3,7,100,65.1330280304,164,0.00251792377783,128,2909,true,1,0.0651330280304 +bf16,1,1,1,64,1,control_optimized_fused,3,8,100,58.5778284073,164,0.00279969409005,128,2909,true,1,0.0585778284073 +bf16,1,1,1,64,1,optimized_fused,3,9,100,28.7609148026,164,0.00570218301907,128,2909,true,1,0.0287609148026 +bf16,1,1,1,64,1,control_shuffle32_fused,3,10,100,89.3803405762,164,0.00183485539373,128,2909,true,1,0.0893803405762 +bf16,1,1,1,64,1,shuffle32_fused,3,11,100,29.6228575706,164,0.00553626535215,128,2909,true,1,0.0296228575706 +bf16,1,1,1,64,1,baseline_transform,3,12,100,9.01257157326,256,0.0284047674872,128,2909,true,1,0.00901257157326 +bf16,1,1,1,64,1,control_optimized_transform,3,13,100,9.51771438122,256,0.0268972139472,128,2909,true,1,0.00951771438122 +bf16,1,1,1,64,1,optimized_transform,3,14,100,9.28525686264,256,0.0275705889225,128,2909,true,1,0.00928525686264 +bf16,1,1,1,64,1,shuffle32_transform,4,0,100,7.99725711346,256,0.0320109753092,128,2909,true,1,0.00799725711346 +bf16,1,1,1,64,1,baseline_split,4,1,100,69.6130275726,420,0.00603335344899,128,2909,true,1,0.0696130275726 +bf16,1,1,1,64,1,control_optimized_split,4,2,100,64.0877723694,420,0.00655351222975,128,2909,true,1,0.0640877723694 +bf16,1,1,1,64,1,optimized_split,4,3,100,33.7853717804,420,0.012431415665,128,2909,true,1,0.0337853717804 +bf16,1,1,1,64,1,control_shuffle32_split,4,4,100,93.8443470001,420,0.00447549600403,128,2909,true,1,0.0938443470001 +bf16,1,1,1,64,1,shuffle32_split,4,5,100,34.8032021523,420,0.0120678550831,128,2909,true,1,0.0348032021523 +bf16,1,1,1,64,1,baseline_fused,4,6,100,65.0850248337,164,0.00251978086233,128,2909,true,1,0.0650850248337 +bf16,1,1,1,64,1,control_optimized_fused,4,7,100,58.5609149933,164,0.00280050269055,128,2909,true,1,0.0585609149933 +bf16,1,1,1,64,1,optimized_fused,4,8,100,28.7595438957,164,0.00570245483011,128,2909,true,1,0.0287595438957 +bf16,1,1,1,64,1,control_shuffle32_fused,4,9,100,89.3183994293,164,0.00183612784206,128,2909,true,1,0.0893183994293 +bf16,1,1,1,64,1,shuffle32_fused,4,10,100,29.7028565407,164,0.005521354479,128,2909,true,1,0.0297028565407 +bf16,1,1,1,64,1,baseline_transform,4,11,100,9.03017103672,256,0.0283494076645,128,2909,true,1,0.00903017103672 +bf16,1,1,1,64,1,control_optimized_transform,4,12,100,9.2550855875,256,0.0276604681372,128,2909,true,1,0.0092550855875 +bf16,1,1,1,64,1,optimized_transform,4,13,100,9.33897137642,256,0.0274120124885,128,2909,true,1,0.00933897137642 +bf16,1,1,1,64,1,control_shuffle32_transform,4,14,100,7.86697149277,256,0.0325411119432,128,2909,true,1,0.00786697149277 +bf16,1,17,1,64,17,baseline_transform,0,0,100,9.27839994431,4352,0.469046390124,2176,2909,true,1,0.00927839994431 +bf16,1,17,1,64,17,control_optimized_transform,0,1,100,9.44640040398,4352,0.460704587344,2176,2909,true,1,0.00944640040398 +bf16,1,17,1,64,17,optimized_transform,0,2,100,9.57120001316,4352,0.454697424985,2176,2909,true,1,0.00957120001316 +bf16,1,17,1,64,17,control_shuffle32_transform,0,3,100,7.95611441135,4352,0.547000680859,2176,2909,true,1,0.00795611441135 +bf16,1,17,1,64,17,shuffle32_transform,0,4,100,7.95794308186,4352,0.546874984558,2176,2909,true,1,0.00795794308186 +bf16,1,17,1,64,17,baseline_split,0,5,100,70.2265119553,7140,0.101671004315,2176,2909,true,1,0.0702265119553 +bf16,1,17,1,64,17,control_optimized_split,0,6,100,64.5444583893,7140,0.110621425575,2176,2909,true,1,0.0645444583893 +bf16,1,17,1,64,17,optimized_split,0,7,100,34.261713028,7140,0.208395884764,2176,2909,true,1,0.034261713028 +bf16,1,17,1,64,17,control_shuffle32_split,0,8,100,93.8681125641,7140,0.0760641692367,2176,2909,true,1,0.0938681125641 +bf16,1,17,1,64,17,shuffle32_split,0,9,100,34.6205711365,7140,0.206235765778,2176,2909,true,1,0.0346205711365 +bf16,1,17,1,64,17,baseline_fused,0,10,100,65.408000946,2788,0.0426247547651,2176,2909,true,1,0.065408000946 +bf16,1,17,1,64,17,control_optimized_fused,0,11,100,58.8153171539,2788,0.0474026178028,2176,2909,true,1,0.0588153171539 +bf16,1,17,1,64,17,optimized_fused,0,12,100,28.920686245,2788,0.0964015852316,2176,2909,true,1,0.028920686245 +bf16,1,17,1,64,17,control_shuffle32_fused,0,13,100,89.0651416779,2788,0.0313029311746,2176,2909,true,1,0.0890651416779 +bf16,1,17,1,64,17,shuffle32_fused,0,14,100,29.661257267,2788,0.0939946670131,2176,2909,true,1,0.029661257267 +bf16,1,17,1,64,17,control_optimized_transform,1,0,100,9.5490282774,4352,0.455753179651,2176,2909,true,1,0.0095490282774 +bf16,1,17,1,64,17,optimized_transform,1,1,100,9.47154283524,4352,0.459481636277,2176,2909,true,1,0.00947154283524 +bf16,1,17,1,64,17,control_shuffle32_transform,1,2,100,8.00799965858,4352,0.543456566626,2176,2909,true,1,0.00800799965858 +bf16,1,17,1,64,17,shuffle32_transform,1,3,100,7.96617150307,4352,0.546310106219,2176,2909,true,1,0.00796617150307 +bf16,1,17,1,64,17,baseline_split,1,4,100,70.2139425278,7140,0.101689205063,2176,2909,true,1,0.0702139425278 +bf16,1,17,1,64,17,control_optimized_split,1,5,100,64.5172548294,7140,0.110668068858,2176,2909,true,1,0.0645172548294 +bf16,1,17,1,64,17,optimized_split,1,6,100,34.3179416656,7140,0.20805443606,2176,2909,true,1,0.0343179416656 +bf16,1,17,1,64,17,control_shuffle32_split,1,7,100,94.6640014648,7140,0.0754246586824,2176,2909,true,1,0.0946640014648 +bf16,1,17,1,64,17,shuffle32_split,1,8,100,34.6795415878,7140,0.205885074401,2176,2909,true,1,0.0346795415878 +bf16,1,17,1,64,17,baseline_fused,1,9,100,65.4591989517,2788,0.0425914164036,2176,2909,true,1,0.0654591989517 +bf16,1,17,1,64,17,control_optimized_fused,1,10,100,58.8393163681,2788,0.047383283357,2176,2909,true,1,0.0588393163681 +bf16,1,17,1,64,17,optimized_fused,1,11,100,28.9494848251,2788,0.0963056861578,2176,2909,true,1,0.0289494848251 +bf16,1,17,1,64,17,control_shuffle32_fused,1,12,100,89.1094875336,2788,0.0312873530885,2176,2909,true,1,0.0891094875336 +bf16,1,17,1,64,17,shuffle32_fused,1,13,100,29.7277712822,2788,0.0937843598679,2176,2909,true,1,0.0297277712822 +bf16,1,17,1,64,17,baseline_transform,1,14,100,9.29782867432,4352,0.468066271432,2176,2909,true,1,0.00929782867432 +bf16,1,17,1,64,17,optimized_transform,2,0,100,9.67199981213,4352,0.449958652247,2176,2909,true,1,0.00967199981213 +bf16,1,17,1,64,17,control_shuffle32_transform,2,1,100,8.03405702114,4352,0.541693939755,2176,2909,true,1,0.00803405702114 +bf16,1,17,1,64,17,shuffle32_transform,2,2,100,8.14788579941,4352,0.534126288358,2176,2909,true,1,0.00814788579941 +bf16,1,17,1,64,17,baseline_split,2,3,100,70.2829694748,7140,0.101589333139,2176,2909,true,1,0.0702829694748 +bf16,1,17,1,64,17,control_optimized_split,2,4,100,64.4333696365,7140,0.110812146567,2176,2909,true,1,0.0644333696365 +bf16,1,17,1,64,17,optimized_split,2,5,100,34.4338297844,7140,0.207354222423,2176,2909,true,1,0.0344338297844 +bf16,1,17,1,64,17,control_shuffle32_split,2,6,100,94.5003414154,7140,0.0755552825848,2176,2909,true,1,0.0945003414154 +bf16,1,17,1,64,17,shuffle32_split,2,7,100,34.6370267868,7140,0.206137785554,2176,2909,true,1,0.0346370267868 +bf16,1,17,1,64,17,baseline_fused,2,8,100,65.4608011246,2788,0.0425903739659,2176,2909,true,1,0.0654608011246 +bf16,1,17,1,64,17,control_optimized_fused,2,9,100,58.8418245316,2788,0.0473812636198,2176,2909,true,1,0.0588418245316 +bf16,1,17,1,64,17,optimized_fused,2,10,100,28.9348578453,2788,0.0963543700441,2176,2909,true,1,0.0289348578453 +bf16,1,17,1,64,17,control_shuffle32_fused,2,11,100,89.1135978699,2788,0.0312859099693,2176,2909,true,1,0.0891135978699 +bf16,1,17,1,64,17,shuffle32_fused,2,12,100,29.7056007385,2788,0.0938543550942,2176,2909,true,1,0.0297056007385 +bf16,1,17,1,64,17,baseline_transform,2,13,100,9.30034279823,4352,0.467939740977,2176,2909,true,1,0.00930034279823 +bf16,1,17,1,64,17,control_optimized_transform,2,14,100,9.52639997005,4352,0.456835742115,2176,2909,true,1,0.00952639997005 +bf16,1,17,1,64,17,control_shuffle32_transform,3,0,100,7.92228579521,4352,0.549336405237,2176,2909,true,1,0.00792228579521 +bf16,1,17,1,64,17,shuffle32_transform,3,1,100,8.15268576145,4352,0.533811817031,2176,2909,true,1,0.00815268576145 +bf16,1,17,1,64,17,baseline_split,3,2,100,70.2374839783,7140,0.10165512196,2176,2909,true,1,0.0702374839783 +bf16,1,17,1,64,17,control_optimized_split,3,3,100,64.5394277573,7140,0.110630048144,2176,2909,true,1,0.0645394277573 +bf16,1,17,1,64,17,optimized_split,3,4,100,34.3211436272,7140,0.208035025801,2176,2909,true,1,0.0343211436272 +bf16,1,17,1,64,17,control_shuffle32_split,3,5,100,94.496679306,7140,0.0755582106423,2176,2909,true,1,0.094496679306 +bf16,1,17,1,64,17,shuffle32_split,3,6,100,34.6182847023,7140,0.206249387033,2176,2909,true,1,0.0346182847023 +bf16,1,17,1,64,17,baseline_fused,3,7,100,65.4045724869,2788,0.0426269891231,2176,2909,true,1,0.0654045724869 +bf16,1,17,1,64,17,control_optimized_fused,3,8,100,58.8137149811,2788,0.0474039091205,2176,2909,true,1,0.0588137149811 +bf16,1,17,1,64,17,optimized_fused,3,9,100,28.9924573898,2788,0.0961629420546,2176,2909,true,1,0.0289924573898 +bf16,1,17,1,64,17,control_shuffle32_fused,3,10,100,89.1152000427,2788,0.0312853474903,2176,2909,true,1,0.0891152000427 +bf16,1,17,1,64,17,shuffle32_fused,3,11,100,29.7129154205,2788,0.0938312501665,2176,2909,true,1,0.0297129154205 +bf16,1,17,1,64,17,baseline_transform,3,12,100,9.35017108917,4352,0.46544602858,2176,2909,true,1,0.00935017108917 +bf16,1,17,1,64,17,control_optimized_transform,3,13,100,9.45508599281,4352,0.460281376955,2176,2909,true,1,0.00945508599281 +bf16,1,17,1,64,17,optimized_transform,3,14,100,9.56754267216,4352,0.454871240101,2176,2909,true,1,0.00956754267216 +bf16,1,17,1,64,17,shuffle32_transform,4,0,100,8.01142811775,4352,0.543223996525,2176,2909,true,1,0.00801142811775 +bf16,1,17,1,64,17,baseline_split,4,1,100,70.2217149734,7140,0.101677949658,2176,2909,true,1,0.0702217149734 +bf16,1,17,1,64,17,control_optimized_split,4,2,100,64.5259428024,7140,0.110653168166,2176,2909,true,1,0.0645259428024 +bf16,1,17,1,64,17,optimized_split,4,3,100,34.3595433235,7140,0.207802529061,2176,2909,true,1,0.0343595433235 +bf16,1,17,1,64,17,control_shuffle32_split,4,4,100,93.8633155823,7140,0.0760680565747,2176,2909,true,1,0.0938633155823 +bf16,1,17,1,64,17,shuffle32_split,4,5,100,34.5675444603,7140,0.206552131818,2176,2909,true,1,0.0345675444603 +bf16,1,17,1,64,17,baseline_fused,4,6,100,65.4452562332,2788,0.0426004902489,2176,2909,true,1,0.0654452562332 +bf16,1,17,1,64,17,control_optimized_fused,4,7,100,58.8356542587,2788,0.0473862326361,2176,2909,true,1,0.0588356542587 +bf16,1,17,1,64,17,optimized_fused,4,8,100,28.8642287254,2788,0.0965901436869,2176,2909,true,1,0.0288642287254 +bf16,1,17,1,64,17,control_shuffle32_fused,4,9,100,89.1088008881,2788,0.0312875941794,2176,2909,true,1,0.0891088008881 +bf16,1,17,1,64,17,shuffle32_fused,4,10,100,29.7572588921,2788,0.0936914253464,2176,2909,true,1,0.0297572588921 +bf16,1,17,1,64,17,baseline_transform,4,11,100,9.37257111073,4352,0.464333633598,2176,2909,true,1,0.00937257111073 +bf16,1,17,1,64,17,control_optimized_transform,4,12,100,9.36114311218,4352,0.464900487883,2176,2909,true,1,0.00936114311218 +bf16,1,17,1,64,17,optimized_transform,4,13,100,9.45302844048,4352,0.4603815621,2176,2909,true,1,0.00945302844048 +bf16,1,17,1,64,17,control_shuffle32_transform,4,14,100,7.9711997509,4352,0.545965492774,2176,2909,true,1,0.0079711997509 +bf16,1,257,1,64,257,baseline_transform,0,0,100,9.53531444073,65792,6.89982489922,32896,2909,true,1,0.00953531444073 +bf16,1,257,1,64,257,control_optimized_transform,0,1,100,9.5759999752,65792,6.87050962514,32896,2909,true,1,0.0095759999752 +bf16,1,257,1,64,257,optimized_transform,0,2,100,9.62902843952,65792,6.83267272635,32896,2909,true,1,0.00962902843952 +bf16,1,257,1,64,257,control_shuffle32_transform,0,3,100,8.84799957275,65792,7.43580506068,32896,2909,true,1,0.00884799957275 +bf16,1,257,1,64,257,shuffle32_transform,0,4,100,8.88342857361,65792,7.40614949001,32896,2909,true,1,0.00888342857361 +bf16,1,257,1,64,257,baseline_split,0,5,100,70.8850288391,107940,1.52274749362,32896,2909,true,1,0.0708850288391 +bf16,1,257,1,64,257,control_optimized_split,0,6,100,64.602971077,107940,1.67082098858,32896,2909,true,1,0.064602971077 +bf16,1,257,1,64,257,optimized_split,0,7,100,34.6795415878,107940,3.11249788947,32896,2909,true,1,0.0346795415878 +bf16,1,257,1,64,257,control_shuffle32_split,0,8,100,103.000917435,107940,1.0479518308,32896,2909,true,1,0.103000917435 +bf16,1,257,1,64,257,shuffle32_split,0,9,100,39.3147444725,107940,2.74553482283,32896,2909,true,1,0.0393147444725 +bf16,1,257,1,64,257,baseline_fused,0,10,100,65.9266281128,42148,0.639316786047,32896,2909,true,1,0.0659266281128 +bf16,1,257,1,64,257,control_optimized_fused,0,11,100,59.1849184036,42148,0.712140882117,32896,2909,true,1,0.0591849184036 +bf16,1,257,1,64,257,optimized_fused,0,12,100,29.1661715508,42148,1.44509881685,32896,2909,true,1,0.0291661715508 +bf16,1,257,1,64,257,control_shuffle32_fused,0,13,100,95.9764575958,42148,0.439149360747,32896,2909,true,1,0.0959764575958 +bf16,1,257,1,64,257,shuffle32_fused,0,14,100,33.282058239,42148,1.26638802496,32896,2909,true,1,0.033282058239 +bf16,1,257,1,64,257,control_optimized_transform,1,0,100,9.62697148323,65792,6.83413263606,32896,2909,true,1,0.00962697148323 +bf16,1,257,1,64,257,optimized_transform,1,1,100,9.68297123909,65792,6.79460863566,32896,2909,true,1,0.00968297123909 +bf16,1,257,1,64,257,control_shuffle32_transform,1,2,100,8.94308567047,65792,7.35674491157,32896,2909,true,1,0.00894308567047 +bf16,1,257,1,64,257,shuffle32_transform,1,3,100,8.85051429272,65792,7.43369230578,32896,2909,true,1,0.00885051429272 +bf16,1,257,1,64,257,baseline_split,1,4,100,70.8589696884,107940,1.52330750044,32896,2909,true,1,0.0708589696884 +bf16,1,257,1,64,257,control_optimized_split,1,5,100,64.6086883545,107940,1.67067313622,32896,2909,true,1,0.0646086883545 +bf16,1,257,1,64,257,optimized_split,1,6,100,34.6875429153,107940,3.11177993389,32896,2909,true,1,0.0346875429153 +bf16,1,257,1,64,257,control_shuffle32_split,1,7,100,102.930059433,107940,1.0486732505,32896,2909,true,1,0.102930059433 +bf16,1,257,1,64,257,shuffle32_split,1,8,100,39.3076562881,107940,2.74602991358,32896,2909,true,1,0.0393076562881 +bf16,1,257,1,64,257,baseline_fused,1,9,100,65.9243440628,42148,0.639338936158,32896,2909,true,1,0.0659243440628 +bf16,1,257,1,64,257,control_optimized_fused,1,10,100,59.1750860214,42148,0.71225920964,32896,2909,true,1,0.0591750860214 +bf16,1,257,1,64,257,optimized_fused,1,11,100,29.1465163231,42148,1.44607333284,32896,2909,true,1,0.0291465163231 +bf16,1,257,1,64,257,control_shuffle32_fused,1,12,100,95.9865093231,42148,0.439103372935,32896,2909,true,1,0.0959865093231 +bf16,1,257,1,64,257,shuffle32_fused,1,13,100,33.2829713821,42148,1.26635328066,32896,2909,true,1,0.0332829713821 +bf16,1,257,1,64,257,baseline_transform,1,14,100,9.58628535271,65792,6.86313807479,32896,2909,true,1,0.00958628535271 +bf16,1,257,1,64,257,optimized_transform,2,0,100,9.57965731621,65792,6.86788658804,32896,2909,true,1,0.00957965731621 +bf16,1,257,1,64,257,control_shuffle32_transform,2,1,100,9.02400016785,65792,7.29078000624,32896,2909,true,1,0.00902400016785 +bf16,1,257,1,64,257,shuffle32_transform,2,2,100,8.87017130852,65792,7.41721864344,32896,2909,true,1,0.00887017130852 +bf16,1,257,1,64,257,baseline_split,2,3,100,70.9097146988,107940,1.52221737823,32896,2909,true,1,0.0709097146988 +bf16,1,257,1,64,257,control_optimized_split,2,4,100,64.6041154861,107940,1.67079139135,32896,2909,true,1,0.0646041154861 +bf16,1,257,1,64,257,optimized_split,2,5,100,34.6262836456,107940,3.11728515554,32896,2909,true,1,0.0346262836456 +bf16,1,257,1,64,257,control_shuffle32_split,2,6,100,102.347202301,107940,1.0546453403,32896,2909,true,1,0.102347202301 +bf16,1,257,1,64,257,shuffle32_split,2,7,100,39.4178271294,107940,2.7383548983,32896,2909,true,1,0.0394178271294 +bf16,1,257,1,64,257,baseline_fused,2,8,100,65.8740568161,42148,0.639826997716,32896,2909,true,1,0.0658740568161 +bf16,1,257,1,64,257,control_optimized_fused,2,9,100,59.1961145401,42148,0.712006190397,32896,2909,true,1,0.0591961145401 +bf16,1,257,1,64,257,optimized_fused,2,10,100,29.1268563271,42148,1.44704940096,32896,2909,true,1,0.0291268563271 +bf16,1,257,1,64,257,control_shuffle32_fused,2,11,100,95.9947395325,42148,0.439065725948,32896,2909,true,1,0.0959947395325 +bf16,1,257,1,64,257,shuffle32_fused,2,12,100,33.3101725578,42148,1.2653191732,32896,2909,true,1,0.0333101725578 +bf16,1,257,1,64,257,baseline_transform,2,13,100,9.58765745163,65792,6.86215588447,32896,2909,true,1,0.00958765745163 +bf16,1,257,1,64,257,control_optimized_transform,2,14,100,9.71588611603,65792,6.77159028156,32896,2909,true,1,0.00971588611603 +bf16,1,257,1,64,257,control_shuffle32_transform,3,0,100,8.90079975128,65792,7.39169533508,32896,2909,true,1,0.00890079975128 +bf16,1,257,1,64,257,shuffle32_transform,3,1,100,8.93828570843,65792,7.36069556805,32896,2909,true,1,0.00893828570843 +bf16,1,257,1,64,257,baseline_split,3,2,100,70.8708572388,107940,1.52305198788,32896,2909,true,1,0.0708708572388 +bf16,1,257,1,64,257,control_optimized_split,3,3,100,64.6244525909,107940,1.67026559874,32896,2909,true,1,0.0646244525909 +bf16,1,257,1,64,257,optimized_split,3,4,100,34.6189713478,107940,3.1179435956,32896,2909,true,1,0.0346189713478 +bf16,1,257,1,64,257,control_shuffle32_split,3,5,100,103.019886017,107940,1.04775887621,32896,2909,true,1,0.103019886017 +bf16,1,257,1,64,257,shuffle32_split,3,6,100,39.4150853157,107940,2.73854538524,32896,2909,true,1,0.0394150853157 +bf16,1,257,1,64,257,baseline_fused,3,7,100,65.9195423126,42148,0.639385507261,32896,2909,true,1,0.0659195423126 +bf16,1,257,1,64,257,control_optimized_fused,3,8,100,59.1428565979,42148,0.712647349562,32896,2909,true,1,0.0591428565979 +bf16,1,257,1,64,257,optimized_fused,3,9,100,29.1275429726,42148,1.44701528858,32896,2909,true,1,0.0291275429726 +bf16,1,257,1,64,257,control_shuffle32_fused,3,10,100,95.9926795959,42148,0.439075147995,32896,2909,true,1,0.0959926795959 +bf16,1,257,1,64,257,shuffle32_fused,3,11,100,33.2653713226,42148,1.2670232835,32896,2909,true,1,0.0332653713226 +bf16,1,257,1,64,257,baseline_transform,3,12,100,9.66331481934,65792,6.80842973969,32896,2909,true,1,0.00966331481934 +bf16,1,257,1,64,257,control_optimized_transform,3,13,100,9.70948517323,65792,6.77605442783,32896,2909,true,1,0.00970948517323 +bf16,1,257,1,64,257,optimized_transform,3,14,100,9.64571416378,65792,6.82085316679,32896,2909,true,1,0.00964571416378 +bf16,1,257,1,64,257,shuffle32_transform,4,0,100,8.98925721645,65792,7.31895844293,32896,2909,true,1,0.00898925721645 +bf16,1,257,1,64,257,baseline_split,4,1,100,70.8642292023,107940,1.52319444119,32896,2909,true,1,0.0708642292023 +bf16,1,257,1,64,257,control_optimized_split,4,2,100,64.5664024353,107940,1.67176729582,32896,2909,true,1,0.0645664024353 +bf16,1,257,1,64,257,optimized_split,4,3,100,34.6068572998,107940,3.11903502433,32896,2909,true,1,0.0346068572998 +bf16,1,257,1,64,257,control_shuffle32_split,4,4,100,102.383089066,107940,1.05427567175,32896,2909,true,1,0.102383089066 +bf16,1,257,1,64,257,shuffle32_split,4,5,100,39.390399456,107940,2.74026162442,32896,2909,true,1,0.039390399456 +bf16,1,257,1,64,257,baseline_fused,4,6,100,65.9250259399,42148,0.639332323333,32896,2909,true,1,0.0659250259399 +bf16,1,257,1,64,257,control_optimized_fused,4,7,100,59.1791963577,42148,0.712209739132,32896,2909,true,1,0.0591791963577 +bf16,1,257,1,64,257,optimized_fused,4,8,100,29.1280007362,42148,1.44699254788,32896,2909,true,1,0.0291280007362 +bf16,1,257,1,64,257,control_shuffle32_fused,4,9,100,95.9894943237,42148,0.439089718067,32896,2909,true,1,0.0959894943237 +bf16,1,257,1,64,257,shuffle32_fused,4,10,100,33.2866287231,42148,1.26621414114,32896,2909,true,1,0.0332866287231 +bf16,1,257,1,64,257,baseline_transform,4,11,100,9.49211418629,65792,6.93122719647,32896,2909,true,1,0.00949211418629 +bf16,1,257,1,64,257,control_optimized_transform,4,12,100,9.64982867241,65792,6.81794488104,32896,2909,true,1,0.00964982867241 +bf16,1,257,1,64,257,optimized_transform,4,13,100,9.62285757065,65792,6.83705432788,32896,2909,true,1,0.00962285757065 +bf16,1,257,1,64,257,control_shuffle32_transform,4,14,100,9.08868551254,65792,7.23889058646,32896,2909,true,1,0.00908868551254 +bf16,4,64,16,64,4096,baseline_transform,0,0,100,22.7705144882,1048576,46.0497280614,524288,2909,true,1,0.0227705144882 +bf16,4,64,16,64,4096,control_optimized_transform,0,1,100,24.0077710152,1048576,43.6765245444,524288,2909,true,1,0.0240077710152 +bf16,4,64,16,64,4096,optimized_transform,0,2,100,24.0038847923,1048576,43.6835957626,524288,2909,true,1,0.0240038847923 +bf16,4,64,16,64,4096,control_shuffle32_transform,0,3,100,11.1332571507,1048576,94.1841175328,524288,2909,true,1,0.0111332571507 +bf16,4,64,16,64,4096,shuffle32_transform,0,4,100,11.4361143112,1048576,91.6898844716,524288,2909,true,1,0.0114361143112 +bf16,4,64,16,64,4096,baseline_split,0,5,100,438.13369751,1720320,3.92647269493,524288,2909,true,1,0.43813369751 +bf16,4,64,16,64,4096,control_optimized_split,0,6,100,380.320205688,1720320,4.52334631258,524288,2909,true,1,0.380320205688 +bf16,4,64,16,64,4096,optimized_split,0,7,100,162.030620575,1720320,10.6172524298,524288,2909,true,1,0.162030620575 +bf16,4,64,16,64,4096,control_shuffle32_split,0,8,100,248.136692047,1720320,6.93295290514,524288,2909,true,1,0.248136692047 +bf16,4,64,16,64,4096,shuffle32_split,0,9,100,72.5872039795,1720320,23.7000449898,524288,2909,true,1,0.0725872039795 +bf16,4,64,16,64,4096,baseline_fused,0,10,100,429.196815491,671744,1.56511878876,524288,2909,true,1,0.429196815491 +bf16,4,64,16,64,4096,control_optimized_fused,0,11,100,370.643005371,671744,1.81237468471,524288,2909,true,1,0.370643005371 +bf16,4,64,16,64,4096,optimized_fused,0,12,100,148.707427979,671744,4.51721887152,524288,2909,true,1,0.148707427979 +bf16,4,64,16,64,4096,control_shuffle32_fused,0,13,100,248.44045639,671744,2.70384304457,524288,2909,true,1,0.24844045639 +bf16,4,64,16,64,4096,shuffle32_fused,0,14,100,65.6420564651,671744,10.2334392945,524288,2909,true,1,0.0656420564651 +bf16,4,64,16,64,4096,control_optimized_transform,1,0,100,23.9638876915,1048576,43.7565061854,524288,2909,true,1,0.0239638876915 +bf16,4,64,16,64,4096,optimized_transform,1,1,100,24.0651416779,1048576,43.5724008625,524288,2909,true,1,0.0240651416779 +bf16,4,64,16,64,4096,control_shuffle32_transform,1,2,100,11.2320005894,1048576,93.3561204575,524288,2909,true,1,0.0112320005894 +bf16,4,64,16,64,4096,shuffle32_transform,1,3,100,11.3099431992,1048576,92.7127556289,524288,2909,true,1,0.0113099431992 +bf16,4,64,16,64,4096,baseline_split,1,4,100,438.721351624,1720320,3.92121330232,524288,2909,true,1,0.438721351624 +bf16,4,64,16,64,4096,control_optimized_split,1,5,100,380.42011261,1720320,4.52215837958,524288,2909,true,1,0.38042011261 +bf16,4,64,16,64,4096,optimized_split,1,6,100,162.110843658,1720320,10.6119983166,524288,2909,true,1,0.162110843658 +bf16,4,64,16,64,4096,control_shuffle32_split,1,7,100,250.778980255,1720320,6.85990507757,524288,2909,true,1,0.250778980255 +bf16,4,64,16,64,4096,shuffle32_split,1,8,100,71.365942955,1720320,24.1056157709,524288,2909,true,1,0.071365942955 +bf16,4,64,16,64,4096,baseline_fused,1,9,100,430.322303772,671744,1.56102529223,524288,2909,true,1,0.430322303772 +bf16,4,64,16,64,4096,control_optimized_fused,1,10,100,371.517715454,671744,1.80810758695,524288,2909,true,1,0.371517715454 +bf16,4,64,16,64,4096,optimized_fused,1,11,100,148.198852539,671744,4.53272065533,524288,2909,true,1,0.148198852539 +bf16,4,64,16,64,4096,control_shuffle32_fused,1,12,100,246.879081726,671744,2.72094336751,524288,2909,true,1,0.246879081726 +bf16,4,64,16,64,4096,shuffle32_fused,1,13,100,68.2784032822,671744,9.8383085677,524288,2909,true,1,0.0682784032822 +bf16,4,64,16,64,4096,baseline_transform,1,14,100,22.7529144287,1048576,46.0853489027,524288,2909,true,1,0.0227529144287 +bf16,4,64,16,64,4096,optimized_transform,2,0,100,24.0486860275,1048576,43.6022158882,524288,2909,true,1,0.0240486860275 +bf16,4,64,16,64,4096,control_shuffle32_transform,2,1,100,11.4269721508,1048576,91.7632410547,524288,2909,true,1,0.0114269721508 +bf16,4,64,16,64,4096,shuffle32_transform,2,2,100,11.1552000046,1048576,93.9988525145,524288,2909,true,1,0.0111552000046 +bf16,4,64,16,64,4096,baseline_split,2,3,100,437.368659973,1720320,3.93334081163,524288,2909,true,1,0.437368659973 +bf16,4,64,16,64,4096,control_optimized_split,2,4,100,381.885452271,1720320,4.50480632287,524288,2909,true,1,0.381885452271 +bf16,4,64,16,64,4096,optimized_split,2,5,100,161.96434021,1720320,10.6215973082,524288,2909,true,1,0.16196434021 +bf16,4,64,16,64,4096,control_shuffle32_split,2,6,100,249.207324982,1720320,6.90316787489,524288,2909,true,1,0.249207324982 +bf16,4,64,16,64,4096,shuffle32_split,2,7,100,72.102856636,1720320,23.8592488601,524288,2909,true,1,0.072102856636 +bf16,4,64,16,64,4096,baseline_fused,2,8,100,429.793815613,671744,1.56294477863,524288,2909,true,1,0.429793815613 +bf16,4,64,16,64,4096,control_optimized_fused,2,9,100,371.5184021,671744,1.80810424518,524288,2909,true,1,0.3715184021 +bf16,4,64,16,64,4096,optimized_fused,2,10,100,148.330516815,671744,4.52869722578,524288,2909,true,1,0.148330516815 +bf16,4,64,16,64,4096,control_shuffle32_fused,2,11,100,248.104228973,671744,2.70750725524,524288,2909,true,1,0.248104228973 +bf16,4,64,16,64,4096,shuffle32_fused,2,12,100,63.6592006683,671744,10.5521903032,524288,2909,true,1,0.0636592006683 +bf16,4,64,16,64,4096,baseline_transform,2,13,100,22.719771862,1048576,46.1525761072,524288,2909,true,1,0.022719771862 +bf16,4,64,16,64,4096,control_optimized_transform,2,14,100,23.924343586,1048576,43.8288305061,524288,2909,true,1,0.023924343586 +bf16,4,64,16,64,4096,control_shuffle32_transform,3,0,100,11.4998865128,1048576,91.181421559,524288,2909,true,1,0.0114998865128 +bf16,4,64,16,64,4096,shuffle32_transform,3,1,100,11.2767994404,1048576,92.9852486553,524288,2909,true,1,0.0112767994404 +bf16,4,64,16,64,4096,baseline_split,3,2,100,436.746292114,1720320,3.93894586185,524288,2909,true,1,0.436746292114 +bf16,4,64,16,64,4096,control_optimized_split,3,3,100,381.788787842,1720320,4.50594688682,524288,2909,true,1,0.381788787842 +bf16,4,64,16,64,4096,optimized_split,3,4,100,161.946277618,1720320,10.6227819824,524288,2909,true,1,0.161946277618 +bf16,4,64,16,64,4096,control_shuffle32_split,3,5,100,249.878864288,1720320,6.88461589138,524288,2909,true,1,0.249878864288 +bf16,4,64,16,64,4096,shuffle32_split,3,6,100,71.3412570953,1720320,24.113956917,524288,2909,true,1,0.0713412570953 +bf16,4,64,16,64,4096,baseline_fused,3,7,100,430.108795166,671744,1.56180019462,524288,2909,true,1,0.430108795166 +bf16,4,64,16,64,4096,control_optimized_fused,3,8,100,370.276565552,671744,1.81416827986,524288,2909,true,1,0.370276565552 +bf16,4,64,16,64,4096,optimized_fused,3,9,100,148.530521393,671744,4.52259908402,524288,2909,true,1,0.148530521393 +bf16,4,64,16,64,4096,control_shuffle32_fused,3,10,100,247.860565186,671744,2.71016891895,524288,2909,true,1,0.247860565186 +bf16,4,64,16,64,4096,shuffle32_fused,3,11,100,65.703086853,671744,10.2239336411,524288,2909,true,1,0.065703086853 +bf16,4,64,16,64,4096,baseline_transform,3,12,100,22.7997708321,1048576,45.9906377009,524288,2909,true,1,0.0227997708321 +bf16,4,64,16,64,4096,control_optimized_transform,3,13,100,23.9369130135,1048576,43.8058157044,524288,2909,true,1,0.0239369130135 +bf16,4,64,16,64,4096,optimized_transform,3,14,100,24.3846869469,1048576,43.0014132347,524288,2909,true,1,0.0243846869469 +bf16,4,64,16,64,4096,shuffle32_transform,4,0,100,11.3965713978,1048576,92.0080227115,524288,2909,true,1,0.0113965713978 +bf16,4,64,16,64,4096,baseline_split,4,1,100,437.560005188,1720320,3.93162075967,524288,2909,true,1,0.437560005188 +bf16,4,64,16,64,4096,control_optimized_split,4,2,100,381.588134766,1720320,4.50831627943,524288,2909,true,1,0.381588134766 +bf16,4,64,16,64,4096,optimized_split,4,3,100,162.61713028,1720320,10.5789592833,524288,2909,true,1,0.16261713028 +bf16,4,64,16,64,4096,control_shuffle32_split,4,4,100,249.642753601,1720320,6.89112732168,524288,2909,true,1,0.249642753601 +bf16,4,64,16,64,4096,shuffle32_split,4,5,100,71.4351987839,1720320,24.0822455776,524288,2909,true,1,0.0714351987839 +bf16,4,64,16,64,4096,baseline_fused,4,6,100,429.552459717,671744,1.56382296226,524288,2909,true,1,0.429552459717 +bf16,4,64,16,64,4096,control_optimized_fused,4,7,100,372.212333679,671744,1.8047333181,524288,2909,true,1,0.372212333679 +bf16,4,64,16,64,4096,optimized_fused,4,8,100,148.442974091,671744,4.52526638,524288,2909,true,1,0.148442974091 +bf16,4,64,16,64,4096,control_shuffle32_fused,4,9,100,247.6836586,671744,2.71210464105,524288,2909,true,1,0.2476836586 +bf16,4,64,16,64,4096,shuffle32_fused,4,10,100,66.2996578217,671744,10.1319376611,524288,2909,true,1,0.0662996578217 +bf16,4,64,16,64,4096,baseline_transform,4,11,100,23.1748557091,1048576,45.2462795524,524288,2909,true,1,0.0231748557091 +bf16,4,64,16,64,4096,control_optimized_transform,4,12,100,23.9888000488,1048576,43.7110650748,524288,2909,true,1,0.0239888000488 +bf16,4,64,16,64,4096,optimized_transform,4,13,100,24.0283441544,1048576,43.6391285752,524288,2909,true,1,0.0240283441544 +bf16,4,64,16,64,4096,control_shuffle32_transform,4,14,100,10.9979426861,1048576,95.3429227566,524288,2909,true,1,0.0109979426861 +bf16,16,64,16,64,16384,baseline_transform,0,0,100,68.1483411789,4194304,61.5466778419,2097152,2909,true,1,0.0681483411789 +bf16,16,64,16,64,16384,control_optimized_transform,0,1,100,73.8925743103,4194304,56.7621853637,2097152,2909,true,1,0.0738925743103 +bf16,16,64,16,64,16384,optimized_transform,0,2,100,73.9419412613,4194304,56.724288387,2097152,2909,true,1,0.0739419412613 +bf16,16,64,16,64,16384,control_shuffle32_transform,0,3,100,21.2016010284,4194304,197.82958817,2097152,2909,true,1,0.0212016010284 +bf16,16,64,16,64,16384,shuffle32_transform,0,4,100,21.2873148918,4194304,197.03302278,2097152,2909,true,1,0.0212873148918 +bf16,16,64,16,64,16384,baseline_split,0,5,100,1548.32000732,6881280,4.44435256759,2097152,2909,true,1,1.54832000732 +bf16,16,64,16,64,16384,control_optimized_split,0,6,100,1360.96511841,6881280,5.05617661094,2097152,2909,true,1,1.36096511841 +bf16,16,64,16,64,16384,optimized_split,0,7,100,576.579208374,6881280,11.9346655239,2097152,2909,true,1,0.576579208374 +bf16,16,64,16,64,16384,control_shuffle32_split,0,8,100,852.029724121,6881280,8.07633795534,2097152,2909,true,1,0.852029724121 +bf16,16,64,16,64,16384,shuffle32_split,0,9,100,228.71131897,6881280,30.0871860256,2097152,2909,true,1,0.22871131897 +bf16,16,64,16,64,16384,baseline_fused,0,10,100,1525.94406128,2686976,1.76086140258,2097152,2909,true,1,1.52594406128 +bf16,16,64,16,64,16384,control_optimized_fused,0,11,100,1333.73123169,2686976,2.01463078629,2097152,2909,true,1,1.33373123169 +bf16,16,64,16,64,16384,optimized_fused,0,12,100,544.588546753,2686976,4.93395613261,2097152,2909,true,1,0.544588546753 +bf16,16,64,16,64,16384,control_shuffle32_fused,0,13,100,867.205047607,2686976,3.09843214983,2097152,2909,true,1,0.867205047607 +bf16,16,64,16,64,16384,shuffle32_fused,0,14,100,211.844100952,2686976,12.6837423743,2097152,2909,true,1,0.211844100952 +bf16,16,64,16,64,16384,control_optimized_transform,1,0,100,73.9193153381,4194304,56.7416510937,2097152,2909,true,1,0.0739193153381 +bf16,16,64,16,64,16384,optimized_transform,1,1,100,73.9508581161,4194304,56.7174486794,2097152,2909,true,1,0.0739508581161 +bf16,16,64,16,64,16384,control_shuffle32_transform,1,2,100,21.3478851318,4194304,196.473982041,2097152,2909,true,1,0.0213478851318 +bf16,16,64,16,64,16384,shuffle32_transform,1,3,100,21.2242293358,4194304,197.618671267,2097152,2909,true,1,0.0212242293358 +bf16,16,64,16,64,16384,baseline_split,1,4,100,1548.51715088,6881280,4.44378675179,2097152,2909,true,1,1.54851715088 +bf16,16,64,16,64,16384,control_optimized_split,1,5,100,1360.95046997,6881280,5.05623103253,2097152,2909,true,1,1.36095046997 +bf16,16,64,16,64,16384,optimized_split,1,6,100,576.602287292,6881280,11.9341878304,2097152,2909,true,1,0.576602287292 +bf16,16,64,16,64,16384,control_shuffle32_split,1,7,100,851.776428223,6881280,8.07873964575,2097152,2909,true,1,0.851776428223 +bf16,16,64,16,64,16384,shuffle32_split,1,8,100,228.688926697,6881280,30.0901320383,2097152,2909,true,1,0.228688926697 +bf16,16,64,16,64,16384,baseline_fused,1,9,100,1525.84655762,2686976,1.76097392401,2097152,2909,true,1,1.52584655762 +bf16,16,64,16,64,16384,control_optimized_fused,1,10,100,1333.6567688,2686976,2.01474327043,2097152,2909,true,1,1.3336567688 +bf16,16,64,16,64,16384,optimized_fused,1,11,100,544.557952881,2686976,4.93423332776,2097152,2909,true,1,0.544557952881 +bf16,16,64,16,64,16384,control_shuffle32_fused,1,12,100,867.053909302,2686976,3.09897224518,2097152,2909,true,1,0.867053909302 +bf16,16,64,16,64,16384,shuffle32_fused,1,13,100,211.941928864,2686976,12.6778878271,2097152,2909,true,1,0.211941928864 +bf16,16,64,16,64,16384,baseline_transform,1,14,100,68.1812572479,4194304,61.5169647686,2097152,2909,true,1,0.0681812572479 +bf16,16,64,16,64,16384,optimized_transform,2,0,100,73.9478874207,4194304,56.7197271795,2097152,2909,true,1,0.0739478874207 +bf16,16,64,16,64,16384,control_shuffle32_transform,2,1,100,21.370742321,4194304,196.263842266,2097152,2909,true,1,0.021370742321 +bf16,16,64,16,64,16384,shuffle32_transform,2,2,100,21.38651371,4194304,196.119108372,2097152,2909,true,1,0.02138651371 +bf16,16,64,16,64,16384,baseline_split,2,3,100,1548.33740234,6881280,4.44430263687,2097152,2909,true,1,1.54833740234 +bf16,16,64,16,64,16384,control_optimized_split,2,4,100,1361.11068726,6881280,5.05563586006,2097152,2909,true,1,1.36111068726 +bf16,16,64,16,64,16384,optimized_split,2,5,100,576.662635803,6881280,11.9329389018,2097152,2909,true,1,0.576662635803 +bf16,16,64,16,64,16384,control_shuffle32_split,2,6,100,851.841125488,6881280,8.07812606612,2097152,2909,true,1,0.851841125488 +bf16,16,64,16,64,16384,shuffle32_split,2,7,100,228.698043823,6881280,30.0889324848,2097152,2909,true,1,0.228698043823 +bf16,16,64,16,64,16384,baseline_fused,2,8,100,1525.92224121,2686976,1.76088658218,2097152,2909,true,1,1.52592224121 +bf16,16,64,16,64,16384,control_optimized_fused,2,9,100,1333.78631592,2686976,2.01454758377,2097152,2909,true,1,1.33378631592 +bf16,16,64,16,64,16384,optimized_fused,2,10,100,544.633369446,2686976,4.93355007376,2097152,2909,true,1,0.544633369446 +bf16,16,64,16,64,16384,control_shuffle32_fused,2,11,100,867.005310059,2686976,3.09914595542,2097152,2909,true,1,0.867005310059 +bf16,16,64,16,64,16384,shuffle32_fused,2,12,100,211.953144073,2686976,12.6772169941,2097152,2909,true,1,0.211953144073 +bf16,16,64,16,64,16384,baseline_transform,2,13,100,68.1206846237,4194304,61.5716653931,2097152,2909,true,1,0.0681206846237 +bf16,16,64,16,64,16384,control_optimized_transform,2,14,100,73.8946294785,4194304,56.7606066855,2097152,2909,true,1,0.0738946294785 +bf16,16,64,16,64,16384,control_shuffle32_transform,3,0,100,21.3615989685,4194304,196.347848594,2097152,2909,true,1,0.0213615989685 +bf16,16,64,16,64,16384,shuffle32_transform,3,1,100,21.3083434105,4194304,196.838577228,2097152,2909,true,1,0.0213083434105 +bf16,16,64,16,64,16384,baseline_split,3,2,100,1548.48724365,6881280,4.44387257836,2097152,2909,true,1,1.54848724365 +bf16,16,64,16,64,16384,control_optimized_split,3,3,100,1360.98373413,6881280,5.05610745186,2097152,2909,true,1,1.36098373413 +bf16,16,64,16,64,16384,optimized_split,3,4,100,576.566390991,6881280,11.934930838,2097152,2909,true,1,0.576566390991 +bf16,16,64,16,64,16384,control_shuffle32_split,3,5,100,851.723175049,6881280,8.07924476119,2097152,2909,true,1,0.851723175049 +bf16,16,64,16,64,16384,shuffle32_split,3,6,100,228.782157898,6881280,30.0778699844,2097152,2909,true,1,0.228782157898 +bf16,16,64,16,64,16384,baseline_fused,3,7,100,1525.76797485,2686976,1.76106462076,2097152,2909,true,1,1.52576797485 +bf16,16,64,16,64,16384,control_optimized_fused,3,8,100,1333.55926514,2686976,2.0148905791,2097152,2909,true,1,1.33355926514 +bf16,16,64,16,64,16384,optimized_fused,3,9,100,544.601364136,2686976,4.93384001023,2097152,2909,true,1,0.544601364136 +bf16,16,64,16,64,16384,control_shuffle32_fused,3,10,100,867.093963623,2686976,3.09882909203,2097152,2909,true,1,0.867093963623 +bf16,16,64,16,64,16384,shuffle32_fused,3,11,100,211.986503601,2686976,12.6752220276,2097152,2909,true,1,0.211986503601 +bf16,16,64,16,64,16384,baseline_transform,3,12,100,68.2086849213,4194304,61.4922279302,2097152,2909,true,1,0.0682086849213 +bf16,16,64,16,64,16384,control_optimized_transform,3,13,100,73.8610267639,4194304,56.7864296472,2097152,2909,true,1,0.0738610267639 +bf16,16,64,16,64,16384,optimized_transform,3,14,100,73.9442300797,4194304,56.7225325828,2097152,2909,true,1,0.0739442300797 +bf16,16,64,16,64,16384,shuffle32_transform,4,0,100,21.3129138947,4194304,196.796365843,2097152,2909,true,1,0.0213129138947 +bf16,16,64,16,64,16384,baseline_split,4,1,100,1548.37249756,6881280,4.44420190287,2097152,2909,true,1,1.54837249756 +bf16,16,64,16,64,16384,control_optimized_split,4,2,100,1361.13937378,6881280,5.05552931063,2097152,2909,true,1,1.36113937378 +bf16,16,64,16,64,16384,optimized_split,4,3,100,576.482048035,6881280,11.9366769936,2097152,2909,true,1,0.576482048035 +bf16,16,64,16,64,16384,control_shuffle32_split,4,4,100,851.804733276,6881280,8.07847119319,2097152,2909,true,1,0.851804733276 +bf16,16,64,16,64,16384,shuffle32_split,4,5,100,228.641376495,6881280,30.0963898376,2097152,2909,true,1,0.228641376495 +bf16,16,64,16,64,16384,baseline_fused,4,6,100,1525.75057983,2686976,1.76108469858,2097152,2909,true,1,1.52575057983 +bf16,16,64,16,64,16384,control_optimized_fused,4,7,100,1333.62854004,2686976,2.01478591627,2097152,2909,true,1,1.33362854004 +bf16,16,64,16,64,16384,optimized_fused,4,8,100,544.597969055,2686976,4.93387076831,2097152,2909,true,1,0.544597969055 +bf16,16,64,16,64,16384,control_shuffle32_fused,4,9,100,867.041549683,2686976,3.09901642082,2097152,2909,true,1,0.867041549683 +bf16,16,64,16,64,16384,shuffle32_fused,4,10,100,211.806411743,2686976,12.685999342,2097152,2909,true,1,0.211806411743 +bf16,16,64,16,64,16384,baseline_transform,4,11,100,68.1273126602,4194304,61.5656751488,2097152,2909,true,1,0.0681273126602 +bf16,16,64,16,64,16384,control_optimized_transform,4,12,100,73.8404560089,4194304,56.8022494267,2097152,2909,true,1,0.0738404560089 +bf16,16,64,16,64,16384,optimized_transform,4,13,100,73.8925743103,4194304,56.7621853637,2097152,2909,true,1,0.0738925743103 +bf16,16,64,16,64,16384,control_shuffle32_transform,4,14,100,21.2235426903,4194304,197.625064826,2097152,2909,true,1,0.0212235426903 +bf16,1,1,1,128,1,baseline_transform,0,0,100,9.24045681953,512,0.0554085160506,256,2909,true,1,0.00924045681953 +bf16,1,1,1,128,1,control_optimized_transform,0,1,100,9.57782864571,512,0.0534567926551,256,2909,true,1,0.00957782864571 +bf16,1,1,1,128,1,optimized_transform,0,2,100,9.61462855339,512,0.0532521872433,256,2909,true,1,0.00961462855339 +bf16,1,1,1,128,1,control_shuffle32_transform,0,3,100,9.12959992886,512,0.056081318348,256,2909,true,1,0.00912959992886 +bf16,1,1,1,128,1,shuffle32_transform,0,4,100,9.23565685749,512,0.0554373130033,256,2909,true,1,0.00923565685749 +bf16,1,1,1,128,1,baseline_split,0,5,100,79.058508873,836,0.0105744468485,256,2909,true,1,0.079058508873 +bf16,1,1,1,128,1,control_optimized_split,0,6,100,63.4836578369,836,0.0131687433977,256,2909,true,1,0.0634836578369 +bf16,1,1,1,128,1,optimized_split,0,7,100,33.8793158531,836,0.0246758229601,256,2909,true,1,0.0338793158531 +bf16,1,1,1,128,1,control_shuffle32_split,0,8,100,161.873836517,836,0.00516451588463,256,2909,true,1,0.161873836517 +bf16,1,1,1,128,1,shuffle32_split,0,9,100,45.4665184021,836,0.0183871567338,256,2909,true,1,0.0454665184021 +bf16,1,1,1,128,1,baseline_fused,0,10,100,74.5270872116,324,0.00434741262704,256,2909,true,1,0.0745270872116 +bf16,1,1,1,128,1,control_optimized_fused,0,11,100,58.1846857071,324,0.00556847555439,256,2909,true,1,0.0581846857071 +bf16,1,1,1,128,1,optimized_fused,0,12,100,28.9622855186,324,0.0111869624306,256,2909,true,1,0.0289622855186 +bf16,1,1,1,128,1,control_shuffle32_fused,0,13,100,157.178506851,324,0.00206135054016,256,2909,true,1,0.157178506851 +bf16,1,1,1,128,1,shuffle32_fused,0,14,100,40.8304023743,324,0.00793526345957,256,2909,true,1,0.0408304023743 +bf16,1,1,1,128,1,control_optimized_transform,1,0,100,9.62674260139,512,0.0531851760455,256,2909,true,1,0.00962674260139 +bf16,1,1,1,128,1,optimized_transform,1,1,100,9.60822880268,512,0.0532876569152,256,2909,true,1,0.00960822880268 +bf16,1,1,1,128,1,control_shuffle32_transform,1,2,100,9.11245703697,512,0.0561868218333,256,2909,true,1,0.00911245703697 +bf16,1,1,1,128,1,shuffle32_transform,1,3,100,9.20594274998,512,0.055616248537,256,2909,true,1,0.00920594274998 +bf16,1,1,1,128,1,baseline_split,1,4,100,79.0781736374,836,0.010571817248,256,2909,true,1,0.0790781736374 +bf16,1,1,1,128,1,control_optimized_split,1,5,100,63.5021686554,836,0.0131649047222,256,2909,true,1,0.0635021686554 +bf16,1,1,1,128,1,optimized_split,1,6,100,34.0034270287,836,0.0245857571737,256,2909,true,1,0.0340034270287 +bf16,1,1,1,128,1,control_shuffle32_split,1,7,100,161.924571991,836,0.00516289769811,256,2909,true,1,0.161924571991 +bf16,1,1,1,128,1,shuffle32_split,1,8,100,45.4509735107,836,0.0183934454078,256,2909,true,1,0.0454509735107 +bf16,1,1,1,128,1,baseline_fused,1,9,100,74.5693731308,324,0.00434494734764,256,2909,true,1,0.0745693731308 +bf16,1,1,1,128,1,control_optimized_fused,1,10,100,58.1732559204,324,0.00556956963941,256,2909,true,1,0.0581732559204 +bf16,1,1,1,128,1,optimized_fused,1,11,100,28.8774871826,324,0.0112198127888,256,2909,true,1,0.0288774871826 +bf16,1,1,1,128,1,control_shuffle32_fused,1,12,100,157.183761597,324,0.00206128162801,256,2909,true,1,0.157183761597 +bf16,1,1,1,128,1,shuffle32_fused,1,13,100,40.8317708969,324,0.0079349975003,256,2909,true,1,0.0408317708969 +bf16,1,1,1,128,1,baseline_transform,1,14,100,9.2699432373,512,0.0552322691621,256,2909,true,1,0.0092699432373 +bf16,1,1,1,128,1,optimized_transform,2,0,100,9.64548528194,512,0.0530818289629,256,2909,true,1,0.00964548528194 +bf16,1,1,1,128,1,control_shuffle32_transform,2,1,100,9.0848004818,512,0.0563578695015,256,2909,true,1,0.0090848004818 +bf16,1,1,1,128,1,shuffle32_transform,2,2,100,9.08411443233,512,0.0563621257541,256,2909,true,1,0.00908411443233 +bf16,1,1,1,128,1,baseline_split,2,3,100,79.0432024002,836,0.0105764945576,256,2909,true,1,0.0790432024002 +bf16,1,1,1,128,1,control_optimized_split,2,4,100,63.5067415237,836,0.0131639567697,256,2909,true,1,0.0635067415237 +bf16,1,1,1,128,1,optimized_split,2,5,100,33.8662862778,836,0.0246853166345,256,2909,true,1,0.0338662862778 +bf16,1,1,1,128,1,control_shuffle32_split,2,6,100,162.029018402,836,0.00515956961441,256,2909,true,1,0.162029018402 +bf16,1,1,1,128,1,shuffle32_split,2,7,100,45.4217147827,836,0.0184052936794,256,2909,true,1,0.0454217147827 +bf16,1,1,1,128,1,baseline_fused,2,8,100,74.5689105988,324,0.00434497429825,256,2909,true,1,0.0745689105988 +bf16,1,1,1,128,1,control_optimized_fused,2,9,100,58.1734848022,324,0.00556954772611,256,2909,true,1,0.0581734848022 +bf16,1,1,1,128,1,optimized_fused,2,10,100,28.8907432556,324,0.0112146647503,256,2909,true,1,0.0288907432556 +bf16,1,1,1,128,1,control_shuffle32_fused,2,11,100,157.182626724,324,0.00206129651064,256,2909,true,1,0.157182626724 +bf16,1,1,1,128,1,shuffle32_fused,2,12,100,40.8795452118,324,0.00792572418116,256,2909,true,1,0.0408795452118 +bf16,1,1,1,128,1,baseline_transform,2,13,100,9.2587429285,512,0.0552990836827,256,2909,true,1,0.0092587429285 +bf16,1,1,1,128,1,control_optimized_transform,2,14,100,9.58971440792,512,0.0533905368003,256,2909,true,1,0.00958971440792 +bf16,1,1,1,128,1,control_shuffle32_transform,3,0,100,9.15291428566,512,0.0559384676859,256,2909,true,1,0.00915291428566 +bf16,1,1,1,128,1,shuffle32_transform,3,1,100,9.07337129116,512,0.0564288601855,256,2909,true,1,0.00907337129116 +bf16,1,1,1,128,1,baseline_split,3,2,100,79.0628623962,836,0.0105738645764,256,2909,true,1,0.0790628623962 +bf16,1,1,1,128,1,control_optimized_split,3,3,100,63.5001134872,836,0.0131653308016,256,2909,true,1,0.0635001134872 +bf16,1,1,1,128,1,optimized_split,3,4,100,33.9065146446,836,0.0246560287532,256,2909,true,1,0.0339065146446 +bf16,1,1,1,128,1,control_shuffle32_split,3,5,100,161.250743866,836,0.00518447220743,256,2909,true,1,0.161250743866 +bf16,1,1,1,128,1,shuffle32_split,3,6,100,45.4224014282,836,0.0184050154486,256,2909,true,1,0.0454224014282 +bf16,1,1,1,128,1,baseline_fused,3,7,100,74.5732593536,324,0.00434472092018,256,2909,true,1,0.0745732593536 +bf16,1,1,1,128,1,control_optimized_fused,3,8,100,58.1773710251,324,0.00556917568276,256,2909,true,1,0.0581773710251 +bf16,1,1,1,128,1,optimized_fused,3,9,100,28.9007997513,324,0.0112107624283,256,2909,true,1,0.0289007997513 +bf16,1,1,1,128,1,control_shuffle32_fused,3,10,100,157.176456451,324,0.00206137743091,256,2909,true,1,0.157176456451 +bf16,1,1,1,128,1,shuffle32_fused,3,11,100,40.8820581436,324,0.00792523700401,256,2909,true,1,0.0408820581436 +bf16,1,1,1,128,1,baseline_transform,3,12,100,9.29394245148,512,0.0550896460434,256,2909,true,1,0.00929394245148 +bf16,1,1,1,128,1,control_optimized_transform,3,13,100,9.6921145916,512,0.0528264492914,256,2909,true,1,0.0096921145916 +bf16,1,1,1,128,1,optimized_transform,3,14,100,9.74720001221,512,0.052527905384,256,2909,true,1,0.00974720001221 +bf16,1,1,1,128,1,shuffle32_transform,4,0,100,9.1942858696,512,0.0556867610233,256,2909,true,1,0.0091942858696 +bf16,1,1,1,128,1,baseline_split,4,1,100,79.052567482,836,0.0105752415972,256,2909,true,1,0.079052567482 +bf16,1,1,1,128,1,control_optimized_split,4,2,100,63.4848022461,836,0.0131685060112,256,2909,true,1,0.0634848022461 +bf16,1,1,1,128,1,optimized_split,4,3,100,34.069943428,836,0.0245377572101,256,2909,true,1,0.034069943428 +bf16,1,1,1,128,1,control_shuffle32_split,4,4,100,161.294174194,836,0.00518307622811,256,2909,true,1,0.161294174194 +bf16,1,1,1,128,1,shuffle32_split,4,5,100,45.4553127289,836,0.0183916895476,256,2909,true,1,0.0454553127289 +bf16,1,1,1,128,1,baseline_fused,4,6,100,74.5753145218,324,0.0043446011871,256,2909,true,1,0.0745753145218 +bf16,1,1,1,128,1,control_optimized_fused,4,7,100,58.1746292114,324,0.00556943816217,256,2909,true,1,0.0581746292114 +bf16,1,1,1,128,1,optimized_fused,4,8,100,28.9229726791,324,0.0112021680342,256,2909,true,1,0.0289229726791 +bf16,1,1,1,128,1,control_shuffle32_fused,4,9,100,157.170057297,324,0.00206146135958,256,2909,true,1,0.157170057297 +bf16,1,1,1,128,1,shuffle32_fused,4,10,100,40.8966827393,324,0.00792240295052,256,2909,true,1,0.0408966827393 +bf16,1,1,1,128,1,baseline_transform,4,11,100,9.34377133846,512,0.0547958614839,256,2909,true,1,0.00934377133846 +bf16,1,1,1,128,1,control_optimized_transform,4,12,100,9.68502819538,512,0.0528651016467,256,2909,true,1,0.00968502819538 +bf16,1,1,1,128,1,optimized_transform,4,13,100,9.5526856184,512,0.053597492941,256,2909,true,1,0.0095526856184 +bf16,1,1,1,128,1,control_shuffle32_transform,4,14,100,9.14148569107,512,0.0560084014024,256,2909,true,1,0.00914148569107 +bf16,1,17,1,128,17,baseline_transform,0,0,100,9.68159973621,8704,0.899024979048,4352,2909,true,1,0.00968159973621 +bf16,1,17,1,128,17,control_optimized_transform,0,1,100,9.81737136841,8704,0.886591702949,4352,2909,true,1,0.00981737136841 +bf16,1,17,1,128,17,optimized_transform,0,2,100,9.89920020103,8704,0.879262952889,4352,2909,true,1,0.00989920020103 +bf16,1,17,1,128,17,control_shuffle32_transform,0,3,100,9.19931471348,8704,0.94615743358,4352,2909,true,1,0.00919931471348 +bf16,1,17,1,128,17,shuffle32_transform,0,4,100,9.18285667896,8704,0.947853190385,4352,2909,true,1,0.00918285667896 +bf16,1,17,1,128,17,baseline_split,0,5,100,80.4283428192,14212,0.176703877039,4352,2909,true,1,0.0804283428192 +bf16,1,17,1,128,17,control_optimized_split,0,6,100,64.6068572998,14212,0.219976649445,4352,2909,true,1,0.0646068572998 +bf16,1,17,1,128,17,optimized_split,0,7,100,34.7590875626,14212,0.408871492223,4352,2909,true,1,0.0347590875626 +bf16,1,17,1,128,17,control_shuffle32_split,0,8,100,163.845710754,14212,0.086740140676,4352,2909,true,1,0.163845710754 +bf16,1,17,1,128,17,shuffle32_split,0,9,100,45.4948568344,14212,0.312386959513,4352,2909,true,1,0.0454948568344 +bf16,1,17,1,128,17,baseline_fused,0,10,100,75.6493711472,5508,0.0728095940056,4352,2909,true,1,0.0756493711472 +bf16,1,17,1,128,17,control_optimized_fused,0,11,100,59.1979408264,5508,0.0930437769136,4352,2909,true,1,0.0591979408264 +bf16,1,17,1,128,17,optimized_fused,0,12,100,29.2326855659,5508,0.188419226402,4352,2909,true,1,0.0292326855659 +bf16,1,17,1,128,17,control_shuffle32_fused,0,13,100,158.858747482,5508,0.0346723116435,4352,2909,true,1,0.158858747482 +bf16,1,17,1,128,17,shuffle32_fused,0,14,100,40.4608011246,5508,0.136131758317,4352,2909,true,1,0.0404608011246 +bf16,1,17,1,128,17,control_optimized_transform,1,0,100,9.84914243221,8704,0.883731762426,4352,2909,true,1,0.00984914243221 +bf16,1,17,1,128,17,optimized_transform,1,1,100,10.0054860115,8704,0.869922759373,4352,2909,true,1,0.0100054860115 +bf16,1,17,1,128,17,control_shuffle32_transform,1,2,100,9.24022853374,8704,0.941968044212,4352,2909,true,1,0.00924022853374 +bf16,1,17,1,128,17,shuffle32_transform,1,3,100,9.27977144718,8704,0.93795413492,4352,2909,true,1,0.00927977144718 +bf16,1,17,1,128,17,baseline_split,1,4,100,80.4230880737,14212,0.176715422653,4352,2909,true,1,0.0804230880737 +bf16,1,17,1,128,17,control_optimized_split,1,5,100,64.6315431595,14212,0.219892629903,4352,2909,true,1,0.0646315431595 +bf16,1,17,1,128,17,optimized_split,1,6,100,34.7149729729,14212,0.409391072005,4352,2909,true,1,0.0347149729729 +bf16,1,17,1,128,17,control_shuffle32_split,1,7,100,163.313598633,14212,0.0870227593965,4352,2909,true,1,0.163313598633 +bf16,1,17,1,128,17,shuffle32_split,1,8,100,45.4978275299,14212,0.312366562792,4352,2909,true,1,0.0454978275299 +bf16,1,17,1,128,17,baseline_fused,1,9,100,75.6464004517,5508,0.0728124532974,4352,2909,true,1,0.0756464004517 +bf16,1,17,1,128,17,control_optimized_fused,1,10,100,59.1458272934,5508,0.0931257580129,4352,2909,true,1,0.0591458272934 +bf16,1,17,1,128,17,optimized_fused,1,11,100,29.3204569817,5508,0.187855189414,4352,2909,true,1,0.0293204569817 +bf16,1,17,1,128,17,control_shuffle32_fused,1,12,100,158.856010437,5508,0.0346729090379,4352,2909,true,1,0.158856010437 +bf16,1,17,1,128,17,shuffle32_fused,1,13,100,40.4493713379,5508,0.136170225094,4352,2909,true,1,0.0404493713379 +bf16,1,17,1,128,17,baseline_transform,1,14,100,9.58445727825,8704,0.908136970859,4352,2909,true,1,0.00958445727825 +bf16,1,17,1,128,17,optimized_transform,2,0,100,9.80297088623,8704,0.88789409874,4352,2909,true,1,0.00980297088623 +bf16,1,17,1,128,17,control_shuffle32_transform,2,1,100,9.17257130146,8704,0.948916036075,4352,2909,true,1,0.00917257130146 +bf16,1,17,1,128,17,shuffle32_transform,2,2,100,9.29211437702,8704,0.93670822881,4352,2909,true,1,0.00929211437702 +bf16,1,17,1,128,17,baseline_split,2,3,100,80.4144001007,14212,0.17673451499,4352,2909,true,1,0.0804144001007 +bf16,1,17,1,128,17,control_optimized_split,2,4,100,64.6084594727,14212,0.219971194423,4352,2909,true,1,0.0646084594727 +bf16,1,17,1,128,17,optimized_split,2,5,100,34.6352005005,14212,0.410333989543,4352,2909,true,1,0.0346352005005 +bf16,1,17,1,128,17,control_shuffle32_split,2,6,100,163.913593292,14212,0.0867042184516,4352,2909,true,1,0.163913593292 +bf16,1,17,1,128,17,shuffle32_split,2,7,100,45.4383993149,14212,0.312775102431,4352,2909,true,1,0.0454383993149 +bf16,1,17,1,128,17,baseline_fused,2,8,100,75.647315979,5508,0.0728115720792,4352,2909,true,1,0.075647315979 +bf16,1,17,1,128,17,control_optimized_fused,2,9,100,59.2016029358,5508,0.0930380213856,4352,2909,true,1,0.0592016029358 +bf16,1,17,1,128,17,optimized_fused,2,10,100,29.3138289452,5508,0.187897664625,4352,2909,true,1,0.0293138289452 +bf16,1,17,1,128,17,control_shuffle32_fused,2,11,100,158.865375519,5508,0.0346708650769,4352,2909,true,1,0.158865375519 +bf16,1,17,1,128,17,shuffle32_fused,2,12,100,40.4555463791,5508,0.136149440385,4352,2909,true,1,0.0404555463791 +bf16,1,17,1,128,17,baseline_transform,2,13,100,9.59314286709,8704,0.907314747689,4352,2909,true,1,0.00959314286709 +bf16,1,17,1,128,17,control_optimized_transform,2,14,100,9.79108631611,8704,0.888971838159,4352,2909,true,1,0.00979108631611 +bf16,1,17,1,128,17,control_shuffle32_transform,3,0,100,9.18171405792,8704,0.947971146247,4352,2909,true,1,0.00918171405792 +bf16,1,17,1,128,17,shuffle32_transform,3,1,100,9.20502901077,8704,0.945570078032,4352,2909,true,1,0.00920502901077 +bf16,1,17,1,128,17,baseline_split,3,2,100,80.4326915741,14212,0.17669432319,4352,2909,true,1,0.0804326915741 +bf16,1,17,1,128,17,control_optimized_split,3,3,100,64.6445703506,14212,0.219848317081,4352,2909,true,1,0.0646445703506 +bf16,1,17,1,128,17,optimized_split,3,4,100,34.9065160751,14212,0.407144613613,4352,2909,true,1,0.0349065160751 +bf16,1,17,1,128,17,control_shuffle32_split,3,5,100,163.280906677,14212,0.0870401830147,4352,2909,true,1,0.163280906677 +bf16,1,17,1,128,17,shuffle32_split,3,6,100,45.4617166519,14212,0.312614679925,4352,2909,true,1,0.0454617166519 +bf16,1,17,1,128,17,baseline_fused,3,7,100,75.641374588,5508,0.0728172911981,4352,2909,true,1,0.075641374588 +bf16,1,17,1,128,17,control_optimized_fused,3,8,100,59.1901731491,5508,0.0930559872857,4352,2909,true,1,0.0591901731491 +bf16,1,17,1,128,17,optimized_fused,3,9,100,29.3174862862,5508,0.18787422449,4352,2909,true,1,0.0293174862862 +bf16,1,17,1,128,17,control_shuffle32_fused,3,10,100,158.816461563,5508,0.0346815433727,4352,2909,true,1,0.158816461563 +bf16,1,17,1,128,17,shuffle32_fused,3,11,100,40.4617166519,5508,0.136128678063,4352,2909,true,1,0.0404617166519 +bf16,1,17,1,128,17,baseline_transform,3,12,100,9.67131376266,8704,0.899981141507,4352,2909,true,1,0.00967131376266 +bf16,1,17,1,128,17,control_optimized_transform,3,13,100,9.92251455784,8704,0.877196999739,4352,2909,true,1,0.00992251455784 +bf16,1,17,1,128,17,optimized_transform,3,14,100,9.97439980507,8704,0.872633959948,4352,2909,true,1,0.00997439980507 +bf16,1,17,1,128,17,shuffle32_transform,4,0,100,9.24045681953,8704,0.94194477286,4352,2909,true,1,0.00924045681953 +bf16,1,17,1,128,17,baseline_split,4,1,100,80.4258346558,14212,0.176709387734,4352,2909,true,1,0.0804258346558 +bf16,1,17,1,128,17,control_optimized_split,4,2,100,64.635887146,14212,0.21987785157,4352,2909,true,1,0.064635887146 +bf16,1,17,1,128,17,optimized_split,4,3,100,34.6749711037,14212,0.409863355257,4352,2909,true,1,0.0346749711037 +bf16,1,17,1,128,17,control_shuffle32_split,4,4,100,163.282279968,14212,0.0870394509604,4352,2909,true,1,0.163282279968 +bf16,1,17,1,128,17,shuffle32_split,4,5,100,45.4594278336,14212,0.312630419636,4352,2909,true,1,0.0454594278336 +bf16,1,17,1,128,17,baseline_fused,4,6,100,75.6464004517,5508,0.0728124532974,4352,2909,true,1,0.0756464004517 +bf16,1,17,1,128,17,control_optimized_fused,4,7,100,59.1924571991,5508,0.093052396549,4352,2909,true,1,0.0591924571991 +bf16,1,17,1,128,17,optimized_fused,4,8,100,29.2580580711,5508,0.188255829782,4352,2909,true,1,0.0292580580711 +bf16,1,17,1,128,17,control_shuffle32_fused,4,9,100,158.8626194,5508,0.0346714665842,4352,2909,true,1,0.1588626194 +bf16,1,17,1,128,17,shuffle32_fused,4,10,100,40.4630851746,5508,0.136124073986,4352,2909,true,1,0.0404630851746 +bf16,1,17,1,128,17,baseline_transform,4,11,100,9.57828581333,8704,0.908722100137,4352,2909,true,1,0.00957828581333 +bf16,1,17,1,128,17,control_optimized_transform,4,12,100,9.79680001736,8704,0.888453370956,4352,2909,true,1,0.00979680001736 +bf16,1,17,1,128,17,optimized_transform,4,13,100,9.81714367867,8704,0.886612265736,4352,2909,true,1,0.00981714367867 +bf16,1,17,1,128,17,control_shuffle32_transform,4,14,100,9.22331392765,8704,0.943695516414,4352,2909,true,1,0.00922331392765 +bf16,1,257,1,128,257,baseline_transform,0,0,100,9.91062819958,131584,13.2770594709,65792,2909,true,1,0.00991062819958 +bf16,1,257,1,128,257,control_optimized_transform,0,1,100,9.95611429214,131584,13.2164011118,65792,2909,true,1,0.00995611429214 +bf16,1,257,1,128,257,optimized_transform,0,2,100,10.0107431412,131584,13.1442789156,65792,2909,true,1,0.0100107431412 +bf16,1,257,1,128,257,control_shuffle32_transform,0,3,100,10.5108571053,131584,12.518864892,65792,2909,true,1,0.0105108571053 +bf16,1,257,1,128,257,shuffle32_transform,0,4,100,10.5124568939,131584,12.5169597676,65792,2909,true,1,0.0105124568939 +bf16,1,257,1,128,257,baseline_split,0,5,100,81.8198871613,214852,2.62591415674,65792,2909,true,1,0.0818198871613 +bf16,1,257,1,128,257,control_optimized_split,0,6,100,65.8777141571,214852,3.26137606244,65792,2909,true,1,0.0658777141571 +bf16,1,257,1,128,257,optimized_split,0,7,100,35.1282286644,214852,6.1162207196,65792,2909,true,1,0.0351282286644 +bf16,1,257,1,128,257,control_shuffle32_split,0,8,100,176.926403046,214852,1.21435803985,65792,2909,true,1,0.176926403046 +bf16,1,257,1,128,257,shuffle32_split,0,9,100,51.5844583511,214852,4.16505294167,65792,2909,true,1,0.0515844583511 +bf16,1,257,1,128,257,baseline_fused,0,10,100,76.7750835419,83268,1.08457062055,65792,2909,true,1,0.0767750835419 +bf16,1,257,1,128,257,control_optimized_fused,0,11,100,60.2084589005,83268,1.38299503958,65792,2909,true,1,0.0602084589005 +bf16,1,257,1,128,257,optimized_fused,0,12,100,29.5611429214,83268,2.81680583938,65792,2909,true,1,0.0295611429214 +bf16,1,257,1,128,257,control_shuffle32_fused,0,13,100,171.163883209,83268,0.48648113398,65792,2909,true,1,0.171163883209 +bf16,1,257,1,128,257,shuffle32_fused,0,14,100,44.8621702194,83268,1.85608497299,65792,2909,true,1,0.0448621702194 +bf16,1,257,1,128,257,control_optimized_transform,1,0,100,10.0144004822,131584,13.1394785174,65792,2909,true,1,0.0100144004822 +bf16,1,257,1,128,257,optimized_transform,1,1,100,9.97165679932,131584,13.1958011239,65792,2909,true,1,0.00997165679932 +bf16,1,257,1,128,257,control_shuffle32_transform,1,2,100,10.5860579014,131584,12.4299339023,65792,2909,true,1,0.0105860579014 +bf16,1,257,1,128,257,shuffle32_transform,1,3,100,10.6438851357,131584,12.3624032318,65792,2909,true,1,0.0106438851357 +bf16,1,257,1,128,257,baseline_split,1,4,100,81.8349742889,214852,2.62543004219,65792,2909,true,1,0.0818349742889 +bf16,1,257,1,128,257,control_optimized_split,1,5,100,65.7087993622,214852,3.26975994213,65792,2909,true,1,0.0657087993622 +bf16,1,257,1,128,257,optimized_split,1,6,100,35.1549696922,214852,6.11156834669,65792,2909,true,1,0.0351549696922 +bf16,1,257,1,128,257,control_shuffle32_split,1,7,100,177.585372925,214852,1.20985189524,65792,2909,true,1,0.177585372925 +bf16,1,257,1,128,257,shuffle32_split,1,8,100,51.6283416748,214852,4.16151270853,65792,2909,true,1,0.0516283416748 +bf16,1,257,1,128,257,baseline_fused,1,9,100,76.7764568329,83268,1.08455122097,65792,2909,true,1,0.0767764568329 +bf16,1,257,1,128,257,control_optimized_fused,1,10,100,60.2173709869,83268,1.38279035825,65792,2909,true,1,0.0602173709869 +bf16,1,257,1,128,257,optimized_fused,1,11,100,29.5174860954,83268,2.82097193951,65792,2909,true,1,0.0295174860954 +bf16,1,257,1,128,257,control_shuffle32_fused,1,12,100,171.217823029,83268,0.486327874792,65792,2909,true,1,0.171217823029 +bf16,1,257,1,128,257,shuffle32_fused,1,13,100,44.8626279831,83268,1.8560660341,65792,2909,true,1,0.0448626279831 +bf16,1,257,1,128,257,baseline_transform,1,14,100,9.92708623409,131584,13.2550475434,65792,2909,true,1,0.00992708623409 +bf16,1,257,1,128,257,optimized_transform,2,0,100,10.0818288326,131584,13.0516002785,65792,2909,true,1,0.0100818288326 +bf16,1,257,1,128,257,control_shuffle32_transform,2,1,100,10.5483436584,131584,12.4743755286,65792,2909,true,1,0.0105483436584 +bf16,1,257,1,128,257,shuffle32_transform,2,2,100,10.6004571915,131584,12.4130495151,65792,2909,true,1,0.0106004571915 +bf16,1,257,1,128,257,baseline_split,2,3,100,81.8130302429,214852,2.62613424001,65792,2909,true,1,0.0818130302429 +bf16,1,257,1,128,257,control_optimized_split,2,4,100,65.7974815369,214852,3.26535294333,65792,2909,true,1,0.0657974815369 +bf16,1,257,1,128,257,optimized_split,2,5,100,35.1302862167,214852,6.11586249752,65792,2909,true,1,0.0351302862167 +bf16,1,257,1,128,257,control_shuffle32_split,2,6,100,176.876583099,214852,1.21470008203,65792,2909,true,1,0.176876583099 +bf16,1,257,1,128,257,shuffle32_split,2,7,100,51.6495990753,214852,4.15979995676,65792,2909,true,1,0.0516495990753 +bf16,1,257,1,128,257,baseline_fused,2,8,100,76.7794275284,83268,1.08450925828,65792,2909,true,1,0.0767794275284 +bf16,1,257,1,128,257,control_optimized_fused,2,9,100,60.2139425278,83268,1.38286909152,65792,2909,true,1,0.0602139425278 +bf16,1,257,1,128,257,optimized_fused,2,10,100,29.4877696037,83268,2.82381479234,65792,2909,true,1,0.0294877696037 +bf16,1,257,1,128,257,control_shuffle32_fused,2,11,100,171.208686829,83268,0.48635382668,65792,2909,true,1,0.171208686829 +bf16,1,257,1,128,257,shuffle32_fused,2,12,100,44.8573684692,83268,1.85628365732,65792,2909,true,1,0.0448573684692 +bf16,1,257,1,128,257,baseline_transform,2,13,100,10.0281143188,131584,13.1215097691,65792,2909,true,1,0.0100281143188 +bf16,1,257,1,128,257,control_optimized_transform,2,14,100,10.0276577473,131584,13.1221072075,65792,2909,true,1,0.0100276577473 +bf16,1,257,1,128,257,control_shuffle32_transform,3,0,100,10.5981719494,131584,12.415726092,65792,2909,true,1,0.0105981719494 +bf16,1,257,1,128,257,shuffle32_transform,3,1,100,10.6454861164,131584,12.3605440429,65792,2909,true,1,0.0106454861164 +bf16,1,257,1,128,257,baseline_split,3,2,100,81.83177948,214852,2.62553254207,65792,2909,true,1,0.08183177948 +bf16,1,257,1,128,257,control_optimized_split,3,3,100,65.7883405685,214852,3.26580664816,65792,2909,true,1,0.0657883405685 +bf16,1,257,1,128,257,optimized_split,3,4,100,35.1494860649,214852,6.11252180482,65792,2909,true,1,0.0351494860649 +bf16,1,257,1,128,257,control_shuffle32_split,3,5,100,176.937160492,214852,1.21428420917,65792,2909,true,1,0.176937160492 +bf16,1,257,1,128,257,shuffle32_split,3,6,100,51.6242313385,214852,4.16184404938,65792,2909,true,1,0.0516242313385 +bf16,1,257,1,128,257,baseline_fused,3,7,100,76.7796564102,83268,1.08450602533,65792,2909,true,1,0.0767796564102 +bf16,1,257,1,128,257,control_optimized_fused,3,8,100,60.2178287506,83268,1.38277984656,65792,2909,true,1,0.0602178287506 +bf16,1,257,1,128,257,optimized_fused,3,9,100,29.5357704163,83268,2.81922559752,65792,2909,true,1,0.0295357704163 +bf16,1,257,1,128,257,control_shuffle32_fused,3,10,100,171.211433411,83268,0.486346024569,65792,2909,true,1,0.171211433411 +bf16,1,257,1,128,257,shuffle32_fused,3,11,100,44.8717737198,83268,1.85568773189,65792,2909,true,1,0.0448717737198 +bf16,1,257,1,128,257,baseline_transform,3,12,100,10.0228571892,131584,13.1283921856,65792,2909,true,1,0.0100228571892 +bf16,1,257,1,128,257,control_optimized_transform,3,13,100,10.0365710258,131584,13.1104537258,65792,2909,true,1,0.0100365710258 +bf16,1,257,1,128,257,optimized_transform,3,14,100,10.0532567501,131584,13.0886938701,65792,2909,true,1,0.0100532567501 +bf16,1,257,1,128,257,shuffle32_transform,4,0,100,10.6020569801,131584,12.4111764582,65792,2909,true,1,0.0106020569801 +bf16,1,257,1,128,257,baseline_split,4,1,100,81.8470859528,214852,2.62504153323,65792,2909,true,1,0.0818470859528 +bf16,1,257,1,128,257,control_optimized_split,4,2,100,65.7341718674,214852,3.26849785882,65792,2909,true,1,0.0657341718674 +bf16,1,257,1,128,257,optimized_split,4,3,100,35.0228571892,214852,6.13462227937,65792,2909,true,1,0.0350228571892 +bf16,1,257,1,128,257,control_shuffle32_split,4,4,100,176.944675446,214852,1.21423263774,65792,2909,true,1,0.176944675446 +bf16,1,257,1,128,257,shuffle32_split,4,5,100,51.6452598572,214852,4.16014946181,65792,2909,true,1,0.0516452598572 +bf16,1,257,1,128,257,baseline_fused,4,6,100,76.7682313919,83268,1.08466742675,65792,2909,true,1,0.0767682313919 +bf16,1,257,1,128,257,control_optimized_fused,4,7,100,60.2086877823,83268,1.38298978216,65792,2909,true,1,0.0602086877823 +bf16,1,257,1,128,257,optimized_fused,4,8,100,29.4427442551,83268,2.82813311418,65792,2909,true,1,0.0294427442551 +bf16,1,257,1,128,257,control_shuffle32_fused,4,9,100,171.210746765,83268,0.486347975073,65792,2909,true,1,0.171210746765 +bf16,1,257,1,128,257,shuffle32_fused,4,10,100,44.8482275009,83268,1.85666200517,65792,2909,true,1,0.0448482275009 +bf16,1,257,1,128,257,baseline_transform,4,11,100,9.95062887669,131584,13.2236868273,65792,2909,true,1,0.00995062887669 +bf16,1,257,1,128,257,control_optimized_transform,4,12,100,9.95862901211,131584,13.21306375,65792,2909,true,1,0.00995862901211 +bf16,1,257,1,128,257,optimized_transform,4,13,100,10.0004565716,131584,13.1577992523,65792,2909,true,1,0.0100004565716 +bf16,1,257,1,128,257,control_shuffle32_transform,4,14,100,10.6415998936,131584,12.3650580097,65792,2909,true,1,0.0106415998936 +bf16,4,64,16,128,4096,baseline_transform,0,0,100,25.9220576286,2097152,80.902219648,1048576,2909,true,1,0.0259220576286 +bf16,4,64,16,128,4096,control_optimized_transform,0,1,100,27.7593159676,2097152,75.5476828914,1048576,2909,true,1,0.0277593159676 +bf16,4,64,16,128,4096,optimized_transform,0,2,100,27.8018283844,2097152,75.4321611875,1048576,2909,true,1,0.0278018283844 +bf16,4,64,16,128,4096,control_shuffle32_transform,0,3,100,15.8230853081,2097152,132.537489318,1048576,2909,true,1,0.0158230853081 +bf16,4,64,16,128,4096,shuffle32_transform,0,4,100,15.7872009277,2097152,132.838747641,1048576,2909,true,1,0.0157872009277 +bf16,4,64,16,128,4096,baseline_split,0,5,100,525.766372681,3424256,6.5128851481,1048576,2909,true,1,0.525766372681 +bf16,4,64,16,128,4096,control_optimized_split,0,6,100,403.140106201,3424256,8.49396015759,1048576,2909,true,1,0.403140106201 +bf16,4,64,16,128,4096,optimized_split,0,7,100,168.953838348,3424256,20.2674057806,1048576,2909,true,1,0.168953838348 +bf16,4,64,16,128,4096,control_shuffle32_split,0,8,100,455.385360718,3424256,7.51946877388,1048576,2909,true,1,0.455385360718 +bf16,4,64,16,128,4096,shuffle32_split,0,9,100,105.741710663,3424256,32.3832097905,1048576,2909,true,1,0.105741710663 +bf16,4,64,16,128,4096,baseline_fused,0,10,100,520.202064514,1327104,2.55113174385,1048576,2909,true,1,0.520202064514 +bf16,4,64,16,128,4096,control_optimized_fused,0,11,100,391.287078857,1327104,3.39163767911,1048576,2909,true,1,0.391287078857 +bf16,4,64,16,128,4096,optimized_fused,0,12,100,154.197940826,1327104,8.60649625337,1048576,2909,true,1,0.154197940826 +bf16,4,64,16,128,4096,control_shuffle32_fused,0,13,100,435.999794006,1327104,3.04381795185,1048576,2909,true,1,0.435999794006 +bf16,4,64,16,128,4096,shuffle32_fused,0,14,100,92.8509712219,1327104,14.2928391867,1048576,2909,true,1,0.0928509712219 +bf16,4,64,16,128,4096,control_optimized_transform,1,0,100,27.8068566322,2097152,75.418520969,1048576,2909,true,1,0.0278068566322 +bf16,4,64,16,128,4096,optimized_transform,1,1,100,27.7810287476,2097152,75.488637194,1048576,2909,true,1,0.0277810287476 +bf16,4,64,16,128,4096,control_shuffle32_transform,1,2,100,15.8662855625,2097152,132.176620151,1048576,2909,true,1,0.0158662855625 +bf16,4,64,16,128,4096,shuffle32_transform,1,3,100,15.7611429691,2097152,133.058370456,1048576,2909,true,1,0.0157611429691 +bf16,4,64,16,128,4096,baseline_split,1,4,100,525.979881287,3424256,6.51024140243,1048576,2909,true,1,0.525979881287 +bf16,4,64,16,128,4096,control_optimized_split,1,5,100,401.798171997,3424256,8.52232846899,1048576,2909,true,1,0.401798171997 +bf16,4,64,16,128,4096,optimized_split,1,6,100,168.607559204,3424256,20.3090301299,1048576,2909,true,1,0.168607559204 +bf16,4,64,16,128,4096,control_shuffle32_split,1,7,100,453.416671753,3424256,7.55211754072,1048576,2909,true,1,0.453416671753 +bf16,4,64,16,128,4096,shuffle32_split,1,8,100,105.403432846,3424256,32.4871392472,1048576,2909,true,1,0.105403432846 +bf16,4,64,16,128,4096,baseline_fused,1,9,100,518.411216736,1327104,2.55994460991,1048576,2909,true,1,0.518411216736 +bf16,4,64,16,128,4096,control_optimized_fused,1,10,100,393.510398865,1327104,3.3724750447,1048576,2909,true,1,0.393510398865 +bf16,4,64,16,128,4096,optimized_fused,1,11,100,153.707885742,1327104,8.63393568646,1048576,2909,true,1,0.153707885742 +bf16,4,64,16,128,4096,control_shuffle32_fused,1,12,100,437.962989807,1327104,3.03017385233,1048576,2909,true,1,0.437962989807 +bf16,4,64,16,128,4096,shuffle32_fused,1,13,100,92.8916549683,1327104,14.2865793537,1048576,2909,true,1,0.0928916549683 +bf16,4,64,16,128,4096,baseline_transform,1,14,100,25.8795428276,2097152,81.0351254645,1048576,2909,true,1,0.0258795428276 +bf16,4,64,16,128,4096,optimized_transform,2,0,100,27.8116583824,2097152,75.4054997787,1048576,2909,true,1,0.0278116583824 +bf16,4,64,16,128,4096,control_shuffle32_transform,2,1,100,15.8148574829,2097152,132.606443167,1048576,2909,true,1,0.0158148574829 +bf16,4,64,16,128,4096,shuffle32_transform,2,2,100,15.7915425301,2097152,132.802226002,1048576,2909,true,1,0.0157915425301 +bf16,4,64,16,128,4096,baseline_split,2,3,100,523.634529114,3424256,6.5394006881,1048576,2909,true,1,0.523634529114 +bf16,4,64,16,128,4096,control_optimized_split,2,4,100,402.711334229,3424256,8.50300378697,1048576,2909,true,1,0.402711334229 +bf16,4,64,16,128,4096,optimized_split,2,5,100,168.410739899,3424256,20.3327650128,1048576,2909,true,1,0.168410739899 +bf16,4,64,16,128,4096,control_shuffle32_split,2,6,100,453.986740112,3424256,7.54263439314,1048576,2909,true,1,0.453986740112 +bf16,4,64,16,128,4096,shuffle32_split,2,7,100,105.914506912,3424256,32.3303775831,1048576,2909,true,1,0.105914506912 +bf16,4,64,16,128,4096,baseline_fused,2,8,100,519.189033508,1327104,2.55610945985,1048576,2909,true,1,0.519189033508 +bf16,4,64,16,128,4096,control_optimized_fused,2,9,100,392.20664978,1327104,3.38368561763,1048576,2909,true,1,0.39220664978 +bf16,4,64,16,128,4096,optimized_fused,2,10,100,153.480463028,1327104,8.64672919157,1048576,2909,true,1,0.153480463028 +bf16,4,64,16,128,4096,control_shuffle32_fused,2,11,100,432.963447571,1327104,3.0651640628,1048576,2909,true,1,0.432963447571 +bf16,4,64,16,128,4096,shuffle32_fused,2,12,100,92.8384017944,1327104,14.2947742997,1048576,2909,true,1,0.0928384017944 +bf16,4,64,16,128,4096,baseline_transform,2,13,100,25.9878849983,2097152,80.6972941482,1048576,2909,true,1,0.0259878849983 +bf16,4,64,16,128,4096,control_optimized_transform,2,14,100,27.7707433701,2097152,75.5165957229,1048576,2909,true,1,0.0277707433701 +bf16,4,64,16,128,4096,control_shuffle32_transform,3,0,100,15.7915425301,2097152,132.802226002,1048576,2909,true,1,0.0157915425301 +bf16,4,64,16,128,4096,shuffle32_transform,3,1,100,15.9113144875,2097152,131.802561105,1048576,2909,true,1,0.0159113144875 +bf16,4,64,16,128,4096,baseline_split,3,2,100,528.050079346,3424256,6.48471827567,1048576,2909,true,1,0.528050079346 +bf16,4,64,16,128,4096,control_optimized_split,3,3,100,402.768936157,3424256,8.50178773137,1048576,2909,true,1,0.402768936157 +bf16,4,64,16,128,4096,optimized_split,3,4,100,168.874511719,3424256,20.2769261338,1048576,2909,true,1,0.168874511719 +bf16,4,64,16,128,4096,control_shuffle32_split,3,5,100,452.794265747,3424256,7.56249859823,1048576,2909,true,1,0.452794265747 +bf16,4,64,16,128,4096,shuffle32_split,3,6,100,106.127319336,3424256,32.2655469056,1048576,2909,true,1,0.106127319336 +bf16,4,64,16,128,4096,baseline_fused,3,7,100,519.928207397,1327104,2.55247547857,1048576,2909,true,1,0.519928207397 +bf16,4,64,16,128,4096,control_optimized_fused,3,8,100,392.218971252,1327104,3.38357931989,1048576,2909,true,1,0.392218971252 +bf16,4,64,16,128,4096,optimized_fused,3,9,100,154.297370911,1327104,8.6009501793,1048576,2909,true,1,0.154297370911 +bf16,4,64,16,128,4096,control_shuffle32_fused,3,10,100,436.633834839,1327104,3.03939799006,1048576,2909,true,1,0.436633834839 +bf16,4,64,16,128,4096,shuffle32_fused,3,11,100,92.9337215424,1327104,14.2801125143,1048576,2909,true,1,0.0929337215424 +bf16,4,64,16,128,4096,baseline_transform,3,12,100,25.9149694443,2097152,80.9243477794,1048576,2909,true,1,0.0259149694443 +bf16,4,64,16,128,4096,control_optimized_transform,3,13,100,27.7709698677,2097152,75.515979816,1048576,2909,true,1,0.0277709698677 +bf16,4,64,16,128,4096,optimized_transform,3,14,100,27.8267431259,2097152,75.3646228202,1048576,2909,true,1,0.0278267431259 +bf16,4,64,16,128,4096,shuffle32_transform,4,0,100,15.8788561821,2097152,132.071981505,1048576,2909,true,1,0.0158788561821 +bf16,4,64,16,128,4096,baseline_split,4,1,100,527.385139465,3424256,6.49289436458,1048576,2909,true,1,0.527385139465 +bf16,4,64,16,128,4096,control_optimized_split,4,2,100,402.50377655,3424256,8.50738850042,1048576,2909,true,1,0.40250377655 +bf16,4,64,16,128,4096,optimized_split,4,3,100,168.50605011,3424256,20.3212644161,1048576,2909,true,1,0.16850605011 +bf16,4,64,16,128,4096,control_shuffle32_split,4,4,100,451.520690918,3424256,7.58382964253,1048576,2909,true,1,0.451520690918 +bf16,4,64,16,128,4096,shuffle32_split,4,5,100,105.831317902,3424256,32.355790969,1048576,2909,true,1,0.105831317902 +bf16,4,64,16,128,4096,baseline_fused,4,6,100,516.638145447,1327104,2.56873018707,1048576,2909,true,1,0.516638145447 +bf16,4,64,16,128,4096,control_optimized_fused,4,7,100,394.231796265,1327104,3.36630381561,1048576,2909,true,1,0.394231796265 +bf16,4,64,16,128,4096,optimized_fused,4,8,100,154.458742142,1327104,8.59196431098,1048576,2909,true,1,0.154458742142 +bf16,4,64,16,128,4096,control_shuffle32_fused,4,9,100,437.962989807,1327104,3.03017385233,1048576,2909,true,1,0.437962989807 +bf16,4,64,16,128,4096,shuffle32_fused,4,10,100,92.8324508667,1327104,14.2956906514,1048576,2909,true,1,0.0928324508667 +bf16,4,64,16,128,4096,baseline_transform,4,11,100,25.8884572983,2097152,81.0072217065,1048576,2909,true,1,0.0258884572983 +bf16,4,64,16,128,4096,control_optimized_transform,4,12,100,27.7136015892,2097152,75.6723009548,1048576,2909,true,1,0.0277136015892 +bf16,4,64,16,128,4096,optimized_transform,4,13,100,27.8180575371,2097152,75.3881537992,1048576,2909,true,1,0.0278180575371 +bf16,4,64,16,128,4096,control_shuffle32_transform,4,14,100,15.7193136215,2097152,133.412440931,1048576,2909,true,1,0.0157193136215 +bf16,16,64,16,128,16384,baseline_transform,0,0,100,78.9270830154,8388608,106.28301059,4194304,2909,true,1,0.0789270830154 +bf16,16,64,16,128,16384,control_optimized_transform,0,1,100,87.1764564514,8388608,96.225613445,4194304,2909,true,1,0.0871764564514 +bf16,16,64,16,128,16384,optimized_transform,0,2,100,87.2308540344,8388608,96.1656066865,4194304,2909,true,1,0.0872308540344 +bf16,16,64,16,128,16384,control_shuffle32_transform,0,3,100,26.618514061,8388608,315.141858812,4194304,2909,true,1,0.026618514061 +bf16,16,64,16,128,16384,shuffle32_transform,0,4,100,26.5606856346,8388608,315.827991619,4194304,2909,true,1,0.0265606856346 +bf16,16,64,16,128,16384,baseline_split,0,5,100,1884.42642212,13697024,7.26853743889,4194304,2909,true,1,1.88442642212 +bf16,16,64,16,128,16384,control_optimized_split,0,6,100,1452.84683228,13697024,9.42771371057,4194304,2909,true,1,1.45284683228 +bf16,16,64,16,128,16384,optimized_split,0,7,100,601.917953491,13697024,22.755632924,4194304,2909,true,1,0.601917953491 +bf16,16,64,16,128,16384,control_shuffle32_split,0,8,100,1606.09085083,13697024,8.52817509851,4194304,2909,true,1,1.60609085083 +bf16,16,64,16,128,16384,shuffle32_split,0,9,100,288.841152191,13697024,47.420611281,4194304,2909,true,1,0.288841152191 +bf16,16,64,16,128,16384,baseline_fused,0,10,100,1864.90310669,5308416,2.84648354167,4194304,2909,true,1,1.86490310669 +bf16,16,64,16,128,16384,control_optimized_fused,0,11,100,1421.36444092,5308416,3.73473251981,4194304,2909,true,1,1.42136444092 +bf16,16,64,16,128,16384,optimized_fused,0,12,100,566.370506287,5308416,9.37269144681,4194304,2909,true,1,0.566370506287 +bf16,16,64,16,128,16384,control_shuffle32_fused,0,13,100,1496.96228027,5308416,3.54612542343,4194304,2909,true,1,1.49696228027 +bf16,16,64,16,128,16384,shuffle32_fused,0,14,100,265.191078186,5308416,20.0173250032,4194304,2909,true,1,0.265191078186 +bf16,16,64,16,128,16384,control_optimized_transform,1,0,100,87.1101665497,8388608,96.298840104,4194304,2909,true,1,0.0871101665497 +bf16,16,64,16,128,16384,optimized_transform,1,1,100,87.2235488892,8388608,96.1736607468,4194304,2909,true,1,0.0872235488892 +bf16,16,64,16,128,16384,control_shuffle32_transform,1,2,100,26.6146278381,8388608,315.187875292,4194304,2909,true,1,0.0266146278381 +bf16,16,64,16,128,16384,shuffle32_transform,1,3,100,26.6395449638,8388608,314.893066356,4194304,2909,true,1,0.0266395449638 +bf16,16,64,16,128,16384,baseline_split,1,4,100,1884.64523315,13697024,7.26769354733,4194304,2909,true,1,1.88464523315 +bf16,16,64,16,128,16384,control_optimized_split,1,5,100,1452.8515625,13697024,9.42768301562,4194304,2909,true,1,1.4528515625 +bf16,16,64,16,128,16384,optimized_split,1,6,100,601.959075928,13697024,22.754078388,4194304,2909,true,1,0.601959075928 +bf16,16,64,16,128,16384,control_shuffle32_split,1,7,100,1614.18518066,13697024,8.48541057375,4194304,2909,true,1,1.61418518066 +bf16,16,64,16,128,16384,shuffle32_split,1,8,100,288.941936493,13697024,47.4040707495,4194304,2909,true,1,0.288941936493 +bf16,16,64,16,128,16384,baseline_fused,1,9,100,1864.77722168,5308416,2.84667569846,4194304,2909,true,1,1.86477722168 +bf16,16,64,16,128,16384,control_optimized_fused,1,10,100,1421.62780762,5308416,3.73404063395,4194304,2909,true,1,1.42162780762 +bf16,16,64,16,128,16384,optimized_fused,1,11,100,566.322746277,5308416,9.37348187919,4194304,2909,true,1,0.566322746277 +bf16,16,64,16,128,16384,control_shuffle32_fused,1,12,100,1496.04660034,5308416,3.54829588783,4194304,2909,true,1,1.49604660034 +bf16,16,64,16,128,16384,shuffle32_fused,1,13,100,265.220565796,5308416,20.0150994478,4194304,2909,true,1,0.265220565796 +bf16,16,64,16,128,16384,baseline_transform,1,14,100,78.9474344254,8388608,106.255612498,4194304,2909,true,1,0.0789474344254 +bf16,16,64,16,128,16384,optimized_transform,2,0,100,87.2340583801,8388608,96.1620742606,4194304,2909,true,1,0.0872340583801 +bf16,16,64,16,128,16384,control_shuffle32_transform,2,1,100,26.6128015518,8388608,315.209504857,4194304,2909,true,1,0.0266128015518 +bf16,16,64,16,128,16384,shuffle32_transform,2,2,100,26.639084816,8388608,314.898505634,4194304,2909,true,1,0.026639084816 +bf16,16,64,16,128,16384,baseline_split,2,3,100,1884.44229126,13697024,7.26847622956,4194304,2909,true,1,1.88444229126 +bf16,16,64,16,128,16384,control_optimized_split,2,4,100,1453.20861816,13697024,9.42536661894,4194304,2909,true,1,1.45320861816 +bf16,16,64,16,128,16384,optimized_split,2,5,100,601.888008118,13697024,22.7567650714,4194304,2909,true,1,0.601888008118 +bf16,16,64,16,128,16384,control_shuffle32_split,2,6,100,1607.91610718,13697024,8.51849417943,4194304,2909,true,1,1.60791610718 +bf16,16,64,16,128,16384,shuffle32_split,2,7,100,288.756561279,13697024,47.434503096,4194304,2909,true,1,0.288756561279 +bf16,16,64,16,128,16384,baseline_fused,2,8,100,1864.51675415,5308416,2.84707337072,4194304,2909,true,1,1.86451675415 +bf16,16,64,16,128,16384,control_optimized_fused,2,9,100,1421.50756836,5308416,3.73435648051,4194304,2909,true,1,1.42150756836 +bf16,16,64,16,128,16384,optimized_fused,2,10,100,566.289596558,5308416,9.37403058836,4194304,2909,true,1,0.566289596558 +bf16,16,64,16,128,16384,control_shuffle32_fused,2,11,100,1497.840271,5308416,3.54404678709,4194304,2909,true,1,1.497840271 +bf16,16,64,16,128,16384,shuffle32_fused,2,12,100,265.188102722,5308416,20.0175496016,4194304,2909,true,1,0.265188102722 +bf16,16,64,16,128,16384,baseline_transform,2,13,100,78.9154338837,8388608,106.298699597,4194304,2909,true,1,0.0789154338837 +bf16,16,64,16,128,16384,control_optimized_transform,2,14,100,87.1807956696,8388608,96.2208240424,4194304,2909,true,1,0.0871807956696 +bf16,16,64,16,128,16384,control_shuffle32_transform,3,0,100,26.6249155998,8388608,315.066087949,4194304,2909,true,1,0.0266249155998 +bf16,16,64,16,128,16384,shuffle32_transform,3,1,100,26.6313123703,8388608,314.990409911,4194304,2909,true,1,0.0266313123703 +bf16,16,64,16,128,16384,baseline_split,3,2,100,1884.67163086,13697024,7.26759175218,4194304,2909,true,1,1.88467163086 +bf16,16,64,16,128,16384,control_optimized_split,3,3,100,1452.9737854,13697024,9.42688996707,4194304,2909,true,1,1.4529737854 +bf16,16,64,16,128,16384,optimized_split,3,4,100,601.821937561,13697024,22.7592634052,4194304,2909,true,1,0.601821937561 +bf16,16,64,16,128,16384,control_shuffle32_split,3,5,100,1609.84893799,13697024,8.50826663098,4194304,2909,true,1,1.60984893799 +bf16,16,64,16,128,16384,shuffle32_split,3,6,100,288.81942749,13697024,47.4241782107,4194304,2909,true,1,0.28881942749 +bf16,16,64,16,128,16384,baseline_fused,3,7,100,1864.68002319,5308416,2.84682408455,4194304,2909,true,1,1.86468002319 +bf16,16,64,16,128,16384,control_optimized_fused,3,8,100,1421.60095215,5308416,3.73411117373,4194304,2909,true,1,1.42160095215 +bf16,16,64,16,128,16384,optimized_fused,3,9,100,566.300125122,5308416,9.37385630783,4194304,2909,true,1,0.566300125122 +bf16,16,64,16,128,16384,control_shuffle32_fused,3,10,100,1497.49313354,5308416,3.54486834102,4194304,2909,true,1,1.49749313354 +bf16,16,64,16,128,16384,shuffle32_fused,3,11,100,265.21074295,5308416,20.0158407648,4194304,2909,true,1,0.26521074295 +bf16,16,64,16,128,16384,baseline_transform,3,12,100,78.9284610748,8388608,106.281154932,4194304,2909,true,1,0.0789284610748 +bf16,16,64,16,128,16384,control_optimized_transform,3,13,100,87.1611404419,8388608,96.2425222693,4194304,2909,true,1,0.0871611404419 +bf16,16,64,16,128,16384,optimized_transform,3,14,100,87.2340583801,8388608,96.1620742606,4194304,2909,true,1,0.0872340583801 +bf16,16,64,16,128,16384,shuffle32_transform,4,0,100,26.6623997688,8388608,314.62314243,4194304,2909,true,1,0.0266623997688 +bf16,16,64,16,128,16384,baseline_split,4,1,100,1884.42626953,13697024,7.26853802744,4194304,2909,true,1,1.88442626953 +bf16,16,64,16,128,16384,control_optimized_split,4,2,100,1452.99743652,13697024,9.42673652114,4194304,2909,true,1,1.45299743652 +bf16,16,64,16,128,16384,optimized_split,4,3,100,601.798858643,13697024,22.7601362204,4194304,2909,true,1,0.601798858643 +bf16,16,64,16,128,16384,control_shuffle32_split,4,4,100,1606.82983398,13697024,8.52425297957,4194304,2909,true,1,1.60682983398 +bf16,16,64,16,128,16384,shuffle32_split,4,5,100,288.802509308,13697024,47.4269563406,4194304,2909,true,1,0.288802509308 +bf16,16,64,16,128,16384,baseline_fused,4,6,100,1864.59838867,5308416,2.84694872218,4194304,2909,true,1,1.86459838867 +bf16,16,64,16,128,16384,control_optimized_fused,4,7,100,1421.51779175,5308416,3.73432962346,4194304,2909,true,1,1.42151779175 +bf16,16,64,16,128,16384,optimized_fused,4,8,100,566.261482239,5308416,9.37449599964,4194304,2909,true,1,0.566261482239 +bf16,16,64,16,128,16384,control_shuffle32_fused,4,9,100,1496.56707764,5308416,3.54706185865,4194304,2909,true,1,1.49656707764 +bf16,16,64,16,128,16384,shuffle32_fused,4,10,100,265.20275116,5308416,20.016443935,4194304,2909,true,1,0.26520275116 +bf16,16,64,16,128,16384,baseline_transform,4,11,100,78.9236545563,8388608,106.287627545,4194304,2909,true,1,0.0789236545563 +bf16,16,64,16,128,16384,control_optimized_transform,4,12,100,87.179889679,8388608,96.2218239882,4194304,2909,true,1,0.087179889679 +bf16,16,64,16,128,16384,optimized_transform,4,13,100,87.213716507,8388608,96.1845032637,4194304,2909,true,1,0.087213716507 +bf16,16,64,16,128,16384,control_shuffle32_transform,4,14,100,26.5469694138,8388608,315.991172825,4194304,2909,true,1,0.0265469694138 +bf16,1,1,1,256,1,baseline_transform,0,0,100,9.76274311543,1024,0.104888553134,512,2909,true,1,0.00976274311543 +bf16,1,1,1,256,1,control_optimized_transform,0,1,100,10.6624007225,1024,0.0960384088584,512,2909,true,1,0.0106624007225 +bf16,1,1,1,256,1,optimized_transform,0,2,100,10.5003428459,1024,0.0975206252811,512,2909,true,1,0.0105003428459 +bf16,1,1,1,256,1,control_shuffle32_transform,0,3,100,11.6790854931,1024,0.0876780977934,512,2909,true,1,0.0116790854931 +bf16,1,1,1,256,1,shuffle32_transform,0,4,100,11.5380573273,1024,0.0887497757166,512,2909,true,1,0.0115380573273 +bf16,1,1,1,256,1,baseline_split,0,5,100,100.542402267,1668,0.0165900153804,512,2909,true,1,0.100542402267 +bf16,1,1,1,256,1,control_optimized_split,0,6,100,67.005944252,1668,0.0248933138488,512,2909,true,1,0.067005944252 +bf16,1,1,1,256,1,optimized_split,0,7,100,35.4809141159,1668,0.0470111901444,512,2909,true,1,0.0354809141159 +bf16,1,1,1,256,1,control_shuffle32_split,0,8,100,304.884357452,1668,0.00547092679315,512,2909,true,1,0.304884357452 +bf16,1,1,1,256,1,shuffle32_split,0,9,100,68.1211423874,1668,0.02448579019,512,2909,true,1,0.0681211423874 +bf16,1,1,1,256,1,baseline_fused,0,10,100,95.954284668,644,0.00671152937285,512,2909,true,1,0.095954284668 +bf16,1,1,1,256,1,control_optimized_fused,0,11,100,61.1494874954,644,0.0105315682335,512,2909,true,1,0.0611494874954 +bf16,1,1,1,256,1,optimized_fused,0,12,100,29.9641132355,644,0.021492376395,512,2909,true,1,0.0299641132355 +bf16,1,1,1,256,1,control_shuffle32_fused,0,13,100,300.694847107,644,0.00214170613895,512,2909,true,1,0.300694847107 +bf16,1,1,1,256,1,shuffle32_fused,0,14,100,61.4982843399,644,0.0104718368474,512,2909,true,1,0.0614982843399 +bf16,1,1,1,256,1,control_optimized_transform,1,0,100,10.6603431702,1024,0.0960569452272,512,2909,true,1,0.0106603431702 +bf16,1,1,1,256,1,optimized_transform,1,1,100,10.5638861656,1024,0.0969340244628,512,2909,true,1,0.0105638861656 +bf16,1,1,1,256,1,control_shuffle32_transform,1,2,100,11.6361141205,1024,0.0880018870043,512,2909,true,1,0.0116361141205 +bf16,1,1,1,256,1,shuffle32_transform,1,3,100,11.7385137081,1024,0.0872342125641,512,2909,true,1,0.0117385137081 +bf16,1,1,1,256,1,baseline_split,1,4,100,100.505828857,1668,0.0165960523779,512,2909,true,1,0.100505828857 +bf16,1,1,1,256,1,control_optimized_split,1,5,100,67.0125722885,1668,0.0248908517169,512,2909,true,1,0.0670125722885 +bf16,1,1,1,256,1,optimized_split,1,6,100,35.5247998238,1668,0.0469531146769,512,2909,true,1,0.0355247998238 +bf16,1,1,1,256,1,control_shuffle32_split,1,7,100,305.533485413,1668,0.00545930341399,512,2909,true,1,0.305533485413 +bf16,1,1,1,256,1,shuffle32_split,1,8,100,68.1261730194,1668,0.024483982089,512,2909,true,1,0.0681261730194 +bf16,1,1,1,256,1,baseline_fused,1,9,100,96.0111999512,644,0.00670755078915,512,2909,true,1,0.0960111999512 +bf16,1,1,1,256,1,control_optimized_fused,1,10,100,61.1561155319,644,0.010530426833,512,2909,true,1,0.0611561155319 +bf16,1,1,1,256,1,optimized_fused,1,11,100,29.9865150452,644,0.0214763202403,512,2909,true,1,0.0299865150452 +bf16,1,1,1,256,1,control_shuffle32_fused,1,12,100,300.685710907,644,0.00214177121373,512,2909,true,1,0.300685710907 +bf16,1,1,1,256,1,shuffle32_fused,1,13,100,61.5407991409,644,0.0104646024912,512,2909,true,1,0.0615407991409 +bf16,1,1,1,256,1,baseline_transform,1,14,100,9.71908569336,1024,0.105359704843,512,2909,true,1,0.00971908569336 +bf16,1,1,1,256,1,optimized_transform,2,0,100,10.5199992657,1024,0.0973384098364,512,2909,true,1,0.0105199992657 +bf16,1,1,1,256,1,control_shuffle32_transform,2,1,100,11.5476572514,1024,0.0886759952872,512,2909,true,1,0.0115476572514 +bf16,1,1,1,256,1,shuffle32_transform,2,2,100,11.5158867836,1024,0.0889206380058,512,2909,true,1,0.0115158867836 +bf16,1,1,1,256,1,baseline_split,2,3,100,100.585136414,1668,0.0165829670215,512,2909,true,1,0.100585136414 +bf16,1,1,1,256,1,control_optimized_split,2,4,100,66.9033145905,1668,0.0249315001837,512,2909,true,1,0.0669033145905 +bf16,1,1,1,256,1,optimized_split,2,5,100,35.5264019966,1668,0.0469509971812,512,2909,true,1,0.0355264019966 +bf16,1,1,1,256,1,control_shuffle32_split,2,6,100,305.574398041,1668,0.00545857248086,512,2909,true,1,0.305574398041 +bf16,1,1,1,256,1,shuffle32_split,2,7,100,68.0507421494,1668,0.0245111213679,512,2909,true,1,0.0680507421494 +bf16,1,1,1,256,1,baseline_fused,2,8,100,96.0111999512,644,0.00670755078915,512,2909,true,1,0.0960111999512 +bf16,1,1,1,256,1,control_optimized_fused,2,9,100,61.1570310593,644,0.0105302691914,512,2909,true,1,0.0611570310593 +bf16,1,1,1,256,1,optimized_fused,2,10,100,29.9488019943,644,0.0215033643123,512,2909,true,1,0.0299488019943 +bf16,1,1,1,256,1,control_shuffle32_fused,2,11,100,300.701713562,644,0.00214165723358,512,2909,true,1,0.300701713562 +bf16,1,1,1,256,1,shuffle32_fused,2,12,100,61.5412569046,644,0.0104645246521,512,2909,true,1,0.0615412569046 +bf16,1,1,1,256,1,baseline_transform,2,13,100,9.81211423874,1024,0.10436079066,512,2909,true,1,0.00981211423874 +bf16,1,1,1,256,1,control_optimized_transform,2,14,100,10.6061720848,1024,0.0965475566314,512,2909,true,1,0.0106061720848 +bf16,1,1,1,256,1,control_shuffle32_transform,3,0,100,11.669485569,1024,0.0877502263442,512,2909,true,1,0.011669485569 +bf16,1,1,1,256,1,shuffle32_transform,3,1,100,11.7279994488,1024,0.0873124188377,512,2909,true,1,0.0117279994488 +bf16,1,1,1,256,1,baseline_split,3,2,100,100.543546677,1668,0.0165898265491,512,2909,true,1,0.100543546677 +bf16,1,1,1,256,1,control_optimized_split,3,3,100,66.8909740448,1668,0.0249360997327,512,2909,true,1,0.0668909740448 +bf16,1,1,1,256,1,optimized_split,3,4,100,35.5131435394,1668,0.0469685258402,512,2909,true,1,0.0355131435394 +bf16,1,1,1,256,1,control_shuffle32_split,3,5,100,304.881839752,1668,0.00547097197182,512,2909,true,1,0.304881839752 +bf16,1,1,1,256,1,shuffle32_split,3,6,100,68.0516576767,1668,0.0245107916096,512,2909,true,1,0.0680516576767 +bf16,1,1,1,256,1,baseline_fused,3,7,100,96.014623642,644,0.00670731161121,512,2909,true,1,0.096014623642 +bf16,1,1,1,256,1,control_optimized_fused,3,8,100,61.1570310593,644,0.0105302691914,512,2909,true,1,0.0611570310593 +bf16,1,1,1,256,1,optimized_fused,3,9,100,29.9849152565,644,0.0214774660689,512,2909,true,1,0.0299849152565 +bf16,1,1,1,256,1,control_shuffle32_fused,3,10,100,300.686168671,644,0.0021417679531,512,2909,true,1,0.300686168671 +bf16,1,1,1,256,1,shuffle32_fused,3,11,100,61.4966869354,644,0.0104721088581,512,2909,true,1,0.0614966869354 +bf16,1,1,1,256,1,baseline_transform,3,12,100,9.92114305496,1024,0.10321391339,512,2909,true,1,0.00992114305496 +bf16,1,1,1,256,1,control_optimized_transform,3,13,100,10.6203424931,1024,0.0964187360878,512,2909,true,1,0.0106203424931 +bf16,1,1,1,256,1,optimized_transform,3,14,100,10.4596567154,1024,0.0978999624809,512,2909,true,1,0.0104596567154 +bf16,1,1,1,256,1,shuffle32_transform,4,0,100,11.5929150581,1024,0.088329811343,512,2909,true,1,0.0115929150581 +bf16,1,1,1,256,1,baseline_split,4,1,100,100.555648804,1668,0.0165878299215,512,2909,true,1,0.100555648804 +bf16,1,1,1,256,1,control_optimized_split,4,2,100,66.9213724136,1668,0.0249247727571,512,2909,true,1,0.0669213724136 +bf16,1,1,1,256,1,optimized_split,4,3,100,35.5712008476,1668,0.0468918664609,512,2909,true,1,0.0355712008476 +bf16,1,1,1,256,1,control_shuffle32_split,4,4,100,305.039539337,1668,0.00546814358435,512,2909,true,1,0.305039539337 +bf16,1,1,1,256,1,shuffle32_split,4,5,100,68.0502891541,1668,0.0245112845329,512,2909,true,1,0.0680502891541 +bf16,1,1,1,256,1,baseline_fused,4,6,100,96.0093688965,644,0.00670767871305,512,2909,true,1,0.0960093688965 +bf16,1,1,1,256,1,control_optimized_fused,4,7,100,61.1490297318,644,0.0105316470731,512,2909,true,1,0.0611490297318 +bf16,1,1,1,256,1,optimized_fused,4,8,100,29.944229126,644,0.0215066481522,512,2909,true,1,0.029944229126 +bf16,1,1,1,256,1,control_shuffle32_fused,4,9,100,300.70104599,644,0.00214166198817,512,2909,true,1,0.30070104599 +bf16,1,1,1,256,1,shuffle32_fused,4,10,100,61.5469694138,644,0.0104635533826,512,2909,true,1,0.0615469694138 +bf16,1,1,1,256,1,baseline_transform,4,11,100,9.74125683308,1024,0.105119905732,512,2909,true,1,0.00974125683308 +bf16,1,1,1,256,1,control_optimized_transform,4,12,100,10.6786286831,1024,0.095892462449,512,2909,true,1,0.0106786286831 +bf16,1,1,1,256,1,optimized_transform,4,13,100,10.598628521,1024,0.0966162742636,512,2909,true,1,0.010598628521 +bf16,1,1,1,256,1,control_shuffle32_transform,4,14,100,11.5661716461,1024,0.0885340483723,512,2909,true,1,0.0115661716461 +bf16,1,17,1,256,17,baseline_transform,0,0,100,10.0061714649,17408,1.739726334,8704,2909,true,1,0.0100061714649 +bf16,1,17,1,256,17,control_optimized_transform,0,1,100,10.7988572121,17408,1.61202242591,8704,2909,true,1,0.0107988572121 +bf16,1,17,1,256,17,optimized_transform,0,2,100,10.7773721218,17408,1.61523605228,8704,2909,true,1,0.0107773721218 +bf16,1,17,1,256,17,control_shuffle32_transform,0,3,100,11.7700564861,17408,1.47900734551,8704,2909,true,1,0.0117700564861 +bf16,1,17,1,256,17,shuffle32_transform,0,4,100,11.8034291267,17408,1.47482564711,8704,2909,true,1,0.0118034291267 +bf16,1,17,1,256,17,baseline_split,0,5,100,101.358394623,28356,0.279759758484,8704,2909,true,1,0.101358394623 +bf16,1,17,1,256,17,control_optimized_split,0,6,100,67.4653720856,28356,0.420304507682,8704,2909,true,1,0.0674653720856 +bf16,1,17,1,256,17,optimized_split,0,7,100,36.240913868,28356,0.78243060049,8704,2909,true,1,0.036240913868 +bf16,1,17,1,256,17,control_shuffle32_split,0,8,100,305.418281555,28356,0.0928431652998,8704,2909,true,1,0.305418281555 +bf16,1,17,1,256,17,shuffle32_split,0,9,100,68.1382846832,28356,0.416153710529,8704,2909,true,1,0.0681382846832 +bf16,1,17,1,256,17,baseline_fused,0,10,100,96.5318870544,10948,0.113413301387,8704,2909,true,1,0.0965318870544 +bf16,1,17,1,256,17,control_optimized_fused,0,11,100,61.6004562378,10948,0.177725956407,8704,2909,true,1,0.0616004562378 +bf16,1,17,1,256,17,optimized_fused,0,12,100,30.2450299263,10948,0.361976828149,8704,2909,true,1,0.0302450299263 +bf16,1,17,1,256,17,control_shuffle32_fused,0,13,100,301.225585938,10948,0.0363448541927,8704,2909,true,1,0.301225585937 +bf16,1,17,1,256,17,shuffle32_fused,0,14,100,61.4676570892,10948,0.17810992835,8704,2909,true,1,0.0614676570892 +bf16,1,17,1,256,17,control_optimized_transform,1,0,100,10.8941709995,17408,1.59791874028,8704,2909,true,1,0.0108941709995 +bf16,1,17,1,256,17,optimized_transform,1,1,100,10.8032000065,17408,1.61137440662,8704,2909,true,1,0.0108032000065 +bf16,1,17,1,256,17,control_shuffle32_transform,1,2,100,11.8852567673,17408,1.46467176443,8704,2909,true,1,0.0118852567673 +bf16,1,17,1,256,17,shuffle32_transform,1,3,100,11.7385137081,17408,1.48298161359,8704,2909,true,1,0.0117385137081 +bf16,1,17,1,256,17,baseline_split,1,4,100,101.37257576,28356,0.27972062254,8704,2909,true,1,0.10137257576 +bf16,1,17,1,256,17,control_optimized_split,1,5,100,67.4399995804,28356,0.420462636068,8704,2909,true,1,0.0674399995804 +bf16,1,17,1,256,17,optimized_split,1,6,100,36.2452554703,28356,0.782336877809,8704,2909,true,1,0.0362452554703 +bf16,1,17,1,256,17,control_shuffle32_split,1,7,100,306.034965515,28356,0.092656079191,8704,2909,true,1,0.306034965515 +bf16,1,17,1,256,17,shuffle32_split,1,8,100,68.7099456787,28356,0.412691346499,8704,2909,true,1,0.0687099456787 +bf16,1,17,1,256,17,baseline_fused,1,9,100,96.5423965454,10948,0.11340095535,8704,2909,true,1,0.0965423965454 +bf16,1,17,1,256,17,control_optimized_fused,1,10,100,61.5913152695,10948,0.177752333297,8704,2909,true,1,0.0615913152695 +bf16,1,17,1,256,17,optimized_fused,1,11,100,30.2354288101,10948,0.362091772164,8704,2909,true,1,0.0302354288101 +bf16,1,17,1,256,17,control_shuffle32_fused,1,12,100,301.229496002,10948,0.0363443824237,8704,2909,true,1,0.301229496002 +bf16,1,17,1,256,17,shuffle32_fused,1,13,100,61.4701700211,10948,0.178102647125,8704,2909,true,1,0.0614701700211 +bf16,1,17,1,256,17,baseline_transform,1,14,100,10.1161146164,17408,1.72081877876,8704,2909,true,1,0.0101161146164 +bf16,1,17,1,256,17,optimized_transform,2,0,100,10.917943716,17408,1.59443943409,8704,2909,true,1,0.010917943716 +bf16,1,17,1,256,17,control_shuffle32_transform,2,1,100,11.8505144119,17408,1.46896576764,8704,2909,true,1,0.0118505144119 +bf16,1,17,1,256,17,shuffle32_transform,2,2,100,11.8368005753,17408,1.47066767657,8704,2909,true,1,0.0118368005753 +bf16,1,17,1,256,17,baseline_split,2,3,100,101.389026642,28356,0.279675236455,8704,2909,true,1,0.101389026642 +bf16,1,17,1,256,17,control_optimized_split,2,4,100,67.3590898514,28356,0.420967683242,8704,2909,true,1,0.0673590898514 +bf16,1,17,1,256,17,optimized_split,2,5,100,36.2559986115,28356,0.782105060845,8704,2909,true,1,0.0362559986115 +bf16,1,17,1,256,17,control_shuffle32_split,2,6,100,306.142406464,28356,0.0926235614581,8704,2909,true,1,0.306142406464 +bf16,1,17,1,256,17,shuffle32_split,2,7,100,68.248000145,28356,0.415484701966,8704,2909,true,1,0.068248000145 +bf16,1,17,1,256,17,baseline_fused,2,8,100,96.5302944183,10948,0.113415172573,8704,2909,true,1,0.0965302944183 +bf16,1,17,1,256,17,control_optimized_fused,2,9,100,61.6077709198,10948,0.17770485503,8704,2909,true,1,0.0616077709198 +bf16,1,17,1,256,17,optimized_fused,2,10,100,30.2662873268,10948,0.361722595236,8704,2909,true,1,0.0302662873268 +bf16,1,17,1,256,17,control_shuffle32_fused,2,11,100,301.224899292,10948,0.0363449370412,8704,2909,true,1,0.301224899292 +bf16,1,17,1,256,17,shuffle32_fused,2,12,100,61.4580535889,10948,0.178137759995,8704,2909,true,1,0.0614580535889 +bf16,1,17,1,256,17,baseline_transform,2,13,100,10.1453721523,17408,1.71585622869,8704,2909,true,1,0.0101453721523 +bf16,1,17,1,256,17,control_optimized_transform,2,14,100,10.7346284389,17408,1.62166768035,8704,2909,true,1,0.0107346284389 +bf16,1,17,1,256,17,control_shuffle32_transform,3,0,100,11.9789707661,17408,1.45321333026,8704,2909,true,1,0.0119789707661 +bf16,1,17,1,256,17,shuffle32_transform,3,1,100,11.9602286816,17408,1.45549056489,8704,2909,true,1,0.0119602286816 +bf16,1,17,1,256,17,baseline_split,3,2,100,101.302633286,28356,0.279913750318,8704,2909,true,1,0.101302633286 +bf16,1,17,1,256,17,control_optimized_split,3,3,100,67.4701738358,28356,0.420274595246,8704,2909,true,1,0.0674701738358 +bf16,1,17,1,256,17,optimized_split,3,4,100,36.2717700005,28356,0.781764992435,8704,2909,true,1,0.0362717700005 +bf16,1,17,1,256,17,control_shuffle32_split,3,5,100,305.433616638,28356,0.092838503869,8704,2909,true,1,0.305433616638 +bf16,1,17,1,256,17,shuffle32_split,3,6,100,68.1120014191,28356,0.416314297176,8704,2909,true,1,0.0681120014191 +bf16,1,17,1,256,17,baseline_fused,3,7,100,96.5375995636,10948,0.113406590277,8704,2909,true,1,0.0965375995636 +bf16,1,17,1,256,17,control_optimized_fused,3,8,100,61.5929174423,10948,0.177747709552,8704,2909,true,1,0.0615929174423 +bf16,1,17,1,256,17,optimized_fused,3,9,100,30.2292561531,10948,0.362165709422,8704,2909,true,1,0.0302292561531 +bf16,1,17,1,256,17,control_shuffle32_fused,3,10,100,301.213493347,10948,0.036346313302,8704,2909,true,1,0.301213493347 +bf16,1,17,1,256,17,shuffle32_fused,3,11,100,61.4740562439,10948,0.17809138796,8704,2909,true,1,0.0614740562439 +bf16,1,17,1,256,17,baseline_transform,3,12,100,10.0589716434,17408,1.73059440041,8704,2909,true,1,0.0100589716434 +bf16,1,17,1,256,17,control_optimized_transform,3,13,100,10.8159995079,17408,1.60946752885,8704,2909,true,1,0.0108159995079 +bf16,1,17,1,256,17,optimized_transform,3,14,100,10.8223998547,17408,1.6085156928,8704,2909,true,1,0.0108223998547 +bf16,1,17,1,256,17,shuffle32_transform,4,0,100,11.8377137184,17408,1.47055423151,8704,2909,true,1,0.0118377137184 +bf16,1,17,1,256,17,baseline_split,4,1,100,101.285486221,28356,0.279961138144,8704,2909,true,1,0.101285486221 +bf16,1,17,1,256,17,control_optimized_split,4,2,100,67.4781703949,28356,0.42022479024,8704,2909,true,1,0.0674781703949 +bf16,1,17,1,256,17,optimized_split,4,3,100,36.2827420235,28356,0.781528584076,8704,2909,true,1,0.0362827420235 +bf16,1,17,1,256,17,control_shuffle32_split,4,4,100,305.499191284,28356,0.0928185763138,8704,2909,true,1,0.305499191284 +bf16,1,17,1,256,17,shuffle32_split,4,5,100,68.2438850403,28356,0.415509755684,8704,2909,true,1,0.0682438850403 +bf16,1,17,1,256,17,baseline_fused,4,6,100,96.5369129181,10948,0.113407396912,8704,2909,true,1,0.0965369129181 +bf16,1,17,1,256,17,control_optimized_fused,4,7,100,61.5819454193,10948,0.177779378768,8704,2909,true,1,0.0615819454193 +bf16,1,17,1,256,17,optimized_fused,4,8,100,30.2500581741,10948,0.361916659366,8704,2909,true,1,0.0302500581741 +bf16,1,17,1,256,17,control_shuffle32_fused,4,9,100,301.223983765,10948,0.0363450475064,8704,2909,true,1,0.301223983765 +bf16,1,17,1,256,17,shuffle32_fused,4,10,100,61.4733695984,10948,0.178093377206,8704,2909,true,1,0.0614733695984 +bf16,1,17,1,256,17,baseline_transform,4,11,100,10.0665140152,17408,1.72929774634,8704,2909,true,1,0.0100665140152 +bf16,1,17,1,256,17,control_optimized_transform,4,12,100,10.7469713688,17408,1.61980519,8704,2909,true,1,0.0107469713688 +bf16,1,17,1,256,17,optimized_transform,4,13,100,10.7183992863,17408,1.62412311158,8704,2909,true,1,0.0107183992863 +bf16,1,17,1,256,17,control_shuffle32_transform,4,14,100,12.0399999619,17408,1.44584718066,8704,2909,true,1,0.0120399999619 +bf16,1,257,1,256,257,baseline_transform,0,0,100,10.4329144955,263168,25.2247825969,131584,2909,true,1,0.0104329144955 +bf16,1,257,1,256,257,control_optimized_transform,0,1,100,11.0338294506,263168,23.8510121239,131584,2909,true,1,0.0110338294506 +bf16,1,257,1,256,257,optimized_transform,0,2,100,11.0166859627,263168,23.8881275995,131584,2909,true,1,0.0110166859627 +bf16,1,257,1,256,257,control_shuffle32_transform,0,3,100,13.6713147163,263168,19.2496482936,131584,2909,true,1,0.0136713147163 +bf16,1,257,1,256,257,shuffle32_transform,0,4,100,13.7803423405,263168,19.0973484909,131584,2909,true,1,0.0137803423405 +bf16,1,257,1,256,257,baseline_split,0,5,100,104.000692368,428676,4.12185717461,131584,2909,true,1,0.104000692368 +bf16,1,257,1,256,257,control_optimized_split,0,6,100,69.7138309479,428676,6.14908109584,131584,2909,true,1,0.0697138309479 +bf16,1,257,1,256,257,optimized_split,0,7,100,36.8434286118,428676,11.635073503,131584,2909,true,1,0.0368434286118 +bf16,1,257,1,256,257,control_shuffle32_split,0,8,100,328.400001526,428676,1.30534713157,131584,2909,true,1,0.328400001526 +bf16,1,257,1,256,257,shuffle32_split,0,9,100,76.7730283737,428676,5.58367969951,131584,2909,true,1,0.0767730283737 +bf16,1,257,1,256,257,baseline_fused,0,10,100,98.8875389099,165508,1.67369925295,131584,2909,true,1,0.0988875389099 +bf16,1,257,1,256,257,control_optimized_fused,0,11,100,63.8022899628,165508,2.59407617025,131584,2909,true,1,0.0638022899628 +bf16,1,257,1,256,257,optimized_fused,0,12,100,30.7497143745,165508,5.3824239791,131584,2909,true,1,0.0307497143745 +bf16,1,257,1,256,257,control_shuffle32_fused,0,13,100,321.583061218,165508,0.514666411138,131584,2909,true,1,0.321583061218 +bf16,1,257,1,256,257,shuffle32_fused,0,14,100,68.3775997162,165508,2.42050029084,131584,2909,true,1,0.0683775997162 +bf16,1,257,1,256,257,control_optimized_transform,1,0,100,11.0797715187,263168,23.7521143424,131584,2909,true,1,0.0110797715187 +bf16,1,257,1,256,257,optimized_transform,1,1,100,11.0548567772,263168,23.8056453651,131584,2909,true,1,0.0110548567772 +bf16,1,257,1,256,257,control_shuffle32_transform,1,2,100,13.8569140434,263168,18.9918187538,131584,2909,true,1,0.0138569140434 +bf16,1,257,1,256,257,shuffle32_transform,1,3,100,13.9389705658,263168,18.8800169107,131584,2909,true,1,0.0139389705658 +bf16,1,257,1,256,257,baseline_split,1,4,100,104.065599442,428676,4.11928631844,131584,2909,true,1,0.104065599442 +bf16,1,257,1,256,257,control_optimized_split,1,5,100,69.6694850922,428676,6.15299509438,131584,2909,true,1,0.0696694850922 +bf16,1,257,1,256,257,optimized_split,1,6,100,36.8868589401,428676,11.6213744493,131584,2909,true,1,0.0368868589401 +bf16,1,257,1,256,257,control_shuffle32_split,1,7,100,328.500556946,428676,1.30494755925,131584,2909,true,1,0.328500556946 +bf16,1,257,1,256,257,shuffle32_split,1,8,100,76.8612575531,428676,5.57727018328,131584,2909,true,1,0.0768612575531 +bf16,1,257,1,256,257,baseline_fused,1,9,100,98.8790893555,165508,1.67384227625,131584,2909,true,1,0.0988790893555 +bf16,1,257,1,256,257,control_optimized_fused,1,10,100,63.7734889984,165508,2.5952476899,131584,2909,true,1,0.0637734889984 +bf16,1,257,1,256,257,optimized_fused,1,11,100,30.7291412354,165508,5.3860275083,131584,2909,true,1,0.0307291412354 +bf16,1,257,1,256,257,control_shuffle32_fused,1,12,100,321.592254639,165508,0.514651698269,131584,2909,true,1,0.321592254639 +bf16,1,257,1,256,257,shuffle32_fused,1,13,100,68.3750867844,165508,2.42058924944,131584,2909,true,1,0.0683750867844 +bf16,1,257,1,256,257,baseline_transform,1,14,100,10.766171217,263168,24.4439731355,131584,2909,true,1,0.010766171217 +bf16,1,257,1,256,257,optimized_transform,2,0,100,11.108571291,263168,23.6905352729,131584,2909,true,1,0.011108571291 +bf16,1,257,1,256,257,control_shuffle32_transform,2,1,100,13.7193143368,263168,19.182299752,131584,2909,true,1,0.0137193143368 +bf16,1,257,1,256,257,shuffle32_transform,2,2,100,13.9209139347,263168,18.9045059279,131584,2909,true,1,0.0139209139347 +bf16,1,257,1,256,257,baseline_split,2,3,100,104.082736969,428676,4.11860806589,131584,2909,true,1,0.104082736969 +bf16,1,257,1,256,257,control_optimized_split,2,4,100,69.6937131882,428676,6.15085608716,131584,2909,true,1,0.0696937131882 +bf16,1,257,1,256,257,optimized_split,2,5,100,36.9122266769,428676,11.6133877198,131584,2909,true,1,0.0369122266769 +bf16,1,257,1,256,257,control_shuffle32_split,2,6,100,328.402290344,428676,1.30533803388,131584,2909,true,1,0.328402290344 +bf16,1,257,1,256,257,shuffle32_split,2,7,100,76.8114280701,428676,5.5808882971,131584,2909,true,1,0.0768114280701 +bf16,1,257,1,256,257,baseline_fused,2,8,100,98.8868522644,165508,1.6737108747,131584,2909,true,1,0.0988868522644 +bf16,1,257,1,256,257,control_optimized_fused,2,9,100,63.7613725662,165508,2.59574085906,131584,2909,true,1,0.0637613725662 +bf16,1,257,1,256,257,optimized_fused,2,10,100,30.6281137466,165508,5.40379343531,131584,2909,true,1,0.0306281137466 +bf16,1,257,1,256,257,control_shuffle32_fused,2,11,100,321.579208374,165508,0.514672577362,131584,2909,true,1,0.321579208374 +bf16,1,257,1,256,257,shuffle32_fused,2,12,100,68.372797966,165508,2.42067028005,131584,2909,true,1,0.068372797966 +bf16,1,257,1,256,257,baseline_transform,2,13,100,10.6514275074,263168,24.7072986055,131584,2909,true,1,0.0106514275074 +bf16,1,257,1,256,257,control_optimized_transform,2,14,100,11.093941927,263168,23.7217755179,131584,2909,true,1,0.011093941927 +bf16,1,257,1,256,257,control_shuffle32_transform,3,0,100,13.7634289265,263168,19.1208165789,131584,2909,true,1,0.0137634289265 +bf16,1,257,1,256,257,shuffle32_transform,3,1,100,13.9106285572,263168,18.9184837276,131584,2909,true,1,0.0139106285572 +bf16,1,257,1,256,257,baseline_split,3,2,100,104.065599442,428676,4.11928631844,131584,2909,true,1,0.104065599442 +bf16,1,257,1,256,257,control_optimized_split,3,3,100,69.6825122833,428676,6.15184478793,131584,2909,true,1,0.0696825122833 +bf16,1,257,1,256,257,optimized_split,3,4,100,36.884572506,428676,11.6220948455,131584,2909,true,1,0.036884572506 +bf16,1,257,1,256,257,control_shuffle32_split,3,5,100,328.390846252,428676,1.3053835236,131584,2909,true,1,0.328390846252 +bf16,1,257,1,256,257,shuffle32_split,3,6,100,76.7757701874,428676,5.58348029533,131584,2909,true,1,0.0767757701874 +bf16,1,257,1,256,257,baseline_fused,3,7,100,98.879776001,165508,1.67383065267,131584,2909,true,1,0.098879776001 +bf16,1,257,1,256,257,control_optimized_fused,3,8,100,63.7650299072,165508,2.59559197637,131584,2909,true,1,0.0637650299072 +bf16,1,257,1,256,257,optimized_fused,3,9,100,30.7195425034,165508,5.38771044464,131584,2909,true,1,0.0307195425034 +bf16,1,257,1,256,257,control_shuffle32_fused,3,10,100,321.576919556,165508,0.514676240536,131584,2909,true,1,0.321576919556 +bf16,1,257,1,256,257,shuffle32_fused,3,11,100,68.376455307,165508,2.42054080249,131584,2909,true,1,0.068376455307 +bf16,1,257,1,256,257,baseline_transform,3,12,100,10.5405712128,263168,24.9671478602,131584,2909,true,1,0.0105405712128 +bf16,1,257,1,256,257,control_optimized_transform,3,13,100,11.0861718655,263168,23.7384016046,131584,2909,true,1,0.0110861718655 +bf16,1,257,1,256,257,optimized_transform,3,14,100,11.0502851009,263168,23.815494134,131584,2909,true,1,0.0110502851009 +bf16,1,257,1,256,257,shuffle32_transform,4,0,100,14.0031993389,263168,18.7934195344,131584,2909,true,1,0.0140031993389 +bf16,1,257,1,256,257,baseline_split,4,1,100,104.071092606,428676,4.11906889096,131584,2909,true,1,0.104071092606 +bf16,1,257,1,256,257,control_optimized_split,4,2,100,69.617600441,428676,6.15758080262,131584,2909,true,1,0.069617600441 +bf16,1,257,1,256,257,optimized_split,4,3,100,36.9085717201,428676,11.6145377624,131584,2909,true,1,0.0369085717201 +bf16,1,257,1,256,257,control_shuffle32_split,4,4,100,328.395233154,428676,1.3053660855,131584,2909,true,1,0.328395233154 +bf16,1,257,1,256,257,shuffle32_split,4,5,100,76.7668581009,428676,5.5841284977,131584,2909,true,1,0.0767668581009 +bf16,1,257,1,256,257,baseline_fused,4,6,100,98.8701725006,165508,1.67399323592,131584,2909,true,1,0.0988701725006 +bf16,1,257,1,256,257,control_optimized_fused,4,7,100,63.7622880936,165508,2.59570358826,131584,2909,true,1,0.0637622880936 +bf16,1,257,1,256,257,optimized_fused,4,8,100,30.7085704803,165508,5.3896354474,131584,2909,true,1,0.0307085704803 +bf16,1,257,1,256,257,control_shuffle32_fused,4,9,100,321.595878601,165508,0.514645898822,131584,2909,true,1,0.321595878601 +bf16,1,257,1,256,257,shuffle32_fused,4,10,100,68.3286857605,165508,2.42223303665,131584,2909,true,1,0.0683286857605 +bf16,1,257,1,256,257,baseline_transform,4,11,100,10.5387425423,263168,24.971480131,131584,2909,true,1,0.0105387425423 +bf16,1,257,1,256,257,control_optimized_transform,4,12,100,10.9408009052,263168,24.0538149153,131584,2909,true,1,0.0109408009052 +bf16,1,257,1,256,257,optimized_transform,4,13,100,11.1227428913,263168,23.6603509199,131584,2909,true,1,0.0111227428913 +bf16,1,257,1,256,257,control_shuffle32_transform,4,14,100,13.7005710602,263168,19.2085423917,131584,2909,true,1,0.0137005710602 +bf16,4,64,16,256,4096,baseline_transform,0,0,100,49.279999733,4194304,85.1116887729,2097152,2909,true,1,0.049279999733 +bf16,4,64,16,256,4096,control_optimized_transform,0,1,100,37.2688007355,4194304,112.541963176,2097152,2909,true,1,0.0372688007355 +bf16,4,64,16,256,4096,optimized_transform,0,2,100,37.3645710945,4194304,112.253503175,2097152,2909,true,1,0.0373645710945 +bf16,4,64,16,256,4096,control_shuffle32_transform,0,3,100,22.0036578178,4194304,190.618488741,2097152,2909,true,1,0.0220036578178 +bf16,4,64,16,256,4096,shuffle32_transform,0,4,100,22.0939421654,4194304,189.839548262,2097152,2909,true,1,0.0220939421654 +bf16,4,64,16,256,4096,baseline_split,0,5,100,1142.19177246,6832128,5.9815944789,2097152,2909,true,1,1.14219177246 +bf16,4,64,16,256,4096,control_optimized_split,0,6,100,500.945358276,6832128,13.6384695199,2097152,2909,true,1,0.500945358276 +bf16,4,64,16,256,4096,optimized_split,0,7,100,188.860797882,6832128,36.1754693225,2097152,2909,true,1,0.188860797882 +bf16,4,64,16,256,4096,control_shuffle32_split,0,8,100,839.615783691,6832128,8.13720767607,2097152,2909,true,1,0.839615783691 +bf16,4,64,16,256,4096,shuffle32_split,0,9,100,149.190397263,6832128,45.7946900428,2097152,2909,true,1,0.149190397263 +bf16,4,64,16,256,4096,baseline_fused,0,10,100,1136.90834045,2637824,2.32017296922,2097152,2909,true,1,1.13690834045 +bf16,4,64,16,256,4096,control_optimized_fused,0,11,100,485.864486694,2637824,5.42913522646,2097152,2909,true,1,0.485864486694 +bf16,4,64,16,256,4096,optimized_fused,0,12,100,171.31521225,2637824,15.3974884388,2097152,2909,true,1,0.17131521225 +bf16,4,64,16,256,4096,control_shuffle32_fused,0,13,100,838.787612915,2637824,3.14480562109,2097152,2909,true,1,0.838787612915 +bf16,4,64,16,256,4096,shuffle32_fused,0,14,100,131.079769135,2637824,20.1238071856,2097152,2909,true,1,0.131079769135 +bf16,4,64,16,256,4096,control_optimized_transform,1,0,100,37.3414850235,4194304,112.322902995,2097152,2909,true,1,0.0373414850235 +bf16,4,64,16,256,4096,optimized_transform,1,1,100,37.3584008217,4194304,112.272043443,2097152,2909,true,1,0.0373584008217 +bf16,4,64,16,256,4096,control_shuffle32_transform,1,2,100,22.4665141106,4194304,186.691356717,2097152,2909,true,1,0.0224665141106 +bf16,4,64,16,256,4096,shuffle32_transform,1,3,100,23.2214856148,4194304,180.621691031,2097152,2909,true,1,0.0232214856148 +bf16,4,64,16,256,4096,baseline_split,1,4,100,1146.57806396,6832128,5.95871159123,2097152,2909,true,1,1.14657806396 +bf16,4,64,16,256,4096,control_optimized_split,1,5,100,500.460548401,6832128,13.6516814799,2097152,2909,true,1,0.500460548401 +bf16,4,64,16,256,4096,optimized_split,1,6,100,188.66147995,6832128,36.2136881456,2097152,2909,true,1,0.18866147995 +bf16,4,64,16,256,4096,control_shuffle32_split,1,7,100,839.024734497,6832128,8.14293991475,2097152,2909,true,1,0.839024734497 +bf16,4,64,16,256,4096,shuffle32_split,1,8,100,149.456224442,6832128,45.7132382778,2097152,2909,true,1,0.149456224442 +bf16,4,64,16,256,4096,baseline_fused,1,9,100,1136.76048279,2637824,2.32047475254,2097152,2909,true,1,1.13676048279 +bf16,4,64,16,256,4096,control_optimized_fused,1,10,100,487.146034241,2637824,5.4148526614,2097152,2909,true,1,0.487146034241 +bf16,4,64,16,256,4096,optimized_fused,1,11,100,171.595668793,2637824,15.3723227314,2097152,2909,true,1,0.171595668793 +bf16,4,64,16,256,4096,control_shuffle32_fused,1,12,100,837.211914062,2637824,3.1507243933,2097152,2909,true,1,0.837211914062 +bf16,4,64,16,256,4096,shuffle32_fused,1,13,100,131.525478363,2637824,20.0556122877,2097152,2909,true,1,0.131525478363 +bf16,4,64,16,256,4096,baseline_transform,1,14,100,49.2331409454,4194304,85.1926958032,2097152,2909,true,1,0.0492331409454 +bf16,4,64,16,256,4096,optimized_transform,2,0,100,37.3632001877,4194304,112.25762191,2097152,2909,true,1,0.0373632001877 +bf16,4,64,16,256,4096,control_shuffle32_transform,2,1,100,21.9926857948,4194304,190.713587196,2097152,2909,true,1,0.0219926857948 +bf16,4,64,16,256,4096,shuffle32_transform,2,2,100,22.4694871902,4194304,186.666654405,2097152,2909,true,1,0.0224694871902 +bf16,4,64,16,256,4096,baseline_split,2,3,100,1146.25991821,6832128,5.96036543845,2097152,2909,true,1,1.14625991821 +bf16,4,64,16,256,4096,control_optimized_split,2,4,100,499.854393005,6832128,13.6682363816,2097152,2909,true,1,0.499854393005 +bf16,4,64,16,256,4096,optimized_split,2,5,100,188.795661926,6832128,36.1879501377,2097152,2909,true,1,0.188795661926 +bf16,4,64,16,256,4096,control_shuffle32_split,2,6,100,838.371658325,6832128,8.14928311585,2097152,2909,true,1,0.838371658325 +bf16,4,64,16,256,4096,shuffle32_split,2,7,100,149.492578506,6832128,45.702121592,2097152,2909,true,1,0.149492578506 +bf16,4,64,16,256,4096,baseline_fused,2,8,100,1138.12164307,2637824,2.31769953244,2097152,2909,true,1,1.13812164307 +bf16,4,64,16,256,4096,control_optimized_fused,2,9,100,482.755889893,2637824,5.46409490848,2097152,2909,true,1,0.482755889893 +bf16,4,64,16,256,4096,optimized_fused,2,10,100,171.619644165,2637824,15.3701752083,2097152,2909,true,1,0.171619644165 +bf16,4,64,16,256,4096,control_shuffle32_fused,2,11,100,835.825805664,2637824,3.15594945995,2097152,2909,true,1,0.835825805664 +bf16,4,64,16,256,4096,shuffle32_fused,2,12,100,131.347198486,2637824,20.0828341251,2097152,2909,true,1,0.131347198486 +bf16,4,64,16,256,4096,baseline_transform,2,13,100,49.2930269241,4194304,85.089195404,2097152,2909,true,1,0.0492930269241 +bf16,4,64,16,256,4096,control_optimized_transform,2,14,100,37.3490285873,4194304,112.300216596,2097152,2909,true,1,0.0373490285873 +bf16,4,64,16,256,4096,control_shuffle32_transform,3,0,100,22.4644565582,4194304,186.70845605,2097152,2909,true,1,0.0224644565582 +bf16,4,64,16,256,4096,shuffle32_transform,3,1,100,22.5718855858,4194304,185.819832555,2097152,2909,true,1,0.0225718855858 +bf16,4,64,16,256,4096,baseline_split,3,2,100,1143.85421753,6832128,5.97290100023,2097152,2909,true,1,1.14385421753 +bf16,4,64,16,256,4096,control_optimized_split,3,3,100,497.549476624,6832128,13.731554993,2097152,2909,true,1,0.497549476624 +bf16,4,64,16,256,4096,optimized_split,3,4,100,188.897609711,6832128,36.16841955,2097152,2909,true,1,0.188897609711 +bf16,4,64,16,256,4096,control_shuffle32_split,3,5,100,841.471786499,6832128,8.11925974182,2097152,2909,true,1,0.841471786499 +bf16,4,64,16,256,4096,shuffle32_split,3,6,100,149.523429871,6832128,45.6926918137,2097152,2909,true,1,0.149523429871 +bf16,4,64,16,256,4096,baseline_fused,3,7,100,1136.43798828,2637824,2.32113324898,2097152,2909,true,1,1.13643798828 +bf16,4,64,16,256,4096,control_optimized_fused,3,8,100,485.77671051,2637824,5.43011623021,2097152,2909,true,1,0.48577671051 +bf16,4,64,16,256,4096,optimized_fused,3,9,100,171.817817688,2637824,15.3524473509,2097152,2909,true,1,0.171817817688 +bf16,4,64,16,256,4096,control_shuffle32_fused,3,10,100,838.523635864,2637824,3.1457956427,2097152,2909,true,1,0.838523635864 +bf16,4,64,16,256,4096,shuffle32_fused,3,11,100,130.888690948,2637824,20.1531849764,2097152,2909,true,1,0.130888690948 +bf16,4,64,16,256,4096,baseline_transform,3,12,100,49.2770290375,4194304,85.1168197825,2097152,2909,true,1,0.0492770290375 +bf16,4,64,16,256,4096,control_optimized_transform,3,13,100,37.295999527,4194304,112.459889886,2097152,2909,true,1,0.037295999527 +bf16,4,64,16,256,4096,optimized_transform,3,14,100,37.421028614,4194304,112.084145074,2097152,2909,true,1,0.037421028614 +bf16,4,64,16,256,4096,shuffle32_transform,4,0,100,22.5554299355,4194304,185.955400185,2097152,2909,true,1,0.0225554299355 +bf16,4,64,16,256,4096,baseline_split,4,1,100,1145.71151733,6832128,5.9632183989,2097152,2909,true,1,1.14571151733 +bf16,4,64,16,256,4096,control_optimized_split,4,2,100,501.460571289,6832128,13.6244570185,2097152,2909,true,1,0.501460571289 +bf16,4,64,16,256,4096,optimized_split,4,3,100,188.692111969,6832128,36.2078092651,2097152,2909,true,1,0.188692111969 +bf16,4,64,16,256,4096,control_shuffle32_split,4,4,100,839.649581909,6832128,8.13688013095,2097152,2909,true,1,0.839649581909 +bf16,4,64,16,256,4096,shuffle32_split,4,5,100,149.429941177,6832128,45.721278789,2097152,2909,true,1,0.149429941177 +bf16,4,64,16,256,4096,baseline_fused,4,6,100,1136.63879395,2637824,2.32072318317,2097152,2909,true,1,1.13663879395 +bf16,4,64,16,256,4096,control_optimized_fused,4,7,100,486.915893555,2637824,5.41741199028,2097152,2909,true,1,0.486915893555 +bf16,4,64,16,256,4096,optimized_fused,4,8,100,171.734638214,2637824,15.3598832911,2097152,2909,true,1,0.171734638214 +bf16,4,64,16,256,4096,control_shuffle32_fused,4,9,100,836.434707642,2637824,3.1536520136,2097152,2909,true,1,0.836434707642 +bf16,4,64,16,256,4096,shuffle32_fused,4,10,100,130.839548111,2637824,20.1607544361,2097152,2909,true,1,0.130839548111 +bf16,4,64,16,256,4096,baseline_transform,4,11,100,49.2795419693,4194304,85.1124793857,2097152,2909,true,1,0.0492795419693 +bf16,4,64,16,256,4096,control_optimized_transform,4,12,100,37.2799992561,4194304,112.508156751,2097152,2909,true,1,0.0372799992561 +bf16,4,64,16,256,4096,optimized_transform,4,13,100,37.3174858093,4194304,112.395138875,2097152,2909,true,1,0.0373174858093 +bf16,4,64,16,256,4096,control_shuffle32_transform,4,14,100,22.2832012177,4194304,188.22717432,2097152,2909,true,1,0.0222832012177 +bf16,16,64,16,256,16384,baseline_transform,0,0,100,170.587215424,16777216,98.3497852306,8388608,2909,true,1,0.170587215424 +bf16,16,64,16,256,16384,control_optimized_transform,0,1,100,120.674285889,16777216,139.028922993,8388608,2909,true,1,0.120674285889 +bf16,16,64,16,256,16384,optimized_transform,0,2,100,120.734624863,16777216,138.959441163,8388608,2909,true,1,0.120734624863 +bf16,16,64,16,256,16384,control_shuffle32_transform,0,3,100,43.0847978592,16777216,389.399900513,8388608,2909,true,1,0.0430847978592 +bf16,16,64,16,256,16384,shuffle32_transform,0,4,100,43.2066297531,16777216,388.301890147,8388608,2909,true,1,0.0432066297531 +bf16,16,64,16,256,16384,baseline_split,0,5,100,4271.84417725,27328512,6.39735694143,8388608,2909,true,1,4.27184417725 +bf16,16,64,16,256,16384,control_optimized_split,0,6,100,1869.99176025,27328512,14.6142419346,8388608,2909,true,1,1.86999176025 +bf16,16,64,16,256,16384,optimized_split,0,7,100,677.922058105,27328512,40.3121740519,8388608,2909,true,1,0.677922058105 +bf16,16,64,16,256,16384,control_shuffle32_split,0,8,100,3203.55224609,27328512,8.53069027774,8388608,2909,true,1,3.20355224609 +bf16,16,64,16,256,16384,shuffle32_split,0,9,100,436.136894226,27328512,62.6603994337,8388608,2909,true,1,0.436136894226 +bf16,16,64,16,256,16384,baseline_fused,0,10,100,4272.9977417,10551296,2.4692959458,8388608,2909,true,1,4.2729977417 +bf16,16,64,16,256,16384,control_optimized_fused,0,11,100,1823.56613159,10551296,5.78607806825,8388608,2909,true,1,1.82356613159 +bf16,16,64,16,256,16384,optimized_fused,0,12,100,631.597251892,10551296,16.7057344984,8388608,2909,true,1,0.631597251892 +bf16,16,64,16,256,16384,control_shuffle32_fused,0,13,100,3042.70233154,10551296,3.46773849371,8388608,2909,true,1,3.04270233154 +bf16,16,64,16,256,16384,shuffle32_fused,0,14,100,390.83065033,10551296,26.9971047335,8388608,2909,true,1,0.39083065033 +bf16,16,64,16,256,16384,control_optimized_transform,1,0,100,120.614862442,16777216,139.097418513,8388608,2909,true,1,0.120614862442 +bf16,16,64,16,256,16384,optimized_transform,1,1,100,120.746965408,16777216,138.945239272,8388608,2909,true,1,0.120746965408 +bf16,16,64,16,256,16384,control_shuffle32_transform,1,2,100,43.105597496,16777216,389.212004347,8388608,2909,true,1,0.043105597496 +bf16,16,64,16,256,16384,shuffle32_transform,1,3,100,43.1568002701,16777216,388.75022928,8388608,2909,true,1,0.0431568002701 +bf16,16,64,16,256,16384,baseline_split,1,4,100,4271.43859863,27328512,6.39796437873,8388608,2909,true,1,4.27143859863 +bf16,16,64,16,256,16384,control_optimized_split,1,5,100,1870.08392334,27328512,14.6135217029,8388608,2909,true,1,1.87008392334 +bf16,16,64,16,256,16384,optimized_split,1,6,100,677.846908569,27328512,40.3166432634,8388608,2909,true,1,0.677846908569 +bf16,16,64,16,256,16384,control_shuffle32_split,1,7,100,3206.95159912,27328512,8.52164778773,8388608,2909,true,1,3.20695159912 +bf16,16,64,16,256,16384,shuffle32_split,1,8,100,436.123199463,27328512,62.6623670414,8388608,2909,true,1,0.436123199463 +bf16,16,64,16,256,16384,baseline_fused,1,9,100,4270.16235352,10551296,2.47093555853,8388608,2909,true,1,4.27016235352 +bf16,16,64,16,256,16384,control_optimized_fused,1,10,100,1823.57971191,10551296,5.78603497893,8388608,2909,true,1,1.82357971191 +bf16,16,64,16,256,16384,optimized_fused,1,11,100,631.59362793,10551296,16.7058303526,8388608,2909,true,1,0.63159362793 +bf16,16,64,16,256,16384,control_shuffle32_fused,1,12,100,3041.76116943,10551296,3.46881145898,8388608,2909,true,1,3.04176116943 +bf16,16,64,16,256,16384,shuffle32_fused,1,13,100,391.056480408,10551296,26.9815142534,8388608,2909,true,1,0.391056480408 +bf16,16,64,16,256,16384,baseline_transform,1,14,100,170.645027161,16777216,98.3164659361,8388608,2909,true,1,0.170645027161 +bf16,16,64,16,256,16384,optimized_transform,2,0,100,120.688686371,16777216,139.012334167,8388608,2909,true,1,0.120688686371 +bf16,16,64,16,256,16384,control_shuffle32_transform,2,1,100,43.1469726562,16777216,388.838775171,8388608,2909,true,1,0.0431469726562 +bf16,16,64,16,256,16384,shuffle32_transform,2,2,100,43.1650257111,16777216,388.676149814,8388608,2909,true,1,0.0431650257111 +bf16,16,64,16,256,16384,baseline_split,2,3,100,4270.2255249,27328512,6.39978189457,8388608,2909,true,1,4.2702255249 +bf16,16,64,16,256,16384,control_optimized_split,2,4,100,1869.90859985,27328512,14.6148918734,8388608,2909,true,1,1.86990859985 +bf16,16,64,16,256,16384,optimized_split,2,5,100,677.840957642,27328512,40.3169972129,8388608,2909,true,1,0.677840957642 +bf16,16,64,16,256,16384,control_shuffle32_split,2,6,100,3192.88269043,27328512,8.55919701714,8388608,2909,true,1,3.19288269043 +bf16,16,64,16,256,16384,shuffle32_split,2,7,100,436.27910614,27328512,62.6399743086,8388608,2909,true,1,0.43627910614 +bf16,16,64,16,256,16384,baseline_fused,2,8,100,4270.48431396,10551296,2.4707492697,8388608,2909,true,1,4.27048431396 +bf16,16,64,16,256,16384,control_optimized_fused,2,9,100,1823.58474731,10551296,5.78601900215,8388608,2909,true,1,1.82358474731 +bf16,16,64,16,256,16384,optimized_fused,2,10,100,631.570053101,10551296,16.7064539368,8388608,2909,true,1,0.631570053101 +bf16,16,64,16,256,16384,control_shuffle32_fused,2,11,100,3042.34802246,10551296,3.46814234338,8388608,2909,true,1,3.04234802246 +bf16,16,64,16,256,16384,shuffle32_fused,2,12,100,391.094856262,10551296,26.9788667149,8388608,2909,true,1,0.391094856262 +bf16,16,64,16,256,16384,baseline_transform,2,13,100,170.64617157,16777216,98.3158065936,8388608,2909,true,1,0.17064617157 +bf16,16,64,16,256,16384,control_optimized_transform,2,14,100,120.679769516,16777216,139.022605589,8388608,2909,true,1,0.120679769516 +bf16,16,64,16,256,16384,control_shuffle32_transform,3,0,100,43.0505180359,16777216,389.709967857,8388608,2909,true,1,0.0430505180359 +bf16,16,64,16,256,16384,shuffle32_transform,3,1,100,43.1727981567,16777216,388.606176025,8388608,2909,true,1,0.0431727981567 +bf16,16,64,16,256,16384,baseline_split,3,2,100,4270.47149658,27328512,6.39941327834,8388608,2909,true,1,4.27047149658 +bf16,16,64,16,256,16384,control_optimized_split,3,3,100,1870.15045166,27328512,14.6130018447,8388608,2909,true,1,1.87015045166 +bf16,16,64,16,256,16384,optimized_split,3,4,100,677.855758667,27328512,40.3161168886,8388608,2909,true,1,0.677855758667 +bf16,16,64,16,256,16384,control_shuffle32_split,3,5,100,3204.79003906,27328512,8.52739545084,8388608,2909,true,1,3.20479003906 +bf16,16,64,16,256,16384,shuffle32_split,3,6,100,436.194725037,27328512,62.6520919016,8388608,2909,true,1,0.436194725037 +bf16,16,64,16,256,16384,baseline_fused,3,7,100,4270.60577393,10551296,2.47067899932,8388608,2909,true,1,4.27060577393 +bf16,16,64,16,256,16384,control_optimized_fused,3,8,100,1823.73352051,10551296,5.78554700089,8388608,2909,true,1,1.82373352051 +bf16,16,64,16,256,16384,optimized_fused,3,9,100,631.765708923,10551296,16.7012800014,8388608,2909,true,1,0.631765708923 +bf16,16,64,16,256,16384,control_shuffle32_fused,3,10,100,3042.13409424,10551296,3.46838622925,8388608,2909,true,1,3.04213409424 +bf16,16,64,16,256,16384,shuffle32_fused,3,11,100,390.831756592,10551296,26.9970283173,8388608,2909,true,1,0.390831756592 +bf16,16,64,16,256,16384,baseline_transform,3,12,100,170.697612762,16777216,98.2861782803,8388608,2909,true,1,0.170697612762 +bf16,16,64,16,256,16384,control_optimized_transform,3,13,100,120.690507889,16777216,139.010236128,8388608,2909,true,1,0.120690507889 +bf16,16,64,16,256,16384,optimized_transform,3,14,100,120.750398636,16777216,138.941288721,8388608,2909,true,1,0.120750398636 +bf16,16,64,16,256,16384,shuffle32_transform,4,0,100,43.1296014786,16777216,388.995386575,8388608,2909,true,1,0.0431296014786 +bf16,16,64,16,256,16384,baseline_split,4,1,100,4271.42486572,27328512,6.39798494861,8388608,2909,true,1,4.27142486572 +bf16,16,64,16,256,16384,control_optimized_split,4,2,100,1869.87503052,27328512,14.6151542504,8388608,2909,true,1,1.86987503052 +bf16,16,64,16,256,16384,optimized_split,4,3,100,677.902832031,27328512,40.3133173498,8388608,2909,true,1,0.677902832031 +bf16,16,64,16,256,16384,control_shuffle32_split,4,4,100,3203.07830811,27328512,8.5319525067,8388608,2909,true,1,3.20307830811 +bf16,16,64,16,256,16384,shuffle32_split,4,5,100,436.092147827,27328512,62.6668288713,8388608,2909,true,1,0.436092147827 +bf16,16,64,16,256,16384,baseline_fused,4,6,100,4273.27880859,10551296,2.4691335325,8388608,2909,true,1,4.27327880859 +bf16,16,64,16,256,16384,control_optimized_fused,4,7,100,1823.75198364,10551296,5.7854884297,8388608,2909,true,1,1.82375198364 +bf16,16,64,16,256,16384,optimized_fused,4,8,100,631.632652283,10551296,16.7047982112,8388608,2909,true,1,0.631632652283 +bf16,16,64,16,256,16384,control_shuffle32_fused,4,9,100,3041.65039062,10551296,3.46893779526,8388608,2909,true,1,3.04165039063 +bf16,16,64,16,256,16384,shuffle32_fused,4,10,100,391.095085144,10551296,26.978850926,8388608,2909,true,1,0.391095085144 +bf16,16,64,16,256,16384,baseline_transform,4,11,100,170.584907532,16777216,98.3511158329,8388608,2909,true,1,0.170584907532 +bf16,16,64,16,256,16384,control_optimized_transform,4,12,100,120.631999969,16777216,139.077657705,8388608,2909,true,1,0.120631999969 +bf16,16,64,16,256,16384,optimized_transform,4,13,100,120.690288544,16777216,139.010488768,8388608,2909,true,1,0.120690288544 +bf16,16,64,16,256,16384,control_shuffle32_transform,4,14,100,43.0173683167,16777216,390.010283207,8388608,2909,true,1,0.0430173683167 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round3.log b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round3.log new file mode 100644 index 00000000..22c2e112 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/paired-round3.log @@ -0,0 +1,32 @@ +DEVICE name=MTT S4000 warp=128 runtime=50100 driver=50100 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=64 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=128 +BENCHMARK_PROGRESS dtype=fp16 rows=1 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=17 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=257 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=4096 n=256 +BENCHMARK_PROGRESS dtype=fp16 rows=16384 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=64 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=128 +BENCHMARK_PROGRESS dtype=bf16 rows=1 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=17 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=257 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=4096 n=256 +BENCHMARK_PROGRESS dtype=bf16 rows=16384 n=256 +PASS requested validation/benchmark operations completed diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/quant-gpu.log b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/quant-gpu.log new file mode 100644 index 00000000..8438ebf0 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/recorded/quant-gpu.log @@ -0,0 +1,2 @@ +DEVICE MTT S4000 arch=2.2 reported_warp=128 +RESULT cases=2074568 legacy_errors=0 candidate_errors=0 different=0 diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/test_analyze_optimization.py b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/test_analyze_optimization.py new file mode 100644 index 00000000..069ad03e --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/repro/test_analyze_optimization.py @@ -0,0 +1,78 @@ +"""Check that damaged or structurally invalid measurements cannot pass the analysis.""" + +import csv +import hashlib +import json +import shutil +import tempfile +import unittest +from pathlib import Path + +from analyze_optimization import analyze + +ROOT = Path(__file__).resolve().parent + + +class AnalysisTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.recorded = Path(self.temporary.name) / "recorded" + shutil.copytree(ROOT / "recorded", self.recorded) + self.csv = self.recorded / "paired-round1.csv" + + def change_records(self, change): + with self.csv.open(newline="") as stream: + rows = list(csv.DictReader(stream)) + fields = list(rows[0]) + change(rows) + with self.csv.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + manifest = self.recorded / "optimization.json" + state = json.loads(manifest.read_text()) + state["artifacts"][self.csv.name]["sha256"] = hashlib.sha256( + self.csv.read_bytes() + ).hexdigest() + manifest.write_text(json.dumps(state)) + + def test_original_observations_are_complete_and_accepted(self): + result = analyze(self.recorded) + self.assertEqual(result["observations"], 6750) + self.assertEqual(len(result["comparisons"]), 540) + self.assertTrue(result["accepted"]) + + def test_a_changed_byte_is_rejected(self): + with self.csv.open("a") as stream: + stream.write("\n") + with self.assertRaisesRegex(ValueError, "Checksum"): + analyze(self.recorded) + + def test_missing_observation_is_rejected_after_rehashing(self): + self.change_records(lambda rows: rows.pop()) + with self.assertRaisesRegex(ValueError, "2250"): + analyze(self.recorded) + + def test_duplicate_group_is_rejected_after_rehashing(self): + self.change_records(lambda rows: rows.__setitem__(1, rows[0].copy())) + with self.assertRaisesRegex(ValueError, "Duplicate"): + analyze(self.recorded) + + def test_wrong_units_are_rejected_after_rehashing(self): + self.change_records( + lambda rows: rows[0].__setitem__("kernel_ms", rows[0]["kernel_us"]) + ) + with self.assertRaisesRegex(ValueError, "units"): + analyze(self.recorded) + + def test_unknown_configuration_is_rejected_after_rehashing(self): + self.change_records( + lambda rows: rows[0].__setitem__("method", "unknown_transform") + ) + with self.assertRaisesRegex(ValueError, "configuration"): + analyze(self.recorded) + + +if __name__ == "__main__": + unittest.main() diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/run_platform.py b/03_hadamard_tc/a962695448-rgb/platforms/moore/run_platform.py new file mode 100644 index 00000000..124bd82d --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/run_platform.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Build and validate native MUSA Hadamard kernels on the rented Moore device.""" + +import argparse +import csv +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import subprocess +import sys +import time + + +PLATFORM = Path(__file__).resolve().parent +ROOT = PLATFORM.parents[1] +BASE_COMMIT = "cb82ea6e1d5c8f78b48b6922b0f7af279696cc44" + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def capture(command, env): + result = subprocess.run(command, cwd=ROOT, env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + check=False, timeout=60) + return {"command": command, "returncode": result.returncode, + "output": result.stdout} + + +def run(command, log, stages, env, expected=0): + command = list(map(str, command)) + print("RUN", log.name, flush=True) + started = time.monotonic() + with log.open("w", encoding="utf-8") as output: + result = subprocess.run(command, cwd=ROOT, env=env, stdout=output, + stderr=subprocess.STDOUT, check=False) + stages.append({"command": command, "log": log.name, + "returncode": result.returncode, "expected_returncode": expected, + "wall_seconds": time.monotonic() - started, "sha256": sha256(log)}) + print("EXIT", result.returncode, log.name, flush=True) + if result.returncode != expected: + print(log.read_text(errors="replace")[-12000:], file=sys.stderr) + raise RuntimeError("Stage failed: " + log.name) + + +def summarize(paths): + groups = {} + for round_index, path in enumerate(paths, 1): + with path.open(newline="", encoding="utf-8") as stream: + for row in csv.DictReader(stream): + us, ms = float(row["kernel_us"]), float(row["kernel_ms"]) + if not math.isfinite(us) or us <= 0 or not math.isclose(ms, us / 1000, rel_tol=1e-9): + raise ValueError("Invalid timing or millisecond conversion") + key = (round_index, row["dtype"], row["batch"], row["seq"], + row["heads"], row["dim"], row["method"]) + groups.setdefault(key, []).append(us) + medians = {key: statistics.median(values) for key, values in groups.items()} + statistics_rows = [] + comparisons = [] + for key, samples in groups.items(): + statistics_rows.append({"round_dtype_shape_method": key, "samples_us": samples, + "median_us": medians[key], "median_ms": medians[key] / 1000}) + if not key[-1].startswith("baseline_"): + continue + operation = key[-1].removeprefix("baseline_") + for candidate in ("optimized", "shuffle32"): + peer = key[:-1] + (candidate + "_" + operation,) + if peer in medians: + comparisons.append({"round_dtype_shape": key[:-1], "operation": operation, + "candidate": candidate, "baseline_us": medians[key], + "candidate_us": medians[peer], + "baseline_over_candidate": medians[key] / medians[peer], + "time_reduction_percent": 100 * (1 - medians[peer] / medians[key])}) + return {"metric": "MUSA event interval per call, warm input; allocation/copy/validation excluded", + "limitations": "Includes any device idle gaps caused by host submission; not application latency or cross-platform speedup", + "rounds": len(paths), "observations": sum(map(len, groups.values())), + "statistics": statistics_rows, "comparisons_including_slowdowns": comparisons} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--musa-root", type=Path, default=Path("/usr/local/musa")) + parser.add_argument("--arch", default="mp_22") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--shuffle32", action="store_true") + parser.add_argument("--probe-only", action="store_true") + parser.add_argument("--quick", action="store_true") + parser.add_argument("--no-benchmark", action="store_true") + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--repeats", type=int, default=100) + parser.add_argument("--groups", type=int, default=5) + args = parser.parse_args() + if not (1 <= args.rounds <= 10 and 1 <= args.repeats <= 10000 and 1 <= args.groups <= 10000): + parser.error("Invalid benchmark repetition count") + musa = args.musa_root.resolve() + compiler = musa / "bin/mcc" + if not compiler.is_file(): + parser.error("MUSA compiler missing: " + str(compiler)) + destination = args.output.resolve() + if destination.exists(): + parser.error("Select a new output directory to preserve prior results") + destination.mkdir(parents=True) + env = os.environ.copy() + env["PATH"] = str(musa / "bin") + os.pathsep + env.get("PATH", "") + env["MUSA_PATH"] = str(musa) + env["LD_LIBRARY_PATH"] = str(musa / "lib") + os.pathsep + env.get("LD_LIBRARY_PATH", "") + sources = [PLATFORM / name for name in ("hadamard_api.h", "hadamard_api.mu", + "validate_and_benchmark.mu", "probe.mu", "run_platform.py", "exact_int4.hpp")] + sources.append(ROOT / "include/reference.hpp") + source_hashes = {str(p.relative_to(ROOT)): sha256(p) for p in sources} + report = {"status": "RUNNING", "platform": "moore", "target_model": "MTT S4000", + "base_commit": BASE_COMMIT, "source_sha256": source_hashes, + "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "python": sys.version, "arch": args.arch, "quick": args.quick, + "shuffle32_enabled": args.shuffle32, "scope": "probe" if args.probe_only else "validation", + "stages": []} + manifest = destination / "run_summary.json" + + def save(): + manifest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + save() + try: + report["compiler"] = capture([str(compiler), "--version"], env) + report["device_before"] = capture(["mthreads-gmi"], env) + version = musa / "version.json" + if version.exists(): + report["sdk_version"] = json.loads(version.read_text()) + common = [str(compiler), "-std=c++17", "-O2", "-fno-fast-math", "-ffp-contract=off", + "--offload-arch=" + args.arch, "-I" + str(ROOT / "include")] + link = ["-L" + str(musa / "lib"), "-lmusart"] + probe = destination / "probe" + run(common + [PLATFORM / "probe.mu"] + link + ["-o", probe], + destination / "probe_build.log", report["stages"], env) + run([probe], destination / "probe.log", report["stages"], env) + report["probe_sha256"] = sha256(probe) + if not args.probe_only: + binary = destination / "validate_and_benchmark" + defines = ["-DHADAMARD_MOORE_SHUFFLE32"] if args.shuffle32 else [] + run(common + defines + [PLATFORM / "hadamard_api.mu", PLATFORM / "validate_and_benchmark.mu"] + + link + ["-o", binary], destination / "build.log", report["stages"], env) + report["binary_sha256"] = sha256(binary) + invalid = [[], ["--validate", "--dim", "0"], ["--validate", "--dim", "3"], + ["--validate", "--dim", "512"], ["--validate", "--dtype", "fp32"], + ["--validate", "--batch", "-1"], ["--validate", "--seq", "0"], + ["--validate", "--heads", "x"], + ["--validate", "--batch", "18446744073709551615", "--seq", "2"], + ["--validate", "--heads", "999999999999999999999999999999"], + ["--validate", "--dim"], ["--validate", "--unknown", "1"], + ["--benchmark", "--repeats", "0"], ["--benchmark", "--groups", "10001"]] + for index, case in enumerate(invalid): + run([binary] + case, destination / f"invalid_{index:02d}.log", report["stages"], env, expected=2) + report["cli_rejection_cases"] = len(invalid) + validation = [binary, "--validate", "--json", destination / "validation.json"] + if args.quick: + validation.append("--quick") + run(validation, destination / "validation.log", report["stages"], env) + result = json.loads((destination / "validation.json").read_text()) + if result["status"] != "PASS" or (not args.quick and not result["full_matrix"]): + raise RuntimeError("Requested validation matrix was not completed") + if result["shuffle32_enabled"] != args.shuffle32: + raise RuntimeError("Validation build mode mismatch") + report["validation"] = result + save() + if not args.no_benchmark: + csv_paths = [] + for index in range(1, args.rounds + 1): + path = destination / f"benchmark_round{index}.csv" + command = [binary, "--benchmark", "--csv", path, "--groups", args.groups, "--repeats", args.repeats] + if args.quick: + command += ["--batch", "1", "--seq", "17", "--heads", "1", "--dim", "128"] + run(command, destination / f"benchmark_round{index}.log", report["stages"], env) + csv_paths.append(path) + report["benchmark"] = summarize(csv_paths) + if {str(p.relative_to(ROOT)): sha256(p) for p in sources} != source_hashes: + raise RuntimeError("Source changed during validation") + report["status"] = "PASS" + except (OSError, RuntimeError, ValueError, KeyError, subprocess.SubprocessError) as error: + report["status"] = "FAIL" + report["error"] = str(error) + print("FAIL", error, file=sys.stderr, flush=True) + finally: + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + report["artifacts"] = {p.name: {"bytes": p.stat().st_size, "sha256": sha256(p)} + for p in destination.iterdir() if p.is_file() and p != manifest} + save() + print(json.dumps({"status": report["status"], "scope": report["scope"], + "quick": args.quick, "summary": str(manifest)}), flush=True) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/platforms/moore/validate_and_benchmark.mu b/03_hadamard_tc/a962695448-rgb/platforms/moore/validate_and_benchmark.mu new file mode 100644 index 00000000..2cb2977f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/platforms/moore/validate_and_benchmark.mu @@ -0,0 +1,589 @@ +// Ported from the verified MetaX implementation at cb82ea6. +// MUSA results require independent validation on Moore Threads hardware. +#include "hadamard_api.h" +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace api = hadamard::moore; +#define CHECK(call) do { musaError_t e_ = (call); if (e_ != musaSuccess) \ + throw std::runtime_error(std::string(#call) + ": " + musaGetErrorString(e_)); } while (0) + +template float read(T); +template<> float read(__half x) { return __half2float(x); } +template<> float read(__mt_bfloat16 x) { return __bfloat162float(x); } +template T rounded(float); +template<> __half rounded(float x) { return __float2half_rn(x); } +template<> __mt_bfloat16 rounded(float x) { return __float2bfloat16_rn(x); } + +std::vector active_methods() { + std::vector methods{api::Method::Baseline, api::Method::Optimized}; +#if defined(HADAMARD_MOORE_SHUFFLE32) + methods.push_back(api::Method::Shuffle32); +#endif + return methods; +} + +const char* method_name(api::Method method) { + if (method == api::Method::Baseline) return "baseline"; + if (method == api::Method::Optimized) return "optimized"; + return "shuffle32"; +} + +// 每个区间都有前后哨兵。17 个元素的偏移同时覆盖仅 2 字节对齐的 FP16/BF16 指针。 +template class Guarded { + static constexpr std::size_t guard = 17; + T* raw_ = nullptr; + std::size_t count_; + std::vector initial_; + std::string name_; +public: + explicit Guarded(std::size_t count, musaStream_t stream, const char* name = "unnamed") + : count_(count), initial_((count + 2 * guard) * sizeof(T), 0xa5), name_(name) { + CHECK(musaMalloc(reinterpret_cast(&raw_), initial_.size())); + // 初始化、上传、kernel、回读使用同一个非阻塞 stream。默认 stream 上的 + // pageable H2D 即使通过 musaMemcpy 调用，也不保证返回时最终 DMA 已结束。 + // 按字节 memset 避免异步初始化读取随后被 upload 更新的主机 shadow。 + CHECK(musaMemsetAsync(raw_, 0xa5, initial_.size(), stream)); + } + ~Guarded() { if (raw_) musaFree(raw_); } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + T* data() { return raw_ + guard; } + void upload(const std::vector& values, musaStream_t stream) { + if (values.size() != count_) throw std::runtime_error("upload size mismatch"); + std::memcpy(initial_.data() + guard * sizeof(T), values.data(), count_ * sizeof(T)); + CHECK(musaMemcpyAsync(raw_, initial_.data(), initial_.size(), musaMemcpyHostToDevice, stream)); + } + std::vector download(musaStream_t stream, bool unchanged = false, const char* phase = "readback") { + std::vector bytes(initial_.size()); + CHECK(musaMemcpyAsync(bytes.data(), raw_, bytes.size(), musaMemcpyDeviceToHost, stream)); + CHECK(musaStreamSynchronize(stream)); + const std::size_t prefix = guard * sizeof(T), end = prefix + count_ * sizeof(T); + for (std::size_t i = 0; i < bytes.size(); ++i) { + if (bytes[i] == initial_[i] || (!unchanged && i >= prefix && i < end)) continue; + const bool is_guard = i < prefix || i >= end; + throw std::runtime_error(std::string(is_guard ? "device buffer guard overwritten" : "read-only input modified") + + " buffer=" + name_ + " phase=" + phase + " region=" + (i < prefix ? "prefix" : (i >= end ? "suffix" : "payload")) + + " byte_from_payload=" + std::to_string(static_cast(i) - static_cast(prefix)) + + " expected=" + std::to_string(static_cast(initial_[i])) + + " actual=" + std::to_string(static_cast(bytes[i])) + + " elements=" + std::to_string(count_) + " element_bytes=" + std::to_string(sizeof(T))); + } + std::vector result(count_); + std::memcpy(result.data(), bytes.data() + prefix, count_ * sizeof(T)); + return result; + } +}; + +struct Options { + bool validate = false, benchmark = false, custom_shape = false, quick = false; + std::size_t batch = 1, seq = 1, heads = 1; + int dim = 128, repeats = 100, groups = 5; + std::string dtype = "both", csv = "moore_benchmark.csv", json = "moore_validation.json"; +}; + +std::size_t positive(const std::string& text, const char* name) { + if (text.empty() || text.find_first_not_of("0123456789") != std::string::npos) + throw std::invalid_argument(std::string(name) + " must be a positive integer"); + std::size_t used = 0; + const auto value = std::stoull(text, &used); + if (!value || value > std::numeric_limits::max() || used != text.size()) + throw std::invalid_argument(std::string(name) + " is outside its supported range"); + return static_cast(value); +} + +std::size_t multiply(std::size_t a, std::size_t b) { + if (b && a > std::numeric_limits::max() / b) + throw std::invalid_argument("shape product overflows size_t"); + return a * b; +} + +std::size_t checked_shape(std::size_t b, std::size_t s, std::size_t h, int n) { + if (n < 1 || n > 256 || !hadamard::power_of_two(n)) + throw std::invalid_argument("dim must be a power of two in [1,256]"); + const auto rows = multiply(multiply(b, s), h); + if (!rows || rows > static_cast(std::numeric_limits::max())) + throw std::invalid_argument("rows outside supported grid range"); + multiply(multiply(rows, static_cast(n)), sizeof(__half)); + return rows; +} + +Options parse(int argc, char** argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string key = argv[i]; + if (key == "--validate") o.validate = true; + else if (key == "--benchmark") o.benchmark = true; + else if (key == "--quick") o.quick = true; + else { + if (i + 1 == argc) throw std::invalid_argument("missing value for " + key); + const std::string value = argv[++i]; + if (key == "--dtype") o.dtype = value; + else if (key == "--csv") o.csv = value; + else if (key == "--json") o.json = value; + else if (key == "--batch" || key == "--seq" || key == "--heads" || key == "--dim") { + const auto v = positive(value, key.c_str()); + o.custom_shape = true; + if (key == "--batch") o.batch = v; + else if (key == "--seq") o.seq = v; + else if (key == "--heads") o.heads = v; + else { + if (v > 256) throw std::invalid_argument("dim is greater than 256"); + o.dim = static_cast(v); + } + } else if (key == "--repeats" || key == "--groups") { + const auto v = positive(value, key.c_str()); + if (v > 10000) throw std::invalid_argument("repeats/groups exceed 10000"); + if (key == "--repeats") o.repeats = static_cast(v); + else o.groups = static_cast(v); + } else throw std::invalid_argument("unknown argument " + key); + } + } + if (!o.validate && !o.benchmark) throw std::invalid_argument("specify --validate and/or --benchmark"); + if (o.dtype != "both" && o.dtype != "fp16" && o.dtype != "bf16") + throw std::invalid_argument("dtype must be fp16, bf16 or both"); + checked_shape(o.batch, o.seq, o.heads, o.dim); + return o; +} + +template std::vector make_input(std::size_t rows, int n, int pattern, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::normal_distribution normal(0.0f, 0.5f); + std::vector result(rows * n); + for (std::size_t i = 0; i < result.size(); ++i) { + float x = 0; + if (pattern == 0) x = uniform(rng); + else if (pattern == 1) x = normal(rng); + else if (pattern == 2) x = i % n == (i / n + seed) % n ? 8.0f : uniform(rng); + else if (pattern == 4) x = i % n == (i / n) % n ? 1.0f : 0.0f; + result[i] = rounded(x); + } + return result; +} + +struct Summary { + std::size_t cases = 0, elements = 0, exact_transform_elements = 0, contract_checks = 0; + std::size_t shuffle32_cases = 0, exact_shuffle32_elements = 0; + std::size_t shuffle32_grid_stride_cases = 0, shuffle32_grid_stride_elements = 0, unsupported_shuffle32_checks = 0; + double max_rounded_error = 0, max_unrounded_error = 0; +}; + +template void one_case(musaStream_t stream, std::size_t rows, int n, float scale, + int pattern, unsigned seed, const char* dtype, Summary& summary) { + const auto input = make_input(rows, n, pattern, seed); + const std::size_t size = input.size(), bytes = rows * ((n + 1) / 2); + Guarded x(size, stream), baseline(size, stream), optimized(size, stream), inplace(size, stream); + Guarded base_split(bytes, stream), opt_split(bytes, stream), base_fused(bytes, stream), opt_fused(bytes, stream); + Guarded bs(rows, stream), os(rows, stream), bfs(rows, stream), ofs(rows, stream); + x.upload(input, stream); + inplace.upload(input, stream); + CHECK(api::transform(x.data(), baseline.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform(x.data(), optimized.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::transform(inplace.data(), inplace.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(api::quantize_int4(baseline.data(), base_split.data(), bs.data(), rows, n, stream, api::Method::Baseline)); + CHECK(api::quantize_int4(optimized.data(), opt_split.data(), os.data(), rows, n, stream, api::Method::Optimized)); + CHECK(api::transform_int4(x.data(), base_fused.data(), bfs.data(), rows, n, scale, stream, api::Method::Baseline)); + CHECK(api::transform_int4(x.data(), opt_fused.data(), ofs.data(), rows, n, scale, stream, api::Method::Optimized)); + CHECK(musaStreamSynchronize(stream)); + x.download(stream, true); + const auto a = baseline.download(stream), b = optimized.download(stream), ip = inplace.download(stream); + const std::string context = std::string(dtype) + " rows=" + std::to_string(rows) + " n=" + std::to_string(n) + + " pattern=" + std::to_string(pattern) + " seed=" + std::to_string(seed) + " scale=" + std::to_string(scale); + if (std::memcmp(a.data(), b.data(), size * sizeof(T)) || std::memcmp(b.data(), ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/optimized/in-place transform not bitwise identical: " + context); + std::vector fx(size), actual(size); + for (std::size_t i = 0; i < size; ++i) { fx[i] = read(input[i]); actual[i] = read(b[i]); } + // 全部元素使用 O(N^2) FP64 稠密矩阵公式，不调用任何 FWHT 作为正确性 oracle。 + const auto dense = hadamard::dense_reference(fx, n, static_cast(scale)); + const double tolerance = std::string(dtype) == "fp16" ? 1e-2 : 5e-2; + for (std::size_t i = 0; i < size; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + const double error = std::abs(static_cast(actual[i]) - expected); + if (!std::isfinite(actual[i]) || !(error < tolerance)) + throw std::runtime_error("rounded FP64 dense oracle mismatch: " + context + " index=" + std::to_string(i) + + " error=" + std::to_string(error)); + summary.max_rounded_error = std::max(summary.max_rounded_error, error); + summary.max_unrounded_error = std::max(summary.max_unrounded_error, std::abs(actual[i] - dense[i])); + } + const auto expected_q = hadamard::quantize_int4(actual, n); + auto compare_packed = [&](const char* name, const std::vector& got) { + for (std::size_t i = 0; i < got.size(); ++i) { + if (got[i] != expected_q.packed[i]) { + throw std::runtime_error(std::string(name) + " packed mismatch: " + context + + " index=" + std::to_string(i) + " got=" + std::to_string(got[i]) + + " expected=" + std::to_string(expected_q.packed[i])); + } + } + }; + auto compare_scales = [&](const char* name, const std::vector& got) { + for (std::size_t i = 0; i < got.size(); ++i) { + if (got[i] != expected_q.scales[i]) { + std::uint32_t actual_bits = 0, expected_bits = 0; + std::memcpy(&actual_bits, &got[i], sizeof(float)); + std::memcpy(&expected_bits, &expected_q.scales[i], sizeof(float)); + std::ostringstream message; + message << name << " scale mismatch: " << context << " index=" << i + << std::setprecision(15) << " got=" << got[i] << " expected=" << expected_q.scales[i] + << " got_bits=" << actual_bits << " expected_bits=" << expected_bits; + throw std::runtime_error(message.str()); + } + } + }; + compare_packed("baseline.split", base_split.download(stream)); + compare_packed("optimized.split", opt_split.download(stream)); + compare_packed("baseline.fused", base_fused.download(stream)); + compare_packed("optimized.fused", opt_fused.download(stream)); + compare_scales("baseline.split", bs.download(stream)); + compare_scales("optimized.split", os.download(stream)); + compare_scales("baseline.fused", bfs.download(stream)); + compare_scales("optimized.fused", ofs.download(stream)); +#if defined(HADAMARD_MOORE_SHUFFLE32) + Guarded warp_output(size, stream, "shuffle32.output"), warp_inplace(size, stream, "shuffle32.inplace"); + Guarded warp_split(bytes, stream, "shuffle32.split"), warp_fused(bytes, stream, "shuffle32.fused"); + Guarded warp_split_scales(rows, stream, "shuffle32.split-scales"), warp_fused_scales(rows, stream, "shuffle32.fused-scales"); + warp_inplace.upload(input, stream); + CHECK(api::transform(x.data(), warp_output.data(), rows, n, scale, stream, api::Method::Shuffle32)); + CHECK(api::transform(warp_inplace.data(), warp_inplace.data(), rows, n, scale, stream, api::Method::Shuffle32)); + CHECK(api::quantize_int4(warp_output.data(), warp_split.data(), warp_split_scales.data(), rows, n, stream, api::Method::Shuffle32)); + CHECK(api::transform_int4(x.data(), warp_fused.data(), warp_fused_scales.data(), rows, n, scale, stream, api::Method::Shuffle32)); + const auto warp = warp_output.download(stream), warp_ip = warp_inplace.download(stream); + if (std::memcmp(a.data(), warp.data(), size * sizeof(T)) || std::memcmp(a.data(), warp_ip.data(), size * sizeof(T))) + throw std::runtime_error("baseline/Shuffle32/in-place transform not bitwise identical: " + context); + if (warp_split.download(stream) != expected_q.packed || warp_fused.download(stream) != expected_q.packed || + warp_split_scales.download(stream) != expected_q.scales || warp_fused_scales.download(stream) != expected_q.scales) + throw std::runtime_error("CPU/baseline/Shuffle32 split/fused INT4 bytes or scales mismatch: " + context); + x.download(stream, true, "after-shuffle32"); + ++summary.shuffle32_cases; + summary.exact_shuffle32_elements += size; +#endif + ++summary.cases; + summary.elements += size; + summary.exact_transform_elements += size; +} + +template void contract_tests(musaStream_t stream, Summary& summary) { + Guarded input(64, stream, "contract.input"), output(64, stream, "contract.output"); + Guarded packed(32, stream, "contract.packed"); + Guarded scales(8, stream, "contract.scales"); + input.download(stream, true, "initialized"); + output.download(stream, true, "initialized"); + packed.download(stream, true, "initialized"); + scales.download(stream, true, "initialized"); + auto reject = [&](musaError_t status) { + if (status != musaErrorInvalidValue) throw std::runtime_error("invalid API input did not return musaErrorInvalidValue"); + ++summary.contract_checks; + }; + auto success = [&](musaError_t status) { CHECK(status); ++summary.contract_checks; }; + for (const auto method : active_methods()) { + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=invalid-parameters" << std::endl; + for (int n : {0, 3, 512}) { + reject(api::transform(input.data(), output.data(), 1, n, 1, stream, method)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, n, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, n, 1, stream, method)); + } + for (float scale : {0.0f, -1.0f, std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { + reject(api::transform(input.data(), output.data(), 1, 8, scale, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, scale, stream, method)); + } + reject(api::transform(static_cast(nullptr), output.data(), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), static_cast(nullptr), 1, 8, 1, stream, method)); + reject(api::transform(input.data(), input.data() + 1, 1, 8, 1, stream, method)); + reject(api::transform(input.data(), output.data(), std::numeric_limits::max(), 256, 1, stream, method)); + reject(api::quantize_int4(input.data(), reinterpret_cast(input.data()), scales.data(), 1, 8, stream, method)); + reject(api::transform_int4(input.data(), packed.data(), nullptr, 1, 8, 1, stream, method)); + auto* odd = reinterpret_cast(reinterpret_cast(input.data()) + 1); + reject(api::transform(odd, output.data(), 1, 8, 1, stream, method)); + auto* bad_scale = reinterpret_cast(reinterpret_cast(scales.data()) + 2); + reject(api::quantize_int4(input.data(), packed.data(), bad_scale, 1, 8, stream, method)); + success(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, method)); + success(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, method)); + success(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, method)); + input.download(stream, true, "after-invalid-and-zero-rows"); + output.download(stream, true, "after-invalid-and-zero-rows"); + packed.download(stream, false, "after-invalid-and-zero-rows"); + scales.download(stream, false, "after-invalid-and-zero-rows"); + // 正负半整数：预期手写，避免舍入测试仅复用 CPU 参考实现。 + const std::vector ties{7, -7, .5f, 1.5f, 2.5f, -.5f, -1.5f, -2.5f}; + std::vector t(64, rounded(0)); + for (std::size_t i = 0; i < ties.size(); ++i) t[i] = rounded(ties[i]); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-upload" << std::endl; + input.upload(t, stream); + input.download(stream, true, "after-ties-upload"); + std::cout << "CONTRACT_PROGRESS method=" << method_name(method) << " phase=ties-quantize" << std::endl; + CHECK(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, method)); + const auto q = packed.download(stream, false, "after-ties-quantize"); + const auto s = scales.download(stream, false, "after-ties-quantize"); + const std::array expected{{0x97, 0x20, 0x02, 0xee}}; + if (!std::equal(expected.begin(), expected.end(), q.begin()) || s[0] != 1.0f) + throw std::runtime_error("positive/negative ties-to-even test failed"); + input.download(stream, true, "after-ties-quantize"); + ++summary.contract_checks; + // A rounded FP32 division lies just below the half-integer boundary. + // This checks the quantization contract independently of the CPU helper. + std::vector boundary(64, rounded(0)); + boundary[0] = rounded(0.75f); + boundary[1] = rounded(0.25f); + input.upload(boundary, stream); + CHECK(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 2, 1.0f, stream, method)); + const auto boundary_q = packed.download(stream); + const auto boundary_s = scales.download(stream); + if (boundary_q[0] != 0x37 || boundary_s[0] != 1.0f / 7.0f) + throw std::runtime_error("FP32 division midpoint regression failed"); + ++summary.contract_checks; + // Regression for the observed one-ULP row-scale difference on S4000. + boundary.assign(64, rounded(0)); + boundary[0] = rounded(0.5009765625f); + input.upload(boundary, stream); + CHECK(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 1, stream, method)); + const auto scale_witness = scales.download(stream); + const auto nibble_witness = packed.download(stream); + if (scale_witness[0] != read(boundary[0]) / 7.0f || nibble_witness[0] != 0x07) + throw std::runtime_error("single-value scale RNE regression failed"); + ++summary.contract_checks; + } + const auto invalid = static_cast(-1); + reject(api::transform(input.data(), output.data(), 1, 8, 1, stream, invalid)); + reject(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, invalid)); + reject(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, invalid)); +#if !defined(HADAMARD_MOORE_SHUFFLE32) + auto unsupported = [&](musaError_t status) { + if (status != musaErrorNotSupported) throw std::runtime_error("uncompiled Shuffle32 path did not return musaErrorNotSupported"); + ++summary.unsupported_shuffle32_checks; + }; + unsupported(api::transform(input.data(), output.data(), 1, 8, 1, stream, api::Method::Shuffle32)); + unsupported(api::quantize_int4(input.data(), packed.data(), scales.data(), 1, 8, stream, api::Method::Shuffle32)); + unsupported(api::transform_int4(input.data(), packed.data(), scales.data(), 1, 8, 1, stream, api::Method::Shuffle32)); + CHECK(api::transform(static_cast(nullptr), static_cast(nullptr), 0, 8, 1, stream, api::Method::Shuffle32)); + CHECK(api::quantize_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, stream, api::Method::Shuffle32)); + CHECK(api::transform_int4(static_cast(nullptr), nullptr, nullptr, 0, 8, 1, stream, api::Method::Shuffle32)); + summary.unsupported_shuffle32_checks += 3; +#endif + CHECK(musaStreamSynchronize(stream)); + output.download(stream, true); +} + + +template Summary validate(musaStream_t stream, const char* dtype, const Options& options) { + Summary result; + const std::vector dims = options.custom_shape ? std::vector{options.dim} + : (options.quick ? std::vector{1, 64, 256} : std::vector{1, 2, 4, 8, 16, 32, 64, 128, 256}); + const std::vector rows = options.custom_shape + ? std::vector{checked_shape(options.batch, options.seq, options.heads, options.dim)} + : (options.quick ? std::vector{3} : std::vector{1, 3, 17, 257}); + for (int n : dims) { + for (auto r : rows) { + for (int normalized = 0; normalized < (n == 1 ? 1 : 2); ++normalized) { + const float scale = normalized ? 1.0f / std::sqrt(static_cast(n)) : 1.0f; + for (int pattern = 0; pattern < 5; ++pattern) { + // 零值/脉冲无随机性，只计一次；N=1 的两种 scale 相同，也只计一次。 + const int seeds = pattern < 3 && !options.quick ? 3 : 1; + for (int seed = 0; seed < seeds; ++seed) + one_case(stream, r, n, scale, pattern, 123 + 7919 * seed, dtype, result); + } + } + } + std::cout << "VALIDATION_PROGRESS dtype=" << dtype << " n=" << n << " cases=" << result.cases << std::endl; + } + if (!options.quick && !options.custom_shape) { + // 超过 65535 个 block 的网格上限，让同一个 block 必须处理下一行。 + for (int n : {1, 2}) one_case(stream, 65537, n, 1.0f, 0, 1847, dtype, result); +#if defined(HADAMARD_MOORE_SHUFFLE32) + // Eight row groups per CTA: 524281 covers grid reuse and an incomplete CTA. + for (int n : {1, 2}) { + one_case(stream, 524281, n, 1.0f, 0, 4541, dtype, result); + ++result.shuffle32_grid_stride_cases; + result.shuffle32_grid_stride_elements += 524281u * n; + } +#endif + } + contract_tests(stream, result); + std::cout << "VALIDATION_PASS dtype=" << dtype << " cases=" << result.cases << " elements=" << result.elements + << " max_rounded_error=" << std::setprecision(12) << result.max_rounded_error + << " max_unrounded_error=" << result.max_unrounded_error << " contract_checks=" << result.contract_checks + << " shuffle32_cases=" << result.shuffle32_cases << " exact_shuffle32_elements=" << result.exact_shuffle32_elements + << " shuffle32_grid_stride_cases=" << result.shuffle32_grid_stride_cases + << " unsupported_shuffle32_checks=" << result.unsupported_shuffle32_checks << std::endl; + return result; +} + +template void benchmark(musaStream_t stream, const char* dtype, const Options& o, std::ofstream& csv) { + struct Shape { std::size_t b, s, h; int n; }; + std::vector shapes; + if (o.custom_shape) shapes.push_back({o.batch, o.seq, o.heads, o.dim}); + else for (int n : {64, 128, 256}) for (auto rows : {1, 17, 257, 4096, 16384}) + shapes.push_back({rows >= 4096 ? static_cast(rows / 1024) : 1, + rows >= 4096 ? 64 : static_cast(rows), rows >= 4096 ? 16u : 1u, n}); + struct Configuration { api::Method method; int operation; const char* name; }; + // 无 Shuffle32 宏时保留原来的 6 路顺序；启用后追加 3 路，独立输出新的原始样本。 + std::vector configurations{ + {api::Method::Baseline, 0, "baseline_transform"}, {api::Method::Optimized, 0, "optimized_transform"}, + {api::Method::Baseline, 1, "baseline_split"}, {api::Method::Optimized, 1, "optimized_split"}, + {api::Method::Baseline, 2, "baseline_fused"}, {api::Method::Optimized, 2, "optimized_fused"}}; +#if defined(HADAMARD_MOORE_SHUFFLE32) + configurations.push_back({api::Method::Shuffle32, 0, "shuffle32_transform"}); + configurations.push_back({api::Method::Shuffle32, 1, "shuffle32_split"}); + configurations.push_back({api::Method::Shuffle32, 2, "shuffle32_fused"}); +#endif + const int configuration_count = static_cast(configurations.size()); + for (const auto shape : shapes) { + const auto rows = checked_shape(shape.b, shape.s, shape.h, shape.n), count = rows * shape.n; + const auto input = make_input(rows, shape.n, 0, 2909); + Guarded x(count, stream), y(count, stream); + Guarded q(rows * ((shape.n + 1) / 2), stream); + Guarded s(rows, stream); + x.upload(input, stream); + const float scale = 1.0f; + auto launch = [&](int which) { + const auto cfg = configurations[which]; + if (cfg.operation < 2) CHECK(api::transform(x.data(), y.data(), rows, shape.n, scale, stream, cfg.method)); + if (cfg.operation == 1) CHECK(api::quantize_int4(y.data(), q.data(), s.data(), rows, shape.n, stream, cfg.method)); + if (cfg.operation == 2) CHECK(api::transform_int4(x.data(), q.data(), s.data(), rows, shape.n, scale, stream, cfg.method)); + }; + // Freeze an output reference before timing and verify every method separately. + CHECK(api::transform(x.data(), y.data(), rows, shape.n, scale, stream, api::Method::Baseline)); + const auto baseline_values = y.download(stream); + std::vector reference_values(count); + for (std::size_t i = 0; i < count; ++i) reference_values[i] = read(baseline_values[i]); + const auto expected_quantized = hadamard::quantize_int4(reference_values, shape.n); + // Full matrices use the all-element dense oracle; large timing shapes use + // three independently checked rows plus full-array method/INT4 comparisons. + for (std::size_t row : {std::size_t(0), rows / 2, rows - 1}) { + std::vector sample(shape.n); + for (int i = 0; i < shape.n; ++i) sample[i] = read(input[row * shape.n + i]); + const auto dense = hadamard::dense_reference(sample, shape.n, scale); + for (int i = 0; i < shape.n; ++i) { + const double expected = read(rounded(static_cast(dense[i]))); + if (!(std::abs(reference_values[row * shape.n + i] - expected) + < (std::string(dtype) == "fp16" ? .01 : .05))) + throw std::runtime_error("benchmark reference dense oracle mismatch"); + } + } + auto verify_output = [&](int which) { + const auto cfg = configurations[which]; + if (cfg.operation != 2) { + const auto values = y.download(stream); + if (std::memcmp(values.data(), baseline_values.data(), count * sizeof(T))) + throw std::runtime_error(std::string("benchmark transform mismatch: ") + cfg.name); + } + if (cfg.operation != 0) { + if (q.download(stream) != expected_quantized.packed || s.download(stream) != expected_quantized.scales) + throw std::runtime_error(std::string("benchmark INT4 mismatch: ") + cfg.name); + } + x.download(stream, true); + }; + for (int which = 0; which < configuration_count; ++which) { + launch(which); + verify_output(which); + } + for (int which = 0; which < configuration_count; ++which) for (int i = 0; i < 10; ++i) launch(which); + CHECK(musaStreamSynchronize(stream)); + musaEvent_t begin, end; + CHECK(musaEventCreate(&begin)); CHECK(musaEventCreate(&end)); + for (int group = 0; group < o.groups; ++group) { + // 各组轮换方法顺序；两端事件之间无分配、CPU 参考或主机设备复制。 + for (int order = 0; order < configuration_count; ++order) { + const int which = (order + group) % configuration_count; + const auto cfg = configurations[which]; + CHECK(musaEventRecord(begin, stream)); + for (int i = 0; i < o.repeats; ++i) launch(which); + CHECK(musaEventRecord(end, stream)); + CHECK(musaEventSynchronize(end)); + float elapsed = 0; + CHECK(musaEventElapsedTime(&elapsed, begin, end)); + const double us = static_cast(elapsed) * 1000.0 / o.repeats; + if (!(us > 0) || !std::isfinite(us)) throw std::runtime_error("invalid event timing"); + const std::size_t logical_bytes = cfg.operation == 0 ? count * sizeof(T) * 2 + : (cfg.operation == 1 ? count * sizeof(T) * 3 : count * sizeof(T)) + rows * ((shape.n + 1) / 2) + rows * sizeof(float); + csv << dtype << ',' << shape.b << ',' << shape.s << ',' << shape.h << ',' << shape.n << ',' << rows + << ',' << cfg.name << ',' << group << ',' << order << ',' << o.repeats << ',' << std::setprecision(12) << us + << ',' << logical_bytes << ',' << logical_bytes / us / 1000.0 << ',' << count * sizeof(T) + << ",2909,true,1," << us / 1000.0 << '\n'; + csv.flush(); + if (group + 1 == o.groups) verify_output(which); + } + } + CHECK(musaEventDestroy(begin)); CHECK(musaEventDestroy(end)); + std::cout << "BENCHMARK_PROGRESS dtype=" << dtype << " rows=" << rows << " n=" << shape.n << std::endl; + } +} + +void write_summary(std::ostream& f, const char* dtype, const Summary& s) { + f << '"' << dtype << "\":{\"cases\":" << s.cases << ",\"elements\":" << s.elements + << ",\"exact_baseline_optimized_elements\":" << s.exact_transform_elements + << ",\"shuffle32_cases\":" << s.shuffle32_cases << ",\"exact_baseline_shuffle32_elements\":" << s.exact_shuffle32_elements + << ",\"shuffle32_grid_stride_cases\":" << s.shuffle32_grid_stride_cases + << ",\"shuffle32_grid_stride_elements\":" << s.shuffle32_grid_stride_elements + << ",\"unsupported_shuffle32_checks\":" << s.unsupported_shuffle32_checks + << ",\"max_abs_error_rounded_fp64\":" << std::setprecision(15) << s.max_rounded_error + << ",\"max_abs_error_unrounded_fp64\":" << s.max_unrounded_error + << ",\"api_contract_checks\":" << s.contract_checks << '}'; +} + +int main(int argc, char** argv) { + Options options; + try { options = parse(argc, argv); } + catch (const std::exception& e) { std::cerr << "INVALID_ARGUMENT " << e.what() << '\n'; return 2; } + try { + CHECK(musaSetDevice(0)); + musaDeviceProp prop{}; + CHECK(musaGetDeviceProperties(&prop, 0)); +#if defined(HADAMARD_MOORE_SHUFFLE32) + if (prop.warpSize != 128) throw std::runtime_error("This validated S4000 build requires host-reported warpSize=128"); +#endif + int runtime = 0, driver = 0; + CHECK(musaRuntimeGetVersion(&runtime)); CHECK(musaDriverGetVersion(&driver)); + std::cout << "DEVICE name=" << prop.name << " warp=" << prop.warpSize << " runtime=" << runtime << " driver=" << driver << std::endl; + musaStream_t stream; + CHECK(musaStreamCreateWithFlags(&stream, musaStreamNonBlocking)); + Summary fp16, bf16; + if (options.validate) { + if (options.dtype != "bf16") fp16 = validate<__half>(stream, "fp16", options); + if (options.dtype != "fp16") bf16 = validate<__mt_bfloat16>(stream, "bf16", options); + std::ofstream json(options.json); + if (!json) throw std::runtime_error("cannot create validation JSON " + options.json); + json << "{\"status\":\"PASS\",\"platform\":\"moore\",\"warp_size\":" << prop.warpSize << ",\"full_matrix\":" << ((!options.quick && !options.custom_shape && options.dtype == "both") ? "true" : "false") +#if defined(HADAMARD_MOORE_SHUFFLE32) + << ",\"shuffle32_enabled\":true,\"methods\":[\"baseline\",\"optimized\",\"shuffle32\"]" +#else + << ",\"shuffle32_enabled\":false,\"methods\":[\"baseline\",\"optimized\"]" +#endif + << ",\"oracle\":\"all-element FP64 dense, rounded to output dtype\",\"fp16_tolerance_strict\":0.01,\"bf16_tolerance_strict\":0.05," + << "\"warmup_not_counted\":true,\"dtypes\":{"; + bool comma = false; + if (options.dtype != "bf16") { write_summary(json, "fp16", fp16); comma = true; } + if (options.dtype != "fp16") { if (comma) json << ','; write_summary(json, "bf16", bf16); } + json << "}}\n"; + if (!json) throw std::runtime_error("failed writing validation JSON"); + } + if (options.benchmark) { + std::ofstream csv(options.csv); + if (!csv) throw std::runtime_error("cannot create benchmark CSV " + options.csv); + csv << "dtype,batch,seq,heads,dim,rows,method,group,order,repeats,kernel_us,logical_io_bytes,logical_GBs,input_working_set_bytes,seed,input_read_only,scale,kernel_ms\n"; + if (options.dtype != "bf16") benchmark<__half>(stream, "fp16", options, csv); + if (options.dtype != "fp16") benchmark<__mt_bfloat16>(stream, "bf16", options, csv); + if (!csv) throw std::runtime_error("failed writing benchmark CSV"); + } + CHECK(musaStreamDestroy(stream)); + std::cout << "PASS requested validation/benchmark operations completed" << std::endl; + return 0; + } catch (const std::exception& e) { + std::cerr << "FAIL " << e.what() << std::endl; + return 1; + } +} + diff --git a/03_hadamard_tc/a962695448-rgb/reports/4090d-routing-20260916.md b/03_hadamard_tc/a962695448-rgb/reports/4090d-routing-20260916.md new file mode 100644 index 00000000..e869271c --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/4090d-routing-20260916.md @@ -0,0 +1,38 @@ +# RTX 4090 D 的限定范围 auto 路由 + +原版本在 4090 D 上采用未知设备回退。剖析发现低维、大批量输入可受益于已有 packed kernel，因此增加精确设备规则： + +- 设备名：`NVIDIA GeForce RTX 4090 D`。 +- 接口：`hadamard` / `hadamard_int4` 的 `row_layout="auto"`。 +- 维度：1、2、4、8、16；行数：4096～65536（含边界）。 +- 选择 packed + 256 线程；范围外回退原路径，原有其他设备规则不变。 + +默认 `original` 及计算内核未变，`quantize_int4` 没有增加 auto 选项。该优化需要调用者显式选择 auto；普通逐次 Python 调用的收益应按实际工作负载另测。 + +## 实测 + +在 RTX 4090 D / NGC Torch 2.6 / CUDA 12.8 上先完成 480 组探索剖析，再冻结门槛与新种子进行确认： + +| 检查 | 结果 | +|---|---| +| 配置 | 120 个目标、60 个回退对照；180 组配置 × 三轮 | +| 配对 | 540 组通过，每组九次旧/新交替，共 9,720 个计时值 | +| 目标加速比 | 三轮各配置中位数为 1.312～7.528× | +| 回退对照 | 最大回退 2.176%，低于预设 3% 上限 | +| 正确性 | FP16/BF16 输出与独立 NumPy 参考逐位一致，输入不变 | +| 新路由执行上下文 | 16 组流/Graph 配置通过，含 64 次变更输入重放与 32 个错误对照 | +| CPU | reference 通过，270 个路由边界检查通过 | + +每个目标每轮要求至少快 5%，全部达到。门槛、种子、留出及边界形状在候选计时之前冻结；不将重复轮次算成更多独立配置。 + +计时是在常驻数据上，用 32 次调用组成的 Graph 重放 20 遍，以 CUDA event 测量设备执行时间。该比值不是默认 original 调用、主机墙钟或大模型端到端加速，也不适用于其他 GPU 或未验证的批量范围。探索中的 CPU/CUDA 时间线不等于硬件性能计数器，未据此声称测得 DRAM 带宽、寄存器或占用率瓶颈。 + +## 使用与复现 + +```python +# x 为受支持的 CUDA FP16/BF16 连续张量。 +y = op.hadamard(x, scale=1.0, row_layout="auto") +packed, scales = op.hadamard_int4(x, scale=1.0, row_layout="auto") +``` + +`make cpu-test` 包含新路由边界测试。基线提交为 `c591085756fec139c55f3de59c2a4ba0742dc5cc`；[固定完整证据](https://github.com/a962695448-rgb/Learning-CUDA/tree/354cc6fd6a70f80a9d3e893f0db731b27fdabbbc/03_hadamard_tc/a962695448-rgb/results/4090d-routing-20260916)保存探索时间线、确认协议、各次计时、错误对照、源码及 SHA-256，以及可独立复现的脚本。当前清单记录 Makefile/row_policy 的新指纹并保留原档案指纹；其余原有 49 份源码/构建文件保持不变。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/a100-calling-validation.md b/03_hadamard_tc/a962695448-rgb/reports/a100-calling-validation.md new file mode 100644 index 00000000..9dfe7640 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/a100-calling-validation.md @@ -0,0 +1,94 @@ +# A100 自动选择与 Python 调用方式复核 + +本轮实际生产源码为 `24dfef776ad73a4128cb6138674c5886c21e49c0`。在 **A100-SXM4-40GB** 与 **RTX 4090** 分别重新构建、验证；默认 `row_layout="original"` 保留。变更仅在 `row_policy.hpp` 增加 A100 型号识别和已通过独立验证的规则，没有改写数学内核、量化语义或现有 API 签名。 + +完整原始记录、失败、复算脚本及哈希见[本轮档案](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_calling_20260908/README.md)。此前 A800/4090 的 [a3703fd 阶段记录](packed-rows-validation.md)独立保留，不用本轮结果覆盖旧负例。 + +## A100 规则的来源 + +先在 A100 对冻结的 a3703fd 生产源完成正确性复验，再用原有候选、输入与计时协议独立执行筛选及留出。独立候选实验的基线为53e7b86，最终生产集成为24dfef7，两阶段的源码和二进制分别记录。450 个正确性配置各含 8 种输入/偏移条件，另有 10 条四维/stream 条件；初筛 216 项变换/融合比较，留出 360 项比较，均三个独立进程。计时对照是同轮更快的 original128/256。 + +初筛完成后冻结规则，再读取留出结果。每条规则必须在所有被其激活的预定留出配置、每轮均减少至少 5% 耗时。最终接受 9 条，拒绝 N16 融合；没有用同一留出集重新拟合阈值。具体原始规则及拒绝项见 `a100/calibration/runs/policy_validation.json`。 + +| A100 路径 | N=1 | N=2 | N=4 | N=8 | N=16 | +|---|---:|---:|---:|---:|---:| +| 变换，启用 packed 的最小行数 M | 4096 | 256 | 4096 | 4096 | 4096 | +| 融合 INT4，启用 packed 的最小行数 M | 1 | 1 | 16 | 64 | 不启用 | + +启用时为 256 线程。M 是展平后的行数，N 是最后一维。其他情况保留调用者指定的 128/256 线程。该规则只识别 `NVIDIA A100-SXM4-40GB`；A100 80GB、其他型号及名称不匹配的设备不自动继承它。已有 A800/4090 规则未改。 + +N16 融合初筛选择的候选是 128 线程、M≥64，但 M=65 的多个留出配置仅减少约 2.78%～4.08%，不满足门槛，未安装该规则。阈值代码能作用于更大的 M，但实测仍只覆盖保存的有限形状与输入，不能保证任意规模都更快。 + +## 最终生产验证 + +两卡均使用最终 24dfef7 源码，在独立目录运行以下检查： + +- 同一 1876 项 CLI 矩阵分别执行 original128、original256、contiguous256、packed128、packed256、auto128、auto256。重复模式与跨卡执行不累计为更多独立输入。 +- 原 1800 项固定 Dao 参考对照、280 项大行数/偏移/非默认 stream 条件、24 项行布局 API 拒绝检查、28 项元数据检查、6 项新增 CLI 拒绝检查全部通过；原有 CLI 拒绝矩阵仍运行。 +- 21/24 列 CSV、毫秒换算、split 独立量化线程及混合表头拒绝通过。数学内核和参考库容差未改变。 +- 主机 C++ 对照检查 18216 个原有设备/布局/形状/线程组合的新旧选择结果一致，另有 5 项 A100 型号与边界断言。它是 CPU 选择逻辑验证，不是新增 GPU 用例。 +- A100 的 160 条原生安全夹具通过，memcheck 与 synccheck 均为零错误。4090 本轮未重复这套独立夹具，其此前记录仍单独保留。 + +最初 A100 启动脚本遗漏虚拟环境的 bin 路径，导致扩展构建找不到已安装的 Ninja，原 `production/` 记录为 FAIL。修正 PATH 后在 `production_retry/` 完整复验通过，未安装替代依赖、修改源码或放宽数值标准。最终 24dfef7 的运行在 `final/production/`，三者分别保留。 + +## Graph 结果 + +每方法捕获 64 份独立且互不重叠的输出，正式计时前重放 256 次；每组 20 次重放、共 5 组，轮换执行顺序，三个独立进程重复。CUDA event 毫秒除以 `64×20` 得到每调用均摊时间。输出在计时前后均核验。此口径包含 GPU 工作和均摊重放调度，不是独立单 kernel 延迟。 + +最终矩阵使用 FP16/BF16、scale=1，包含各规则的阈值、阈值+1、较大行数，以及阈值-1和不支持维度的回退控制。 + +| GPU 与比较范围 | 配置×进程 | 每轮均减少≥5%的配置 | 耗时减少完整范围 | +|---|---:|---:|---:| +| A100 启用项，相对同轮更快的 original128/256 | 64×3 | 64/64 | 5.2446%～68.5562% | +| RTX 4090 启用项，相对同轮更快的 original128/256 | 62×3 | 62/62 | 5.5101%～66.7383% | +| A100 回退项，相对 original128 | 24×3 | 0/24 | -0.2997%～0.1234% | +| RTX 4090 回退项，相对 original128 | 24×3 | 0/24 | -0.3084%～0.1546% | + +负数表示退化。5% 是工程采用门槛，不是统计显著性或跨日期置信区间。没有应用工作负载权重，不能由此计算大模型端到端收益；两卡数值分别统计，不跨卡相除。 + +## 普通 Python 调用的开销 + +先对 a3703fd 的 A100 运行做控制实验：该版本在 A100 的 auto 仍回退原路径，因此可比较同一内核下的调用写法。12 个配置、7 种写法、三个进程，每方法每组预热 200 次，计时 2000 次分配型 API 调用，共 9 组；以同步包围的主机墙钟记录。将同一 auto 功能从关键字形式改为位置参数形式，12 个配置每轮均减少至少 5%，范围 5.54%～10.80%。这支持参数传递写法是该实验中明显的额外开销来源，不证明旧 A800/4090 运行的所有波动都已解释。 + +最终 24dfef7 又在两卡重复相同的 12 配置、7 写法、三进程对照：N1/M1 融合、N4/M17 融合、N16/M4096 变换、N16/M65 融合、N8/M4096 融合、N2/M4096 变换，分别测两种精度，scale=1。 + +| 比较 | A100 耗时减少范围 | RTX 4090 耗时减少范围 | +|---|---:|---:| +| 同一 auto 功能：位置参数相对关键字参数 | 6.2160%～9.4916% | 5.9455%～12.3069% | +| 位置参数 auto 相对原默认三参数 original 调用 | -2.9632%～0.5794% | -3.4531%～0.2805% | + +第一行两卡均为 12/12 配置每轮至少减少 5%。第二行仍有额外成本，因此不将 auto 改为默认，也不宣称普通逐次调用已经稳定快过原路径。这是已有接口的使用方式改进，没有新增专用函数或改写输出结果。基准缓存方法引用，各方法使用一致的计时流程。 + +在项目目录内，下面是可直接运行的位置参数示例；首次调用构建扩展的时间不属于上述计时： + +```python +import torch +from scripts.build_torch_extension import load_extension + +op = load_extension() +x = torch.randn((4096, 8), device="cuda", dtype=torch.float16) +y = op.hadamard(x, 1.0, 128, "auto") +packed, scales = op.hadamard_int4(x, 1.0, 128, "original", "auto") +``` + +关键字写法继续支持，适合强调参数含义；普通小输入不应仅凭 Graph 表强制切换 auto。真实应用需要按其调用、分配、复制和同步方式另测。 + +## 时间线与计数器 + +两卡均获得 nsys 原始时间线。A100 的 N2/M256 变换调用 packed；N2/M255 变换回退 warp，但同形状融合仍调用 packed，因为融合 N2 的阈值是 M≥1。最初分析脚本错误地要求整个 M255 复合 benchmark 都没有 packed kernel，原检查报告保留 FAIL。独立 `trace-scope-audit.json` 按模板参数分别核对变换和融合，确认实际执行符合规则；没有重写旧日志或改动内核。 + +4090 的 N8/M4096 活动控制、N16/M65 回退控制也通过执行路径检查。Profiler 时间只用于其自身时间线分析，不替代正式基准。两卡 ncu 仍实际报 `ERR_NVGPUCTRPERM`，没有带宽利用率、occupancy 等硬件计数器结果。 + +## 离线复算与边界 + +原始事件、组样本、每轮中位数和全部负例均保存为 JSON，并提供毫秒 CSV。复算脚本只需 Python 标准库；复制到新目录执行，避免改写归档中的派生文件： + +```bash +# 在 results/nvidia_a100_calling_20260908 内 +WORK=$(mktemp -d) +cp analyze_final.py a100-final-metrics.zip rtx4090-final-metrics.zip "$WORK/" +python "$WORK/analyze_final.py" a100 +python "$WORK/analyze_final.py" rtx4090 +``` + +脚本核对原始区间换算、组中位数、三轮配置集合、二进制与验证报告指纹，并输出完整比较表。原生 nsys/SQLite 与运行会话元数据留在私有完整备份，公开副本逐项记录省略原因及哈希。新文档提交不视为新一轮 GPU 测试。本轮没有双 GPU 验证、全场景最优证明或官方评审结果；PR 与正式提交仍待项目所有者验收。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/a100-validation.md b/03_hadamard_tc/a962695448-rgb/reports/a100-validation.md new file mode 100644 index 00000000..bc040c5f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/a100-validation.md @@ -0,0 +1,71 @@ +# Hadamard CUDA：A100跨卡验收报告 + +## 2026-09-08 A100 补充验证 + +现有融合源码已在 A100 sm80 重新编译，完整回归及三轮 52 配置验证通过。51/52 三轮更快，47/52 每轮至少减少 5%；一配置第三轮退化 8.43%，后续预热诊断没有重现，但不覆盖原负例。默认 original 保留。见[A100 报告](fused-layout-a100-validation.md)和[原始数据](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_fused_a100_20260908/README.md)。以下历史实验和适用范围分别保留。 + +**固定源码已在真实A100重新编译并完成全部预定验证。**源码为`12c76d8331ef7cf3fd4c8c14a049162559be4302`，参考库为`e7706faf8d1c3b9f241e36860640ad1dac644ede`。完整公开证据、源码和离线复算入口见[档案首页](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_20260906/README.md)。 + +## 环境与构建 + +实际设备为NVIDIA A100-SXM4-40GB、MIG Disabled、capability `[8,0]`，驱动550.127.05。独立环境复用Conda Python3.12.7、Torch2.5.0+cu124、NumPy2.1.3，使用NVCC12.4.99、G++11.4与C++11 ABI=false单并发构建；没有升级原Torch或修改其他项目环境。 + +CLI、默认扩展、线程扩展的cuobjdump均显示`sm_80.cubin`；固定Dao含sm75/80/87/90。来源和构建参数见[源码清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_20260906/source_manifest.json)及[实际ELF检查](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_20260906/retrieved/results/collection_metadata.json)，没有将sm89产物当作A100验收。 + +| 二进制 | 实际SHA256 | +|---|---| +| CLI | `aadc59aa1e8c2ba40dd0763c89db945b0dc53f5f35699828162f1381b13c6263` | +| 默认扩展 | `4f1f44550a6e57b39ec654394f6175048bb806f6d08d1e3989473c674ce5ddb0` | +| 线程扩展 | `0857075b7199d2040457da62610a98daec4f6cd6d4ae1cb0a3972cc16737291d` | +| Dao参考 | `faa31572adfbf82c0261a27ebfaed6a2c5fb6261b81a499e9d3a9bc4819726e5` | + +## 正确性：不同范围分别记账 + +| 验证范围 | A100结果 | +|---|---| +| 原1,876组CLI矩阵 | 默认128、显式256均PASS；最大绝对误差0.0078125；CPU/分步/融合INT4 bytes与scales精确一致 | +| 原1,800组Dao矩阵 | PASS；FP16/BF16最大绝对差均0；既有10项非法张量和非默认stream检查通过 | +| 同一1,800组调用兼容验证 | 旧默认、显式128、显式256位一致；27项无效线程值拒绝通过 | +| 候选范围336个不同输入 | 三轮均通过；覆盖CPU全量量化、样本行FP64稠密参考和捕获输出核查 | + +CLI保留35个warp相对稠密舍入参考的容差内差异，没有修改期望或放宽阈值。重复1,876矩阵、三种API调用和三个进程均不计作更多独立算法用例。当前仅一张可见GPU，多GPU切换未实际覆盖。 + +## 默认128线程：原12组对照及负例 + +原12组eager API的`Dao时间/本项目时间`为2.295～2.380；事件区间含主机调用造成的GPU空档，**不能称纯kernel加速比**。 + +相同输入的64份独立输出CUDA Graph，比值为0.9895～2.1297；两个负例完整保留： + +| 形状与dtype | 本项目默认128（ms） | Dao（ms） | 本项目耗时增加 | +|---|---:|---:|---:| +| `[17,256]` FP16 | 0.0017600 | 0.0017416 | 1.06% | +| `[17,256]` BF16 | 0.0017592 | 0.0017488 | 0.59% | + +[所有12组eager/12组Graph](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_20260906/derived/dao_all_configurations.csv)包含各组样本和中位数；小差异不是跨日期统计结论。CLI另保存16组配置、110行kernel-only/CPU compute/host end-to-end数据，不能与API/Graph直接相除。 + +## 显式256线程：预定72配置三轮 + +原24个目标和48个M±1邻近配置，均在每轮相对同轮128线程减少耗时至少5%。**72/72达标，无观测退化**；全范围降时6.19%～26.09%。 + +| 原目标范围（两精度、两scale、三轮） | 耗时减少范围 | +|---|---:| +| transform，M=4096，N=16 | 13.68%～14.61% | +| transform，M=4096，N=64 | 11.07%～12.25% | +| transform，M=16384，N=16 | 25.75%～26.09% | +| transform，M=16384，N=64 | 15.17%～16.52% | +| fused INT4，M=4096，N=16 | 8.21%～8.68% | +| fused INT4，M=4096，N=64 | 6.28%～7.02% | + +邻近范围仅为上述M±1；融合未扩大到M=16384。图中每方法保留64份独立输出，25次API预热、5次图预热、每组20次重放、5组，三进程轮换测量顺序。[全部72配置](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_20260906/derived/promotion_all_configurations.csv)及三份原始JSON均保留。 + +默认仍为128，没有自动派发。正确性支持全部已实现N，但此次性能结论只支持预定N=16/64范围，不声称全局最优或其他形状必然更快。 + +## 跨卡比较与结论边界 + +A100与4090若作横向观察，只能比较一致的测量口径，例如相同64独立输出的Graph设置，或相同CLI的kernel-only；不能将A100的Graph与4090的eager、CPU或含复制时间相除。两端Torch、CUDA、CPU和时钟条件并未全部控制，不能把绝对差异仅归因于GPU型号。 + +本报告的所有加速/降时比值均取自**同轮A100内部配对**，没有用4090数值作分母。Graph时间包含均摊调度开销，既不是独立单kernel延迟，也不是端到端速度；logical tensor I/O不等同物理内存带宽。 + +运行于2026-09-06北京时间02:55:05～03:05:35。最终GPUUtil=0%、显存14MiB、compute-app列表为空，全部阶段退出0。首次归档时，48 份原始文本的大小和 SHA 均核验且保留原字节；当前公开档案保留其中 47 份原字节，另 1 份实验脚本仅整理模块说明文字。历史执行 hash 不变，文字编辑副本和派生路径处理分别记录原/公开双 SHA，见 [文字来源说明](public-wording-provenance.json) 与归档内的 public_derivation_provenance.json。没有公开访问地址、连接材料或编译二进制。 + +本报告用于核对 A100 的数值、性能与复现记录。后续按已测慢例实施优化；PR 和课程登记在项目所有者验收具体代码及证据后单独执行。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/execution-context-20260916.md b/03_hadamard_tc/a962695448-rgb/reports/execution-context-20260916.md new file mode 100644 index 00000000..897068f3 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/execution-context-20260916.md @@ -0,0 +1,26 @@ +# 非默认流与 CUDA Graph 补充验收 + +2026-09-16 在 RTX 4090 D 24GB 单卡实测。新增测试脚本，原 51 份计算/构建源码保持不变。 + +| 检查 | 结果 | +|---|---| +| 接口及配置 | 120 组通过；hadamard / hadamard_int4 / quantize_int4，FP16/BF16、128/256 线程、2D/4D | +| 非默认流 | 120 次延迟生产者与消费者正确性检查通过 | +| CUDA Graph | 每组四次变更输入后的重放，共 480 次通过 | +| 负例 | 120 个默认流消费者错误、120 个旧输出错误，全部检出 | +| 参考及输入 | NumPy 独立 FWHT/INT4 参考逐位一致，输入未被改写 | + +维度覆盖 1/8/64/256，布局覆盖合法的 original/auto/packed/contiguous256 组合。错误对照不能被检出时测试失败，避免将缺少区分能力的测试写为通过。负例、重放次数不累计为更多独立配置。 + +从本项目目录执行： + +```bash +export PATH=/usr/local/cuda/bin:$PATH +python scripts/verify_execution_context.py --build-directory /tmp/hadamard-context-build --json /tmp/new-context-report.json +``` + +输出路径必须不存在。延迟生产者测试要求 `torch.cuda._sleep` 可用；未提供或负例失效时明确失败。CUDA Graph 预热、流依赖和固定内存更新规则见 [PyTorch 2.6 官方说明](https://docs.pytorch.org/docs/2.6/notes/cuda.html)。 + +这是正确性补测，未修改内核、不宣称新的加速比。单卡代表性配置不能代替任意流调度、多 GPU 或新 A100 验证。实际环境为 NGC Torch 2.6.0a0+ecf3bae40a.nv25.01 / CUDA 12.8 / NumPy 1.26.4。 + +[固定原始证据](https://github.com/a962695448-rgb/Learning-CUDA/tree/3c0db6daa462bfba7ec0904e65d5a841064ab5b3/03_hadamard_tc/a962695448-rgb/results/execution-context-20260916)包含每组配置、每次重放、负例记录、原始日志、脚本和指纹清单。结果回收后逐文件 SHA-256 核对通过。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/fused-layout-a100-validation.md b/03_hadamard_tc/a962695448-rgb/reports/fused-layout-a100-validation.md new file mode 100644 index 00000000..f5589a89 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/fused-layout-a100-validation.md @@ -0,0 +1,32 @@ +# N256 融合布局：A100 实机复验 + +日期：2026-09-08。此轮验证源码 `35f79b9cc595b0459a6172e0e2a63dcb9f4854af` 的现有融合实现，不增加新的算子或自动派发规则。13 份生产源文件从 Git 读取，与 `155a05a` 实现按 LF 逐字核对；使用独立目录重新编译 sm80。设备为 NVIDIA A100-SXM4-40GB，驱动 550.127.05，PyTorch 2.5.0+cu124，CUDA 12.4。 + +## 正确性与接口 + +| 范围 | 实际结果 | +|---|---| +| CLI 自测 | 同一 1,876 输入矩阵在 original/128、original/256、候选融合三种模式分别通过；不记为 5,628 个不同输入 | +| 固定 Dao 参考 | 原 1,800 用例通过，其中 200 个 N256 用例额外验证候选融合；不是新增 200 个输入 | +| 前端与边界 | 28 项 Tensor 元数据、16 项非默认 stream/形状/对齐定向检查，以及全部非法参数与 CSV 合同检查通过 | +| 三轮邻近配置 | 固定 52 配置，每轮 728 种输入/对齐条件通过；FP16/BF16 对 Dao 的全元素最大绝对误差均为 0 | +| 离线独立复算 | 核验 470 个去重位模式缓冲区，重放 364 份去重精确数值证书，并复算 1,560 条原始计时 | + +原始数据与测试范围见[证据索引](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_fused_a100_20260908/README.md)。证书保持此前已审阅的 FP32 前向误差与最近偶数舍入模型；官方参考阈值未修改。旧 4090 辅助稠密舍入检查的失败与解释仍保留在[原报告](fused-layout-validation.md)。 + +## 性能与适用范围 + +候选与 original 使用相同输入、scale、128 线程及独立保留输出的 CUDA Graph。每个图保留 64 次调用的输出，计时包含 20 次重放，取五组中位数；三轮为独立进程。单位换算到毫秒，比较的是融合 API 的 Graph 区间，不是独立纯内核或端到端延迟。 + +- 51/52 个配置三轮均更快；47/52 每轮至少减少 5% 耗时。 +- 全部配置、全部轮次的耗时减少比例范围为 **-8.43% 至 12.04%**。负数表示退化，不能省略后只报正向范围。 +- BF16、M=16385、N=256、scale=1 的三轮分别为 **+7.30%、+7.30%、-8.43%**。第三轮 original 与候选的中位数分别为 0.016612 ms、0.018013 ms。 +- 第三轮原始分组中，两实现均出现从较慢到较快耗时区间的变化。没有逐组时钟/功率证据足以确定唯一原因，不将其直接解释为内核必然退化或外部干扰。 + +因此维持默认 `fused_layout="original"`；`contiguous256` 保留为 N256/128 线程显式选项。上述不稳定配置使用 original。其他未测形状、平台及应用工作负载不据此宣称更快；47/52 是预定工程采用门槛，不是统计显著性或全局最优证明。 + +## 分开的预热对照 + +在原三轮结束后，预先固定 BF16 的 M=16383/16385、scale=1/0.0625 四个配置，对照 5 与 256 次图预热；三次独立进程共保存 240 个原始计时样本，其他计时设置与算子源码保持不变。八个“配置×预热条件”均在三轮超过 5%。原负例 M=16385、scale=1：5 次预热为 +8.05%、+7.92%、+8.84%，256 次为 +7.71%、+7.83%、+7.49%。 + +此次没有重现原退化，而且短、长预热均通过，因此不能断言原异常一定是预热不足。这组诊断不替换原 52 配置的三轮数据，也不把原 47/52 稳定通过数改成 52/52。[诊断原文与逐组数据](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_fused_a100_20260908/warmup-diagnostic/analysis.json)单独归档。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/fused-layout-validation.md b/03_hadamard_tc/a962695448-rgb/reports/fused-layout-validation.md new file mode 100644 index 00000000..ef67f561 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/fused-layout-validation.md @@ -0,0 +1,55 @@ +# N256 融合布局：实现与验证 + +## 2026-09-08 A100 补充验证 + +现有融合源码已在 A100 sm80 重新编译，完整回归及三轮 52 配置验证通过。51/52 三轮更快，47/52 每轮至少减少 5%；一配置第三轮退化 8.43%，后续预热诊断没有重现，但不覆盖原负例。默认 original 保留。见[A100 报告](fused-layout-a100-validation.md)和[原始数据](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_fused_a100_20260908/README.md)。以下历史实验和适用范围分别保留。 + +源码提交：`155a05a8b957bdf558ef93a2db1e3aea9fadf36f`。新增 `fused_layout="contiguous256"`，仅支持 N256、128 线程的融合 Hadamard 与 INT4；默认 `original`、独立变换和独立量化保持原路径。完整源文件、编译、失败记录和复算入口见[证据目录](https://github.com/a962695448-rgb/Learning-CUDA/tree/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_fused_integration_20260907)。 + +## 实现与接口 + +每个线程持有相邻 8 个元素，保持原全局 bit0～7 蝶形顺序。对齐输入使用 16B 向量读取；合法 2B 偏移走标量读取。相邻量化值在本线程寄存器内打包，减少跨线程交换。输出先按 FP16/BF16 舍入，再计算 FP32 行尺度、除法和最近偶数量化，保持与分步路径相同的语义。 + +另修复公共 PyTorch 适配器对惰性负号视图读取底层正号数据的问题：明确拒绝 `is_neg`，提示先调用 `resolve_neg()`。普通负数张量保持支持。 + +## 生产回归 + +| 范围 | 结果与计数口径 | +|---|---| +| 原 CPU 参考 | 矩阵、逆变换、舍入、打包、零值及非法形状检查通过 | +| CLI 全矩阵 | 默认128、原256、候选融合各执行同一套1,876项；候选模式只有212项N256输入的融合路径切换，不累计为5,628个独立输入 | +| 固定 Dao 对照 | 原1,800项通过，其中200项N256同时核查新融合位一致；200是子集 | +| 张量元数据 | 28项惰性负号拒绝及物化正例通过 | +| 定向接口 | 16项覆盖两精度、2D/4D、scale、实际2B偏移和非默认stream | +| 参数与日志 | CLI15+17项拒绝、API27线程/5布局/10原输入拒绝；28行CSV格式和两种旧表头保护通过 | + +回归在重启前的4090完成。9月7日重启后，主机和物理GPU UUID变化；重新核验原文件、回归报告、二进制及软件环境后，在新4090完成以下V2矩阵。没有将旧回归称为新设备重跑，也没有把两次设备的时间交叉相除。 + +## 52 个邻近配置 + +固定 M={2,3,16,18,63,65,255,256,258,4095,4097,16383,16385}，N256，两种精度和 scale={1,1/16}，共52配置。三独立进程使用相同生产二进制，每个进程先完成728个输入/对齐条件的验证，再计时。 + +| 精度 | 三轮均更快 | 三轮每轮降时≥5% | +|---|---:|---:| +| FP16 | 26/26 | 15/26 | +| BF16 | 26/26 | 26/26 | +| 合计 | 52/52 | 41/52 | + +相对同轮原融合布局，耗时减少 **4.7837%～14.3440%**，没有退化配置。其余11项仍为正收益，但未全部达到5%门槛。1,560个原始组样本完整保留；470份唯一uint16位数组、364份唯一CPU数值证书已离线复算。 + +计时为64份独立输出CUDA Graph，每组20次重放、5组取中位数；包含捕获GPU工作和均摊调度开销，不是独立kernel或端到端延迟。结论仅覆盖固定输入域与当前4090，不自动派发，不保证任意M或其他GPU同样受益。新融合布局尚无A100实测。 + +## 原失败与修订口径 + +V1在FP16 M16383、normal seed95811的辅助FP64已舍入检查处停止，**状态仍为FAIL，性能条目为零**。第8191行第230列： + +| 对象 | 值 | +|---|---:| +| 精确数学值 | −75333627/4194304，约−17.9609363079 | +| CPU逐阶段FP32值 | −17.9609375，恰在FP16中点 | +| 项目与固定Dao输出 | −17.96875（cc7e） | +| 精确值直接舍入到FP16 | −17.953125（cc7d） | + +项目与Dao在该输入全部4,194,048个元素上逐位一致。中点的最近偶数舍入解释了一个FP16 ULP（0.015625）的差异；本例不是双舍入问题。两项不同参考的0.01门槛在该存储网格上不能同时满足。 + +V2单独冻结并保留同一矩阵、种子、计时和参考库0.01/0.05阈值，补充全元素Dao对照。辅助数学检查以整数FWHT和Fraction精确值为依据，对CPU逐阶段FP32模型检验固定前向误差界；独立整数RNE必须等于实际GPU样本位，再计入实际存储舍入误差。FP64差异完整报告，不被加进预算放宽门槛。V2的通过不追认V1通过，也不意味着改变过内核数学。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/host-attribution-20260917.md b/03_hadamard_tc/a962695448-rgb/reports/host-attribution-20260917.md new file mode 100644 index 00000000..0d6d1551 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/host-attribution-20260917.md @@ -0,0 +1,32 @@ +# 普通调用耗时的交叉诊断 + +本轮针对此前两版候选的普通调用慢例，检验模块、输出缓冲和测量顺序的影响。**没有采用新计算内核，也没有改变原来的 REJECT 判定。** + +## 发现与结果 + +旧报告中最严重的 FP16 fused / rows=16387 / N2 allocating 案例，两个独立中位数相除为 **0.809675**；但 9 组成对记录中有 5 组候选更快，组内比值中位数为 **1.007713**。两侧早期样本都明显波动。这不足以证明稳定慢约 23%，也不足以直接翻转旧判定。 + +新诊断使用三个独立进程、四种配置、五种调用/缓冲条件，每组交叉交换三个模块的次序和输出池。两个控制模块由**同一源码路径、相同参数、等长不同模块名**编译。18 组覆盖六种测量次序 × 三种缓冲轮换，每块 5000 次，共 **3240 个计时块**。 + +所有模块/输出池组合通过独立 CPU 数值核对。20 种条件的同源 A/A 均在预设 5% 范围内，未观察到跨三个进程、同时满足两种统计口径的稳定超过 5% 候选退化或加速。 + +| 对照 | 独立中位数比范围 | 组内配对比值中位数范围 | +|---|---:|---:| +| 同源码控制 A / 控制 B | 0.975662–1.032983 | 0.973501–1.040054 | +| 控制 A / 候选 | 0.963083–1.036172 | 0.965565–1.034012 | + +比值定义为左侧耗时/右侧耗时。相同源码的模块也能产生几个百分点差异，不能把所有变化直接归于算法。该实验没有唯一确定 GC、调度、地址布局或具体指令为旧慢例的原因。 + +## 计时与证据边界 + +记录包括同步区间墙钟、进程和当前线程 CPU 时间、时间线、调度/缺页/GC 计数及可用 CPU 频率。墙钟包含区间内等待 GPU 完成的时间，不能直接解释为纯 CPU 指令成本。 + +本次仍为 RTX4090，但重启后平台显示的 PCI 地址与前次不同；结论限定在本轮内部对照，不直接用两个会话的绝对时间相除。四种诊断配置也不能替代原来完整 588 设备 + 588 普通调用矩阵的采纳验证。 + +静态汇编中，公开融合入口的 opcode 和直接调用名称序列相同，被调用的分派函数结构则发生变化。指令计数包含冷/错误路径，不是动态性能剖析。 + +[固定源码、原始时间线、复算程序及合成测试](https://github.com/a962695448-rgb/Learning-CUDA/tree/dbe195f4a4c60947e42eb11cafe8aab689c6b7bc/03_hadamard_tc/a962695448-rgb/results/host-attribution-20260917)已归档。14 份结果回收后逐文件校验，26 份输入前后相同，实例已关机。 + +## 后续实施方向 + +下一项实现实验可保留现有 kernel 入口与主机分派，在设备实现内部整合成对蝶形，并同步检查 CLI 的网格计算。其收益仍需重新通过完整正确性与冻结的性能范围；本轮诊断不构成采纳结论。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/implementation-optimization-plan.md b/03_hadamard_tc/a962695448-rgb/reports/implementation-optimization-plan.md new file mode 100644 index 00000000..58e31e77 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/implementation-optimization-plan.md @@ -0,0 +1,97 @@ +# 实施与优化方案 + +## 2026-09-08 A100规则与调用方式复核 + +最终源24dfef7已在A100-SXM4-40GB与4090完成完整回归。A100独立筛选/留出接受9条规则，N16融合未采用；已有A800/4090规则保持不变。最终Graph的64/62个启用配置在各自三轮中均至少减少5%耗时，两卡各24个回退控制保留全部波动。 + +普通调用的对照说明参数传递写法带来明显开销：同一auto改用现有位置参数形式后，A100减少6.22%～9.49%、4090减少5.95%～12.31%，均为12配置每轮通过5%门槛。相对原默认仍存在最高约3.45%的成本，默认original保留，没有新增专用API或放宽数值条件。详见[最终报告](a100-calling-validation.md)与[完整证据](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_a100_calling_20260908/README.md)。 + +当前已完成本轮可采用规则及验证。后续价值在真实应用调用流程、尚未覆盖的规模与型号，以及获准后采集硬件计数器；这些不作为本轮已完成成果。以下段落保留各历史阶段的范围和当时状态。 + + +## 2026-09-08 小维度行打包与自动选择 + +生产源码`a3703fd`已完成A800-SXM4-40GB与RTX 4090实测，加入N=1/2/4/8/16的显式packed/auto；默认仍original。两卡的62个启用Graph配置三轮均减少至少5%耗时，范围分别6.17%～68.85%、5.20%～66.82%；完整正确性、内存/同步及实际内核路径检查通过。N16融合两次未达门槛，自动规则未采用。 + +普通逐次Python调用的8配置均退化，因此不把auto推广为默认，也不将Graph收益外推为应用整体加速。后续针对关键字调用、字符串解析、设备/形状判断和分配分别做消融；在提出新候选前保留本轮全部负例。新版A100验证、硬件计数器和更广工作负载仍是明确未完成的扩展，不能借用旧结果。详见[本轮报告及复现入口](packed-rows-validation.md)。以下段落按对应历史版本和实验分别阅读。 + + +## 2026-09-08 A100 补充验证 + +现有融合源码已在 A100 sm80 重新编译，完整回归及三轮 52 配置验证通过。51/52 三轮更快，47/52 每轮至少减少 5%；一配置第三轮退化 8.43%，后续预热诊断没有重现，但不覆盖原负例。默认 original 保留。见[A100 报告](fused-layout-a100-validation.md)和[原始数据](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_fused_a100_20260908/README.md)。以下历史实验和适用范围分别保留。 + +本方案按已保存的慢例和测量边界组织实现、验证与采用决定。默认接口保持兼容，所有收益限定到实际测试范围，历史失败和原始数据单独保留。此前N256的4090与A100验证已分别归档；最新小维度版本以文首A800/4090报告为准，更广形状与其他设备仍需另行实测。 + +## 当前优先处理的形状 + +| 优先级与对象 | 已有观察 | 实施方向 | 验收重点 | +|---|---|---|---| +| P0：所有变更的数值一致性 | 融合量化必须先舍入到 FP16/BF16，再计算每行 INT4；昇腾已有除法中点改变 packed byte 的反例 | 保留独立稠密参考、实际设备输出的 CPU 量化、RNE 边界和原位/尾部检查 | 任何 packed bytes/scales 不一致、越界或阈值失败都停止采用该候选 | +| P1：NVIDIA `[17,256]` | A100 原 64 份独立输出 Graph 下，项目默认路径比固定 Dao 参考慢约 1.06%/0.59%；4090 同形状也有负例 | 比较启动配置和每 CTA 的行映射，检查小批量调度、寄存器及访问开销；先单因素实验 | 固定原始条件并补邻近 M，不能把约 1% 的一次差异当作确定结论 | +| P1：已验证的 NVIDIA N=16/64 大批量配置 | 显式 256 线程在预定 72 个原/邻近配置中稳定优于同轮 128；范围有限 | 保留已验证候选，评估是否有必要增加明确的逐形状选择；单独测选择分支本身的成本 | 不能将 N=16/64 的结果外推为所有 N、M 或 GPU 都选 256 | +| P2：NVIDIA Tensor Core 路径 | 当前 dense WMMA 实现有额外计算和转换，在部分大维度配置慢于 FWHT | 分开测转换、搬运、矩阵计算及融合成本，评估数据复用和布局；收益不足时保留为对照实现 | 不通过隐藏转换成本或改变数学输出制造加速 | +| P2：壁仞 N=256 fused | 大 M 的 Warp32 仍明显慢于共享实现；小批量方案也有 N=256 负例 | 明确保留共享路径，限定小批量候选范围；新的寄存器/打包方案分别测量 | 不能从 local-memory 元数据为零推断没有其他瓶颈，也不能把寄存器数变化当成已证实因果 | +| P2：昇腾短 N/小 M | vector-scale 有混合方向结果，尤其 N=16；特定较大离散 M/N 有较稳定收益 | 保持显式选项，按实际形状验证是否启用；量化仍采用已核验的 NPU 标量除法 | 不把向量路径的局部收益解释为应用整体加速，不改变精确量化契约 | + +依据：[A100 报告](a100-validation.md)、[NVIDIA 线程配置实验](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_thread_promotion_20260905/RESULTS.md)、[壁仞报告](../platforms/biren/README.md)、[昇腾报告](../platforms/ascend/README.md)。不同设备、配额和计时方式的数据不交叉作分母。 + +当前执行状态：本轮 32/64 线程小批量筛选没有满足预定 24 配置的三轮收益门槛，未采用；后续留出验证未执行，也未扩展生产 API 或改变数学实现。具体数据入口待原始结果归档后补入，不能将未运行的留出流程列为通过。后续若继续研究该慢例，应针对新的可检验瓶颈提出独立候选，不重复宣传本轮未采用的配置。 + +本轮小批量线程筛选的 [结果、完整负例和复算入口](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_small_batch_20260906/RESULTS.md) 已归档。实际只执行 24 个配置、168 个不同输入、三个有效进程；32/64 线程均未达到事前采用门槛，因此没有执行 120 项后续留出验证，也没有扩大生产接口。一次启动前设备利用率检查失败的记录单独保留。 + +## 本轮内核实验与采用决定 + +| 实验 | 已完成的证据 | 当前决定与下一步 | +|---|---|---| +| N256 相邻元素布局 | [24 配置、三独立进程](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_contiguous256_20260906/RESULTS.md)，包含实际 2B 错位、非默认 stream、量化中点；融合全部配置三轮均快，其中 20 项三轮各降低至少 5% | 显式融合选项完成生产回归及[修订后的52邻近配置](fused-layout-validation.md)：52项三轮均快，41项每轮至少减少5%，范围4.7837%～14.3440%。默认保持原布局；普通transform不采用新布局 | +| 四 warp WMMA 输入复用 | [120 配置与 21,600 条事件](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_wmma_reuse_20260906/RESULTS.md) 全部核验，保留 24 筛选与 96 留出；一般输入 old/new 全元素位一致 | 保留为独立对照实验。N64/128/256 全部 72 配置仍慢于 warp128，不推广生产派发；N16/N32 大 M 的局部胜例单独说明 | +| PyTorch 惰性负号视图 | 原/新实验布局、两种精度、三种 API 的 12 项独立诊断均复现底层符号读取错误，见相邻元素布局档案的 diagnostics | 公共接口增加明确拒绝并提示 resolve_neg；生产整合的 28 项拒绝与物化正例已通过。诊断的 REPRODUCED 不代表接口行为正确 | + +生产源 `155a05a8b957bdf558ef93a2db1e3aea9fadf36f` 已通过三种模式各自的同一套 1,876 项 CLI 矩阵、原 1,800 项参考对照（其中 200 项 N256 同时核查新融合选项）、28 项元数据边界和 16 项定向接口检查。重复执行和子集不累计为更多独立输入。该N256融合版本的A100补充实测已在文首链接中完成；更新的行打包版本仍不能沿用该记录。 + +52 配置原协议的辅助 FP64 舍入检查失败保留为 FAIL，性能条目为零。固定见证中，FP32 中间值恰落在 FP16 中点：项目与固定参考库均返回 `-17.96875`，精确数学值应舍入到相邻的 `-17.953125`，间隔为 `0.015625`。该输入的全部 4,194,048 个元素与参考库逐位相同，融合与分步也一致。单独修订的V2保持同一矩阵和参考库阈值，经数学及代码审阅后完成三轮实测和离线精确证书复算；辅助数学误差按固定前向误差界核查。旧失败不追认为通过。 + +## 先固定输入和候选，再运行实验 + +每轮实验先保存配置表，明确 GPU/SDK、源码提交或补丁 hash、实现方法、M/N、dtype、scale、stream、工作集大小、预热/重复数及计时方式。一个实验只回答一个问题，例如“同一形状从 128 改为 256 线程是否减少 Graph 区间耗时”。不在看到结果后删除不利形状或改变采用阈值。 + +NVIDIA 新形状候选应保留当前完整正确性集，并为影响的派发边界预先加入上下相邻值。以 `[17,256]` 为例，可固定 M=16/17/18 作为直接邻近集，再选择一个不触发新逻辑的控制点。新的 shape 列表必须在计时前落盘；原 72 配置的测量范围维持原样，不能事后扩写它的覆盖。 + +留出验证使用不参与选择配置的预定形状，覆盖边界两侧、未满 warp/CTA 的尾部、全零、脉冲、均匀/正态/离群值及多个随机种子。基线与候选使用完全相同的输入、dtype 与归一化约定。随机种子和样本来源随原始结果保存。 + +## 比较必须使用同一计时边界 + +| 口径 | 区间约定 | 可比较的对象 | +|---|---|---| +| kernel/event | 预分配缓冲区，预热在区间外，按同一设备 stream/event 记录 | 同机相同调用序列的基线与候选；说明是否包含发射空档 | +| CUDA Graph | 保留 64 份独立输出，图构建在区间外，使用相同重放次数和组数 | 相同 Graph 测量流程下的两种实现，不能与 eager 或端到端数值相除 | +| eager API | 两实现使用相同分配和调用方式 | API 可见执行流程；不能把结果称为隔离的纯内核时长 | +| host end-to-end | 明确包含分配、H2D/D2H、同步中的哪些步骤 | 用户实际调用流程，单独成表 | + +每种候选和对应基线应在同机、同配额、同组内轮换顺序；跨二进制实验使用预定交错独立进程，并记录它不是同进程 A/B。编译、属性查询和其他 GPU 工作不与主计时并发。若出现预先可识别的干扰，先记录原因与排除决定，原始运行保留，再补干净轮次。 + +结果用毫秒保存或提供明确的毫秒派生表。原始事件读数、每组样本、各轮中位数以及全部退化配置必须可追溯。配置百分比的中位数没有应用工作负载权重，不能推导应用级总收益；“三轮均达到 5%”是工程采用规则，不是统计显著性或置信区间。 + +## 正确性与负例保护 + +1. 保持 Sylvester Hadamard、FP16/BF16 存储、FP32 内部计算和 scale 的原契约。NVIDIA 核心改动重跑当前 1,876 项自测与 1,800 项固定第三方对照；其他平台使用各自报告中的完整矩阵，不能由 NVIDIA 的通过替代。 +2. 同时检查项目 CPU FWHT、独立 FP64 稠密参考以及相应输出舍入。FP16/BF16 的绝对误差阈值不放宽；已舍入与未舍入参考分别报告。 +3. CPU 对实际设备变换输出量化、设备分步及融合 INT4 的 packed bytes/scales 必须精确一致；保留零行、RNE 中点和链式输入反例。 +4. NVIDIA Tensor 前端按其真实契约检查非默认 stream、输入保持、dtype/形状/连续性/设备及非法线程配置；不将原生指针 API 的原位或部分重叠规则当成 PyTorch 前端能力。国产原生指针接口另外检查完全原位、部分重叠拒绝、指针对齐、尾部哨兵、容量计算溢出和非法启动配置。多设备或大索引只声明实际覆盖的范围。 +5. 性能表保留所有负例。若候选在某形状变慢，就保留原路径或缩小显式适用范围，不通过删除基线、少测输入或修改输出精度解决。 + +源码、编译参数或默认派发变化后，重新验证受影响路径。只修改文字、单位换算或报表展示时，核验计算逻辑和数据不变即可，不把文档检查记成新的设备测试。 + +## 逐形状选择的采用标准 + +候选先作为显式选项或独立实验存在。只有预定目标及留出边界上的正确性全部通过，且同一计时方式下三轮每轮均减少至少 5% 耗时，才考虑在该离散范围采用。这个阈值用于排除收益过小、波动较大的改动，不能外推为任意输入保证。 + +自动选择还必须检查控制点与新增分支开销：没有触发候选的形状不应引入持续退化；若出现无法区分于测量噪声的结果，继续保留显式开关和原默认。不同 dtype、N、M、GPU 架构分别建立证据，不从“一个平台 warp 宽度相同”推断另一个平台也应采用同一策略。 + +每次选择规则变更记录：采用条件、回退路径、预定输入表、实际通过/失败列表、原始计时文件、稳定性规则和限制。已存在的慢例成为长期回归输入，不能在下一轮选择时遗失。 + +## 交付与提交流程 + +每个可审阅改动应包含最小源码差异、构建/运行命令、固定环境及版本、完整正确性结果、同口径性能正负例、原始文件 hash 和简短结论。若硬件计数器不可用，应保留实际错误和可用的 event/时间线证据，不用估算数据冒充实测。 + +历史实测源码与原始日志按实际执行版本保留；后期文字整理或路径去标识产生的公开副本，分别记录原/公开双 hash 与修改范围。PR 和课程登记在项目所有者验收具体代码及证据后单独执行；本方案本身不创建 PR、不代替提交确认。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/integrated-pairs-20260917.md b/03_hadamard_tc/a962695448-rgb/reports/integrated-pairs-20260917.md new file mode 100644 index 00000000..a10a5367 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/integrated-pairs-20260917.md @@ -0,0 +1,34 @@ +# 整合入口候选的完整验证 + +本轮保持 `packed_rows_kernel` 的名称、模板参数及 host 派发结构，在设备分支整合相邻成对蝶形；CLI 与 PyTorch 共享编译期行宽计算。**它仍是未采用的候选，生产计算源码保持原版。** + +## 完整检查 + +- CLI 候选 original/packed/auto × 128/256 线程，六种配置的完整 **1876 案例集合**均通过；重复配置不累计为更多独立输入。 +- 原版 original128、packed256 自测通过；两版各执行既有 **15 项 CLI 拒绝检查**。 +- 原版、同源副本和候选的 128 项分量/scale/偏移配置通过；候选 24 项流/Graph、512 项溢出与抵消兼容通过。 +- 原量化分量、singleton、packed 700 正例及旧接口 372 正例/63 负例/4 mutation 等回归通过，详见原始日志。 + +## 采纳判据与结果 + +原 RTX4090 没有可用 GPU，本轮采用已授权的 RTX4090D，原版和候选同机重编。三个新进程、完整 588 项设备及 588 项普通调用对照，使用 18 组平衡次序、相同输出对象和配对时间线；目标 1.10×、单项退化不超过 5% 的限值保持不变。 + +| 项目 | 本轮结果 | +|---|---| +| 588 项设备比值、24 个目标分组 | 全部通过 | +| Hadamard 目标设备几何平均 | 1.270357–1.400775× | +| 融合 INT4 目标设备几何平均 | 1.314354–1.407615× | +| 588 项普通调用的两个相对比值口径 | 全部符合 5% 限值 | +| 18 项同源码 A/A 校准 | 全部通过 | +| 绝对耗时稳定性 | 1 项未过 | +| 最终判定 | **REJECT，未采用** | + +未过项为第二轮 BF16 packed Hadamard / rows65536 / N8 / out。独立中位数比 **0.985208**、配对比值 **0.987443**，都仍在 5% 限值内；但两侧 IQR/中位数分别为 **49.732% / 44.103%**，超过运行前固定的 10% 稳定性限制。因此不能将该轮追认为通过。 + +前半段约 3.9–4.0 微秒，随后两侧共同升至约 6.3 微秒，再回落。线程 CPU 时间基本同步变化，相关块没有 GC 回收记录。它不足以证明具体根因，不能简单称为算法退化、GC 或失去 CPU 调度时间。 + +## 证据与后续 + +[固定完整档案](https://github.com/a962695448-rgb/Learning-CUDA/tree/704b0b3625cf93048b21ec47ac578f9bc0c32c0b/03_hadamard_tc/a962695448-rgb/results/integrated-pairs-20260917)包含候选源码、协议、CLI 日志、无损压缩的全部时间线及恢复/审计工具。245 份输入前后相同，39 份原始文件回收并逐项校验后关闭实例。 + +上述数值属于本轮 RTX4090D 和具体 Graph/普通调用口径，不外推其他显卡、模型端到端或生产版本收益。后续若继续确认，必须先制定前瞻性的稳定性测量方案，重新采样；保留本轮 REJECT 与全部旧数据，不仅补跑失败点，也不在旧数据上改规则追认成功。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/out-buffers-20260917.md b/03_hadamard_tc/a962695448-rgb/reports/out-buffers-20260917.md new file mode 100644 index 00000000..6aab7ed1 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/out-buffers-20260917.md @@ -0,0 +1,45 @@ +# 可复用输出接口与普通 Python 调用性能 + +新增 `hadamard_out`、`hadamard_int4_out`、`quantize_int4_out`，用于形状稳定、能够复用输出的前向调用。接口写入调用者的缓冲区并返回 `None`；原分配式接口和默认参数保持不变。 + +## 用法 + +```python +y = torch.empty_like(x) +packed = torch.empty((*x.shape[:-1], (x.shape[-1] + 1) // 2), device=x.device, dtype=torch.uint8) +scales = torch.empty(x.shape[:-1], device=x.device, dtype=torch.float32) + +op.hadamard_out(x, y) +op.hadamard_int4_out(x, packed, scales) +op.quantize_int4_out(y, packed, scales) +``` + +输出必须具有精确形状、dtype、相同 CUDA 设备和连续布局，不会自动 resize。输入与各输出的活动字节区间必须互不重叠，同一分配中的不重叠视图允许使用。再次调用会覆盖旧输出；需要保留旧结果时先复制。跨流使用、复用和释放缓冲区时由调用者管理同步与生命周期。 + +输入和输出均遵守前向约束，不能 requires_grad。写入维护 PyTorch 版本计数，能够识别其他计算为反向传播保存的数据已经被修改；inference tensor 须在 inference_mode 内更新。CUDA Graph 重放遵循 PyTorch 的捕获规则，主机端版本更新发生于调用/捕获阶段。 + +## 验证 + +| 检查 | 结果 | +|---|---| +| 正常调用 | 372 组配置，每组两份输入，与独立 NumPy 参考逐位一致 | +| 拒绝行为 | 63 项异常在写入和版本变化前拒绝，含 DLPack/部分/输出间重叠 | +| 存储与修改记录 | 输出地址和哨兵保持；保存给反向的数据能检测修改；inference_mode 通过 | +| 输出分配 | 预热后重复 50 次调用，无额外 PyTorch 输出张量显存分配 | +| 执行上下文 | 24 组流/Graph 检查，含 96 次变更输入重放、48 个错误对照，全部通过 | + +覆盖 FP16/BF16、2D/4D、dim=1～256 二次幂及合法线程/布局。首次负例工具没有捕获非法布局产生的 ValueError，修正工具后完整通过；初始错误、修正和原始记录均保留，未更改 C++ 实现来解决该工具问题。 + +## 三轮普通调用配对 + +基线 `bce004ec`。在 RTX 4090 D / NGC Torch 2.6 / CUDA 12.8 上，目标 rows=1/17/256、dim=8/64/256、FP16/BF16；另有 rows=65536 对照。72 个配置 × 三轮共 216 组，每组九次交替测量三种路径（基线分配、候选分配、候选 out）。每个计时样本调用 2000 次，前后 CUDA 同步。 + +| 接口 | 三轮目标配置几何平均加速比 | +|---|---| +| Hadamard | 1.747 / 1.756 / 1.751× | +| Hadamard + INT4 | 2.008 / 2.023 / 2.006× | +| INT4 | 2.030 / 2.046 / 2.035× | + +每方法每轮要求至少快 10%，任何配置或原接口回退不超过 5%，全部满足；原接口最大观测回退 2.765%以内。计时包含普通 Python 调用、旧接口输出分配/返回处理和 GPU 完成；out 的一次性预分配在计时之外。该收益要求复用缓冲区，不等于内核加速、初次分配收益或模型端到端提升。 + +[固定原始证据](https://github.com/a962695448-rgb/Learning-CUDA/tree/008d11dff2aa54d3bc02a3c54bdac78667daf882/03_hadamard_tc/a962695448-rgb/results/out-buffers-20260917)包含完整计时、协议、校验、源码指纹、错误记录和准备脚本。单独正确性可执行 `python scripts/verify_out_buffers.py --build-directory /tmp/out-build --json /tmp/new-out-report.json`。本轮仅在上述 NVIDIA 单卡环境实测。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/packed-rows-validation.md b/03_hadamard_tc/a962695448-rgb/reports/packed-rows-validation.md new file mode 100644 index 00000000..cd0e08e4 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/packed-rows-validation.md @@ -0,0 +1,122 @@ +# 小维度行打包与自动选择验证 + +> 本文固定记录a3703fd阶段。后续24dfef7已补A100实测及调用方式分解，见[A100规则与调用方式复核](a100-calling-validation.md)。以下设备与待验证状态仅对应本阶段，原始数据不改写。 + +## 实现与适用范围 + +实际生产测试源码为 `a3703fda3cfd7aa1b45342210dc57fb030f0d904`，基于原生产版本 `53e7b8641cb9f8038ce2d86e8f836740bae96684`。本轮分别在 NVIDIA A800-SXM4-40GB（sm80）和 NVIDIA GeForce RTX 4090（sm89）构建、验证。A800 是本次实际设备，不能标为 A100；早前 A100 结果对应此前的源码。 + +原路径每个 warp 处理一行。对 N=1/2/4/8/16，新路径把 `32/N` 行放入同一个 warp，每个连续 N lane 子组独立执行蝶形和归约。例如 N=8 时，一个 warp 同时处理四行。尾部不足的行参与 shuffle 的控制流，但不读取或写出越界元素。该设计增加短行的有效工作量；没有硬件计数器数据，不据此宣称已测得 occupancy、带宽利用率或唯一瓶颈。 + +所有蝶形加减仍按原顺序使用 FP32。融合 INT4 仍先把变换结果舍入为 FP16/BF16，再按行求 scale、执行除法、最近偶数舍入及打包；N=1 的高半字节固定为零。`kernels.cuh` 与 `contiguous256.cuh` 没有修改。 + +API 增加 `row_layout="original"|"packed"|"auto"`，默认 `original`。`packed` 明确要求 N≤16，`auto` 根据设备名称、N 和行数选择；没有规则时回退原路径。现有三参数调用、128/256 线程选项、非默认 stream 与元数据拒绝行为保持兼容。`quantize_int4` 不增加此选项。 + +## 自动选择规则 + +表中数值是开始启用 packed 的行数下限；启用时选择 256 线程。`M=batch*seq*heads`，二维输入则是第一维。未触发时保留调用者的 128/256 线程。两种精度共用下表规则。 + +| 路径与设备 | N=1 | N=2 | N=4 | N=8 | N=16 | +|---|---:|---:|---:|---:|---:| +| 变换，A800 / RTX 4090 | 4096 | 4096 | 4096 | 4096 | 4096 | +| 融合 INT4，A800 | 1 | 1 | 16 | 256 | 不启用 | +| 融合 INT4，RTX 4090 | 1 | 1 | 256 | 64 | 不启用 | + +设备匹配限定为 `NVIDIA A800-SXM4-40GB` 和 `NVIDIA GeForce RTX 4090`。A100、4090 D 及其他设备全部回退原路径；没有借用相同计算能力编号来推广策略。阈值规则可作用于更大的 M，但性能证据仅覆盖本档案列出的有限形状，不能保证任意 M、任意 scale 或其他软硬件版本更快。N>16 回退原路径。 + +N16 融合的第一次留出与独立收窄实验均未达到采用门槛，因此没有安装自动规则。显式 packed 仍可供用户按实际工作负载测量。`row_layout` 的非 original 选项不能与 `fused_layout="contiguous256"` 同时使用。 + +## 实验顺序与采用依据 + +输入协议、候选源码及基线先冻结，再运行计时。参考库固定为 Dao-AILab `fast-hadamard-transform` 的 `e7706faf8d1c3b9f241e36860640ad1dac644ede`。 + +| 阶段 | 每张卡的实际范围 | 用途与结论 | +|---|---|---| +| 候选正确性 | 450 个形状/精度/scale 配置，分别检查 8 种输入/偏移条件，共 3600 条；另有 10 条四维/非默认 stream 条件 | packed128/256、基线、Dao 和 CPU 量化检查通过；重复方法不增加独立输入数量 | +| 初筛 | 108 个输入配置、变换/融合共 216 项比较，每项三独立进程 | 比较 original128、original256、packed128、packed256；用同轮更快的 original 作分母 | +| 第一次留出 | 180 个输入配置、360 项比较，每项三独立进程 | A800 接受 5/10 条规则，4090 接受 8/10；失败规则与全量原始结果保留 | +| 独立收窄 | A800 每轮 192 项比较，4090 每轮 112 项，均三独立进程 | 只复查前轮未接受规则，改用预先冻结的新行数和随机种子；各增加 4 条、1 条规则，N16 融合仍拒绝 | +| 生产整合 | 完整正确性回归；72 项 Graph、8 项 eager，各三独立进程 | 验证真实生产 API 和选择分支，不以独立候选结果代替生产结果 | + +初筛 M 为 `1,17,64,257,4096,16384`。第一次留出 M 为 `16,18,63,65,256,258,4095,4097,16383,16385`。独立收窄的新 M 池为 `253,259,511,513,1023,1025,4093,4099,8191,8192,8193,32767,32768,32769`，随机种子 `9090807`；阈值以下控制点不纳入采用条件。初筛、第一次留出与收窄均包括 scale=1 和归一化 scale；N1 重复的 scale 去重。 + +规则采用条件为：所有被该规则激活的预定配置，在三个进程中每轮耗时至少减少 5%。这是工程门槛，不是统计显著性或跨日期置信区间。看到首次失败后开展的收窄实验是独立后续实验，不能将它称为首轮事前方案，也不能覆盖首次拒绝。两卡各最终接受 9 条规则，完整拒绝原因见原始 `policy_validation.json` 和 `final_policy.json`。 + +## 生产正确性与工具验证 + +下列检查在两张卡分别完成，源码、构建产物指纹和退出码均已保存： + +- 同一套 1876 项 CLI 矩阵分别执行 original128、original256、contiguous256、packed128、packed256、auto128、auto256；七次执行不是 13132 个不同输入。 +- 原 1800 项 Dao 参考矩阵逐输入比较旧默认、显式 128/256，以及适用的 packed128/256、auto128/256。该生产矩阵的两种精度最大 Dao 绝对差均为 0。 +- 另有 280 项大行数、0/2B 偏移、四维输入和非默认 stream 条件；24 项新增 API 拒绝检查、28 项元数据检查及 6 项新增 CLI 拒绝检查。原有非法参数矩阵仍运行。 +- CSV 21/24 列约定、微秒到毫秒换算、split 独立量化线程记录及混合表头拒绝通过。拒绝追加后原 CSV 字节保持不变。 +- 独立原生夹具的 160 项条件通过 CPU FWHT/量化和哨兵检查；Compute Sanitizer 的 memcheck、synccheck 均报告零错误。 +- Nsight Systems 的 packed 控制配置（FP16，N8/M4096）包含真实 `packed_rows_kernel`，回退控制配置（FP16，N16/M65）没有 packed 内核，保留原 `warp_kernel`。此处仅核查实际执行路径，不能把 profiler 时间替换为常规基准。 +- 两卡实际 ncu 都返回 `ERR_NVGPUCTRPERM`，没有硬件计数器结果；单卡环境没有复验多 GPU 切换。 + +候选初轮正确性报告记录的是配置及条件数量，不记录每条 Dao 误差值；上述最大差为 0 仅指生产 1800 项矩阵，不能回填到候选报告。 + +## 生产性能结果 + +每个 Graph 保留 64 份独立输出，各方法输出地址不重叠；正式计时前重放 256 次。每组计时 20 次重放，共 5 组，轮换方法顺序。原始 CUDA event 毫秒除以 `64*20` 得每调用均摊值，取组中位数；三个进程独立重复。它包含捕获的 GPU 工作及均摊重放调度，不能称为独立单 kernel 延迟。本次充分预热协议与早期 5 次 Graph 预热协议分别保存。 + +整合后 72 项中，62 项会激活 packed，10 项是回退控制；本阶段 scale=1，FP16/BF16 分别测量。下表是每轮配置中位数的完整范围，没有删除负例。 + +| GPU / 口径 | 配置×进程 | 三轮均减少≥5%的配置 | 耗时减少范围 | +|---|---:|---:|---:| +| A800，激活 Graph，相对同轮更快的 original128/256 | 62×3 | 62/62 | 6.1717%～68.8534% | +| RTX 4090，激活 Graph，相对同轮更快的 original128/256 | 62×3 | 62/62 | 5.2017%～66.8207% | +| A800，回退 Graph，相对 original128 | 10×3 | 0/10 | -0.2483%～0.1768% | +| RTX 4090，回退 Graph，相对 original128 | 10×3 | 0/10 | -0.1719%～0.3998% | +| A800，eager API，相对 original128 | 8×3 | 0/8 | -10.4024%～-7.1345% | +| RTX 4090，eager API，相对 original128 | 8×3 | 0/8 | -13.5193%～-8.7615% | + +负的减少比例表示退化。eager 使用 500 次带分配的 Python API 调用，计时前后同步，5 组主机墙钟中位数，单独保存为毫秒。八项是 N1/M1 融合、N4/M17 融合、N16/M4096 变换、N16/M65 融合的两种精度；其中含激活与回退情况。 + +**本轮不推荐仅凭 Graph 表就在普通逐次 Python 调用中启用 auto。** 实际 eager 观察全部退化。测量同时涉及关键字参数、字符串选择、设备/形状判断以及调用/分配，尚未用单因素消融确定各部分成本，不能断言唯一原因。默认仍 original；Graph 用户可显式启用 auto 并复测自己的真实形状。显式 packed 也不是已证实消除 eager 退化的方案。 + +本实验没有应用工作负载权重，没有端到端大模型收益，也没有证明全局最优。跨卡结果分别统计，不能相除为 A800 对 4090 的加速比。 + +## 使用与复现 + +在项目目录中构建：4090 使用 `ARCH=89`，A800 使用 `ARCH=80`，PyTorch 扩展分别设置 `TORCH_CUDA_ARCH_LIST=8.9/8.0`。 + +```bash +make CUDA_HOME=/usr/local/cuda ARCH=89 +./build/hadamard --self-test --row-layout auto +./build/hadamard --benchmark --batch 1 --seq 4096 --heads 1 \ + --dim 8 --dtype fp16 --row-layout auto --csv results/packed-new-run.csv +``` + +```python +op = load_extension() +# x 为连续 CUDA FP16/BF16 张量；此例最后一维为8。 +y_default = op.hadamard(x, 1.0) # 保留原路径 +y_auto = op.hadamard(x, 1.0, row_layout="auto") +packed, scales = op.hadamard_int4(x, 1.0, row_layout="auto") +y_explicit = op.hadamard(x, 1.0, block_threads=256, row_layout="packed") +``` + +原始运行控制器在归档的各设备目录中，保留实际环境和参数。复验时复制输入源码到新的工作目录，使用新的输出目录和独立构建缓存；不要覆盖归档的 `runs`。生产 API 回归脚本为 `production/validate_row_api.py --reference-repo <固定参考源码> --out <新结果目录>`，其相邻 `project/` 是已冻结的生产源。脚本不代替新设备上的实际验证。 + +离线逐项复算生产计时，无需 GPU： + +```bash +cd results/nvidia_packed_rows_20260908 +python analyze_production.py a800 +python analyze_production.py rtx4090 +``` + +脚本核对每组原始 event 换算、组中位数、输出独立性标志、三轮配置集合、源/策略与二进制指纹，并生成全量毫秒 CSV。源输入已逐文件与 Git 提交匹配；这不是对新文档提交再次运行 GPU。 + +## 证据入口 + +[完整档案与清单](https://github.com/a962695448-rgb/Learning-CUDA/blob/3676727fc21ea27bda668d743d10e52d7e138f68/03_hadamard_tc/a962695448-rgb/results/nvidia_packed_rows_20260908/README.md)含两卡全部初筛、留出、独立收窄、拒绝规则、生产回归、Graph/eager 原始样本和 profiler 日志；新增文件的精确字节哈希见 `manifest.json`。 + +原始二进制 `.nsys-rep` 和 SQLite 时间线保存在本地完整 ZIP，公开副本仅省略这两类格式，其哈希及省略原因逐一记录。远端归档过滤器漏收的两份同内容 CPU `.cpp` 输入，从运行前冻结的本地输入包补齐并匹配服务器验证过的 manifest；`restored_input_sources.json` 明确记录来源，不冒称下载产物。 + +后续有价值的方向是普通 Python 调用的分项开销分析、新版在 A100 的独立复测及允许性能计数器的环境。当前所有新增选择都为显式选项，不因这些尚未完成事项更改默认路径。PR 与训练营正式提交仍待项目所有者验收。 + +## 后续设备规则 + +以上结论对应本文开头的历史受测版本。后续提交已单独验证 A100；2026-09-16 又加入 [4090 D 限定范围规则](4090d-routing-20260916.md)。当前行为以 `include/row_policy.hpp` 和对应设备报告为准，旧版“4090 D 回退”描述不代表新版本的已验证范围。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/paired-hadamard-20260917.md b/03_hadamard_tc/a962695448-rgb/reports/paired-hadamard-20260917.md new file mode 100644 index 00000000..0044b366 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/paired-hadamard-20260917.md @@ -0,0 +1,42 @@ +# 打包 Hadamard 边界验证与未采用的成对变换 + +## 本次保留的变更 + +新增 `scripts/verify_hadamard_packed.py`，验证现有 `row_layout="packed"` 的 Hadamard 和融合 INT4，以及相应 out 入口。该文件与实机运行的验证器内容逐字节相同，只采用更直接的文件名。 + +**生产 CUDA 计算和构建源码保持 `b49fff4ac57ae6702213077ebf7ff848cb8715d9`。**本轮两项优化候选均未通过完整性能门槛，实验源码保存在证据分支。 + +在 RTX4090、Torch 2.6.0a0+ecf3bae40a.nv25.01、CUDA 12.8、NumPy 1.26.4 上，基线实现通过： + +- **128 项分量/scale/偏移/线程配置**：N=2/4/8/16，两种 dtype，两种 scale，选定首尾分量遍历 128768 个有限编码；其他分量为六类有界锚点。不是全部向量组合的穷举。 +- 独立 CPU FP32 蝶形、输出精度舍入及 INT4 参考；allocating/out、2B 输入与输出偏移、奇数字节打包输出偏移、护栏、存储地址和版本记录。 +- **24 项流与 CUDA Graph 配置**，包含错误 stream 和陈旧输出负对照。 + +## 两个未采用的候选 + +首版让线程持有相邻两个值，在线程内完成第一级蝶形，减少线程间交换；第二版仅隔离融合路径的主机分派，设备内核头文件逐字节相同。每版分别与固定基线比较，协议在计时前冻结。 + +| 候选 | 设备目标分组几何平均 | 设备记录 | 普通调用记录 | 普通调用超限项 | +|---|---:|---:|---:|---:| +| 成对蝶形首版 | 1.264908–1.402975× | 588 | 588 | 3 | +| 隔离融合主机分派 | 1.264442–1.402666× | 588 | 588 | 5 | + +每版 24 个目标分组均通过 1.10× 门槛，588 个设备项与 9 个 A/A 校准也通过。普通调用要求每项耗时增加不超过 5%；由于超限，两版判定均为 **REJECT**。 + +FP16 / rows=65536 / N8 / out 的融合调用在两版中都有退化。第二版还出现分配式调用超限，主机分派简化未充分解决问题。原始样本、失败点及主机端符号/选定汇编保留，尚未据此证明这些差异的原因。 + +## 正确性与计时边界 + +候选的分量、流/Graph、旧接口与量化回归全部通过；每版另有 **512 项**有限输入的溢出、抵消、2D/4D 及偏移检查，逐位等于旧 packed 实现。这是兼容性记录，不对非有限中间值产生的 INT4 输出作数学意义保证。 + +计时为固定地址的预热 CUDA Graph：64 节点、10 次重放；普通调用在独立的墙钟区间内同步，每项 9 组、每组 5000 次，三轮交替旧新顺序并绑定 CPU。设备结果不能代替普通调用或模型端到端收益；两种计时口径及不同显卡不直接相除。 + +## 复现 + +```bash +TORCH_CUDA_ARCH_LIST=8.9 MAX_JOBS=1 python scripts/verify_hadamard_packed.py \ + --build-directory /tmp/hadamard-packed-check \ + --report /tmp/new-hadamard-packed-check.json +``` + +[完整原始证据和重建工具](https://github.com/a962695448-rgb/Learning-CUDA/tree/3ff75126f3b29458c9a2dfb2665349afb3a4e58e/03_hadamard_tc/a962695448-rgb/results/paired-hadamard-20260917)包含两版源码、冻结协议、完整数据、独立门槛复算及基线 128+24 检查。数据回收并逐文件校验后，两次实机实例均已关机。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/paired-quantization-20260917.md b/03_hadamard_tc/a962695448-rgb/reports/paired-quantization-20260917.md new file mode 100644 index 00000000..e605c19a --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/paired-quantization-20260917.md @@ -0,0 +1,51 @@ +# 成对量化与启动分派优化（2026-09-17） + +## 采用的实现 + +基线为 `fabe4aa1beb4f623564ba02ce8cdf24af5932b11`。显式 `quantize_int4_packed` 与其 out 入口在 N=2/4/8/16 时，每线程持有相邻两个值，在寄存器内组成一个 INT4 字节。行最大绝对值在更小的线程组中归约。输入保留标量读取，支持原有合法的 2 字节偏移。 + +新路径只在 quantize-only 分派中使用；通用 Hadamard 启动/分派定义保持原样，N=1 及原始量化路径继续调用原内核。最终启动函数按编译期维度实例化，避免新分派重复进行运行时整数除法。已有 Python 函数名、参数、dtype/形状、缓冲区和当前 stream 合同保持不变。 + +## 最终验证 + +在 RTX4090 24GB、CUDA 12.8.61、Torch 2.6.0a0+ecf3bae40a.nv25.01、NumPy 1.26.4 上： + +- 64 组分量/位置/输入偏移/线程配置，遍历 FP16/BF16 的 **128768 个有限分量编码**，其余分量使用七类固定锚点。选定低/高分量分别比较独立 NumPy 参考、原实现和候选。它不是所有向量组合的穷举。 +- allocating/out、奇数字节输出偏移、输入 2B 偏移、哨兵、存储地址和版本计数检查全部通过。 +- 既有 packed 回归 **700 组正例、4 组非法条件、16 组流/Graph** 通过；原接口 **372 正例、63 负例、4 项 mutation** 通过。 +- N=1 的原有 **131072 种存储编码** 检查通过，非有限编码的回退对照不扩展有限值输入契约。 + +## 设备与普通调用分别计时 + +最终协议在测量前固定：同一允许的 CPU 核、Torch CPU 线程数 1；每配置每轮 9 个交替样本、每样本 5000 次普通调用，另做 CUDA event / 64 节点 Graph × 10 次重放。3 轮共 **288 条设备 + 288 条主机记录**，并有 **9 组同一函数的 A/A 校准**。 + +目标是 rows=16387/65536/262147、N=2/4/8/16、FP16/BF16、128/256 线程。小行数、N=1、原量化及 Hadamard/fused 作为对照。 + +| 指标 | 最终结果 | +|---|---| +| 每 dtype/线程/轮次目标设备几何平均 | **1.361–1.425×** | +| 单目标设备加速范围 | **1.114–1.873×** | +| 普通 allocating 调用目标整体几何平均 | **1.069×** | +| 普通 out 调用目标整体几何平均 | **1.117×** | +| 全部主机记录最大观测退化 | **3.534%** | +| 同函数 A/A 中值时间比范围 | **0.9945–1.0027** | + +所有设备、主机和 A/A 配置均满足 5% 退化限制，每个目标设备分组同时超过预设 1.10×。结论只属于该 4090、明确形状与两种计时方式，不外推为 4090D、A100、国产后端或模型端到端收益，也不与早先不同形状的比值相乘。 + +## 未采用版本与计时稳定性记录 + +1. 首版把新分派放进通用 launch：数值和设备门槛通过，但一个 Hadamard 普通调用控制点越过 5%，判 REJECT。 +2. 第二版隔离 quantize-only 分派：仍有一个小批量主机样本越过门槛，原始值在约 8/13 微秒间跳变。追加固定 CPU、长采样及 A/A 检查后，仍有 N4/16387 的分配式调用不达标，继续保留 REJECT。 +3. 代码检查发现新网格计算使用运行时维度。最终版本改用编译期维度专用启动函数，在完整正确性、同样长采样及原门槛下通过，并额外加入该 N4 控制点的 A/A 校准。 + +没有覆盖初版、第二版或失败的长采样数据。固定 CPU/延长采样是单独记录的协议变化，所有阶段仍保留原始判定，未把失败重算成通过。 + +## 复现 + +```bash +TORCH_CUDA_ARCH_LIST=8.9 MAX_JOBS=1 python scripts/verify_paired_quantization.py \ + --build-directory /tmp/hadamard-pair-check \ + --report /tmp/hadamard-pair-check.json +``` + +[完整准备工具、三版源码、冻结协议与原始数据](https://github.com/a962695448-rgb/Learning-CUDA/tree/9b3bb79f3c67b6c5e284be7ded9559148b5ae448/03_hadamard_tc/a962695448-rgb/results/paired-quantization-20260917) 已固定归档。CLI FWHT、Tensor Core、第三方 Hadamard 对照及其他平台的计算路径本轮未改动，沿用其原版本记录，不把这些历史测试记为本轮新运行。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/public-wording-provenance.json b/03_hadamard_tc/a962695448-rgb/reports/public-wording-provenance.json new file mode 100644 index 00000000..d3058dd3 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/public-wording-provenance.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "date": "2026-09-06", + "public_snapshot_commit": "e902aee9ad3ce38e4c64843bbb19f24c07ac50ee", + "record_scope": "The following fingerprints describe the documentation-cleanup snapshot at public_snapshot_commit. Later tested implementation or documentation updates on this branch are separate revisions.", + "scope": "Narrative-only public documentation update; no GPU execution, mathematical code, test thresholds, measured results or historical build identities changed.", + "historical_identity_rule": "Existing source_manifest, transfer_manifest, result JSON experiment-script hashes and pinned commits still identify actual executed originals. Public archive manifests identify current annotated copies. These are distinct fingerprints, not a rewrite of history.", + "a100_original_archive": { + "original_server_files": 48, + "current_public_byte_identical_files": 47, + "annotated_public_script_files": 1, + "annotated_file": "results/nvidia_a100_20260906/retrieved/promotion_a100.py" + }, + "technical_identifiers_retained": "Existing promotion experiment IDs, file names, binary names and dtype-promotion terminology retained.", + "original_storage": "Private original working-tree byte copies retained outside the public repository. No local user paths or original narrative phrases are included here.", + "new_plan": "reports/implementation-optimization-plan.md", + "records": [ + { + "project_relative_path": "platforms/biren/README.md", + "original_bytes": 31658, + "original_sha256": "ec493e176bb5f3abaa21a1545ef4a0a9cfc0b367504f12ec837066ec1cc4bd87", + "public_bytes": 31706, + "public_sha256": "977252c6fdc9a35850275154002fb643ad4161e52c3cf6856139ffe91c18369e", + "change_scope": "Public documentation wording and evidence-boundary clarification", + "private_original_retained": true + }, + { + "project_relative_path": "platforms/iluvatar/README.md", + "original_bytes": 28852, + "original_sha256": "02edeb2c497ec7f2cc96315ef00d9562ae32fa88fced3cb0e36a5c39fae9d98b", + "public_bytes": 28804, + "public_sha256": "4c1e768e00fe49c5e8f1fd84e39f147299d7684316ce2645a48a130b7d066fe7", + "change_scope": "Public documentation wording and evidence-boundary clarification", + "private_original_retained": true + }, + { + "project_relative_path": "platforms/metax/README.md", + "original_bytes": 18221, + "original_sha256": "5ab898e7c2fea9dfadaf71925d048b979900c0d37a5f19df2175a8551050a1bf", + "public_bytes": 18265, + "public_sha256": "69f25d8fe94f90f74555755922d215660fe8edbbef43a84a5d894af16ed382be", + "change_scope": "Public documentation wording and evidence-boundary clarification", + "private_original_retained": true + }, + { + "project_relative_path": "reports/a100-validation.md", + "original_bytes": 5067, + "original_sha256": "0f58f4944f60b37404aa2b956310a185de21b6ffde582bb08ebedf1ae9d28b74", + "public_bytes": 5408, + "public_sha256": "2f8feafb107aa02e2b1093bb0e1b58222c23e283b7c7c3def8dbfe2379475042", + "change_scope": "Public documentation wording and evidence-boundary clarification", + "private_original_retained": true + }, + { + "project_relative_path": "results/nvidia_a100_20260906/retrieved/promotion_a100.py", + "original_bytes": 14366, + "original_sha256": "f19f94444fb7e86583676985c2294d1a72b7a35936029581e51775944170f576", + "public_bytes": 14372, + "public_sha256": "b3c79dcf280af95087a6d584bf36009c547a7c1514ad245709d2c033bd82cf3b", + "change_scope": "Module docstring only; executable AST unchanged", + "private_original_retained": true, + "executable_ast_unchanged": true + }, + { + "project_relative_path": "results/nvidia_thread_promotion_20260905/run_experiment.py", + "original_bytes": 14582, + "original_sha256": "0a7a13e869c29fb3479847e26fbc495da714ef7192c3b00a7f185f938642f553", + "public_bytes": 14588, + "public_sha256": "5cda45e0d624da9a7472845bc47d7d5154347c9f0ffcc88e0cc2fe5035c019ec", + "change_scope": "Module docstring only; executable AST unchanged", + "private_original_retained": true, + "executable_ast_unchanged": true + }, + { + "project_relative_path": "results/nvidia_a100_20260906/README.md", + "original_bytes": 5024, + "original_sha256": "28ba5765ce362c606577162ecbe0c7bf415d8225894d97976bc45a673abe1ec7", + "public_bytes": 5749, + "public_sha256": "48aa1d4a75eb99ea97af59ebc5cf6e64e09e16ff7cfcf9e0f45b1174092c2cd9", + "change_scope": "Public documentation wording and evidence-boundary clarification", + "private_original_retained": true + }, + { + "project_relative_path": "results/nvidia_thread_promotion_20260905/README.md", + "original_bytes": 2643, + "original_sha256": "10853f665828bbfd1469d41b95452184ec46488aee29b493c3640452d766e72c", + "public_bytes": 3179, + "public_sha256": "7907b7c45645cf23aa8e5074aa6a6a303137a1a09183fda5fe04c2c4f8d1e424", + "change_scope": "Public documentation wording and evidence-boundary clarification", + "private_original_retained": true + }, + { + "project_relative_path": "results/nvidia_a100_20260906/public_derivation_provenance.json", + "original_bytes": 2103, + "original_sha256": "639e247471bd5126aaaa04316b7501fece849e214f22a16a0592f3a8fcd5c5bd", + "public_bytes": 2740, + "public_sha256": "be9350e343331d4ceeb8fc87a4fb3ff1a144330f68b088ee5ccb172eaef3c3dc", + "change_scope": "Inventory/provenance update for one annotated public source snapshot", + "private_original_retained": true + }, + { + "project_relative_path": "results/nvidia_a100_20260906/archive_manifest.json", + "original_bytes": 25272, + "original_sha256": "d373d36cd1abf6cd4cba0f39f0615c598a1ea1382c4756f648ffec42a325f821", + "public_bytes": 25915, + "public_sha256": "679fdeac22dc5424e95861b362fab270c944af4711ae0a0ef7e0c35c8aa932fe", + "change_scope": "Current public inventory hashes and source-annotation metadata only", + "private_original_retained": true + }, + { + "project_relative_path": "results/nvidia_thread_promotion_20260905/archive_manifest.json", + "original_bytes": 10401, + "original_sha256": "8b399f3f9a0732bb3b7c1eca7f8a77d2c34dc8d3015e6d2e2a9fb4b326298e3b", + "public_bytes": 10817, + "public_sha256": "e01a2c013d81303c5e347436354219a6785f94dd27b04acc96551ef028082b05", + "change_scope": "Current public inventory hashes and source-annotation metadata only", + "private_original_retained": true + }, + { + "project_relative_path": "README.md", + "original_bytes": 26681, + "original_sha256": "e652e1706e7b8618b41812d30f9dbf32482a288c6a4c9f9a7f39153a34127311", + "public_bytes": 26608, + "public_sha256": "3781e6132101772425d20f3926ae4f9abaa9bff0bb76f659c4585f41cccdb9cd", + "change_scope": "Public plan link and A100 validation status; mixed working-tree line endings normalized to the existing Git LF form.", + "private_original_retained": true + }, + { + "project_relative_path": "platforms/ascend/README.md", + "original_bytes": 20854, + "original_sha256": "5184449504389dbf168f366819831440cd090a157f26d16445b42dec2056982c", + "public_bytes": 20872, + "public_sha256": "74ec23610a632ea55e6afc63f2016d2ac3b3a4e320ffc590062bcf75276da362", + "change_scope": "Public documentation wording and evidence-boundary clarification", + "private_original_retained": true + } + ], + "provenance_self_hash_rule": "This file does not include its own hash; outer Git commit identifies it.", + "baseline_public_git_commit": "9f5fdc363b4149d4a211701f24ab0548084ca3e5" +} diff --git a/03_hadamard_tc/a962695448-rgb/reports/quantize-packed-20260917.md b/03_hadamard_tc/a962695448-rgb/reports/quantize-packed-20260917.md new file mode 100644 index 00000000..bb652147 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/quantize-packed-20260917.md @@ -0,0 +1,65 @@ +# 小维度独立 INT4 量化（2026-09-17） + +## 改动与使用 + +为已舍入的 FP16/BF16 输入增加两个显式入口，复用现有的行打包 kernel： + +```python +packed, scales = op.quantize_int4_packed(x, block_threads=256) +# 复用已分配的缓冲区，返回 None： +op.quantize_int4_packed_out(x, packed, scales, block_threads=256) +``` + +- 支持连续、非空的二维/四维 NVIDIA CUDA 输入，最后一维为 1、2、4、8、16；线程数为 128（默认）或 256。 +- 同 `quantize_int4` 的按行最大绝对值缩放、最近偶数舍入、[-7,7] 截断和低/高半字节顺序；N=1 的高半字节为 0，全零行 scale=1。 +- `_out` 使用调用者的 uint8 packed 和 float32 scales，要求形状、设备、连续性与活动字节区间互不重叠；写入记录版本计数。原来的梯度、惰性负号和 inference buffer 约束继续适用。 +- 原来的 `quantize_int4` 与 `quantize_int4_out` 保留原签名、默认值和参数解析路径。新入口显式选择 packed，不引入自动设备路由，也不借用 Hadamard/fused 的 auto 范围。 + +## 三轮同机配对测量 + +基线为 `28fb3a95ca8bcdf53a66949cfde23820bf7e4005`。RTX4090D 24GB、sm89、CUDA 12.8.61、驱动 570.124.06、Torch 2.6.0a0+ecf3bae40a.nv25.01、NumPy 1.26.4。 + +目标覆盖 4096、16387、65536、131071 行 × N=1/2/4/8/16 × FP16/BF16 × 128/256 线程。每轮 80 组设备测量，共 240 条；每条包含 7 次轮换顺序的旧版/新版本原接口/packed 配对。 + +设备时间使用 CUDA event，捕获 64 节点的 Graph 并重放 10 次，再除以实际调用数量。输入与输出预分配，warmup 不计时；同一组复用数据，因此属于热缓存下的量化微基准，不代表模型端到端效果。 + +| dtype | 线程 | 第 1 轮几何平均 | 第 2 轮 | 第 3 轮 | +|---|---:|---:|---:|---:| +| float16 | 128 | 2.886× | 2.888× | 2.885× | +| float16 | 256 | 3.168× | 3.171× | 3.169× | +| bfloat16 | 128 | 2.871× | 2.891× | 2.888× | +| bfloat16 | 256 | 3.158× | 3.168× | 3.167× | + +全部设备配置三轮的加速范围为 **1.282–11.276×**；12 个 dtype/线程/轮次分组的几何平均为 **2.871–3.171×**。旧接口的最大设备计时退化为 **3.074%**，低于预设 5%。 + +普通 Python 调用单独测量：256 线程、每组 500 次调用，调用前后同步；输出分配计入 allocating，预先分配的输出不计入 out。共 240 条成对记录。整体几何平均： + +| 普通调用 | packed / 基线加速 | 单配置范围 | 原接口最大退化 | +|---|---:|---:|---:| +| allocating | 1.361× | 0.987–2.652× | 1.716% | +| out | 1.891× | 0.989–5.430× | 2.040% | + +小批量可能只持平或略慢，故保持显式启用。所有设备与主机记录均满足“不退化超过 5%”，每个设备分组同时满足几何平均至少 1.10×。这些结论仅属于本次 RTX4090D 和受测范围，不外推到 A100、普通 4090、MUSA 或其他国产后端。 + +## 正确性验证 + +- **700 组**：独立 NumPy 参考、原版、候选默认及候选 packed 的输出字节和 float32 scales 精确一致。覆盖 N=1/2/4/8/16，二维与四维，尾行，FP16/BF16，128/256 线程，以及全零、普通有限值、舍入临界值、极大有限值和次正规数。每组验证 allocating 与 out、输入不变、缓冲区地址、边界哨兵与版本计数。 +- **4 组非法条件**同时检查两个新入口：N=32/256、不支持的线程数和 auto 参数；全部在修改输入/输出或版本计数之前拒绝。 +- **16 组执行上下文**：64 次输入变化的 CUDA Graph 重放、32 个故意错误的流/旧输出负例，全部符合预期。 +- 原接口回归：**372 组正例、63 组负例、4 项 mutation 合同检查**全部通过。 + +从本项目目录独立重跑正确性： + +```bash +TORCH_CUDA_ARCH_LIST=8.9 MAX_JOBS=1 python scripts/verify_quantize_packed.py \ + --build-directory /tmp/hadamard-packed-check \ + --report /tmp/hadamard-packed-check.json +``` + +## 取舍与证据 + +初版尝试给现有 API 添加 `row_layout` 参数。虽然设备计时通过，但普通 out 调用在小批量时多了约 0.2 微秒开销，21/120 条原接口记录退化超过 5%。因此保留原始失败实验，改用独立入口，并在复测前增加逐场景主机门槛；最终版本全部通过。 + +最终源码在测试前后按 SHA-256 核对；8 份最终原始结果已恢复并逐份校验，完整备份为 888114 字节，SHA-256 `0c7a03721759bb748dbda5ad784fe6f065e0b59739de2aa0a7a144d15bfb2341`。初版失败实验另有独立备份，不覆盖。 + +[固定验证档案](https://github.com/a962695448-rgb/Learning-CUDA/tree/29bb34a0b2817e282a9554bd92c3911a3c010762/03_hadamard_tc/a962695448-rgb/results/quantize-packed-20260917) 保存实测源码、基线 Git blob 清单、`PROTOCOL-CUDA-FINAL.json`、`run_gpu.py`、原始 JSON、校验清单和已验证的复现准备脚本。初版与最终版数据分开保留；复现时按档案 README 准备准确的对照与候选源码。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/relative-stability-confirm-20260918.md b/03_hadamard_tc/a962695448-rgb/reports/relative-stability-confirm-20260918.md new file mode 100644 index 00000000..31db1e82 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/relative-stability-confirm-20260918.md @@ -0,0 +1,21 @@ +# 成对变换的相对稳定性确认（未采用） + +三轮新采样分别通过、拒绝、通过，最终 **REJECT**。本轮只补充验证结论，生产计算保持 b49fff4。 + +## 固定方案与完整范围 + +[固定证据](https://github.com/a962695448-rgb/Learning-CUDA/tree/06b985425e3192fb58e0500ddaffde3476de78ef/03_hadamard_tc/a962695448-rgb/results/relative-stability-confirm-20260918) 保存新采样前的预登记和全部完成记录。计算源码、三个已测二进制及 Torch/NumPy/CUDA 环境经过核对；新做 512 项溢出兼容检查通过。旧 CLI 和接口证据继续使用原有范围。 + +本次提前明确把“各版本绝对耗时的 IQR”改为诊断标记，稳定性门槛采用“组内控制/候选比值的 IQR/中位数 <= 10%”。这个定义改变在采样前公开；之前的拒绝结论保持原样。独立中位数与配对中位数的 5% 退化限值、设备分组 1.10 倍目标、三个新进程、588 设备和 588 主机记录、24 目标分组、18 同源校准全部保留。 + +## 为什么仍未采用 + +唯一未过项是第 2 轮 FP16 packed_fused、16387 行、N=2、256 线程、allocating：独立中位数比 1.031311，配对中位数比 1.029464，未显示中位耗时退化；但配对比值 IQR/中位数为 **11.840853%**，超过已登记的 10% 限值。 + +其余条件通过，设备目标分组为 1.270161–1.407428 倍。该范围仅描述未采用候选在本轮 RTX 4090 D 的 CUDA Graph 设备执行，不能当作生产、普通调用、跨卡或模型端到端加速。 + +因此保留生产实现，停止对这个候选的重复确认。后续优化转向其他热点，不提高限值或补跑到通过。 + +## 复核 + +21 份记录已无损回收并逐文件验证；独立分析再次得出 REJECT / 1 项未过。证据目录含预登记、固定源码指纹、协议、运行脚本、全部计时、压缩恢复工具和恢复收据。GPU 在备份校验后已确认关机。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/singleton-quantization-20260917.md b/03_hadamard_tc/a962695448-rgb/reports/singleton-quantization-20260917.md new file mode 100644 index 00000000..c92b8dc5 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/singleton-quantization-20260917.md @@ -0,0 +1,31 @@ +# 单元素 INT4 穷举与性能实验（2026-09-17） + +## 保留结果 + +生产计算内核保持基线 `f9da76087f6cbf6aad52e9effa6cf3ac3aabccf0` 的字节版本。本轮提交 `scripts/verify_singleton_quantization.py` 穷举检查和可复现实验记录，未纳入收益不足的计算特化。 + +FP16 和 BF16 各有 65,536 种 16 位存储编码；本轮覆盖共 **131,072 种编码**，其中 **128,768 个有限值**（FP16 63,488、BF16 65,280）与独立 NumPy INT4 参考逐位一致。范围仅为 N=1 的独立量化，不是任意长度向量的穷举。 + +二维 `(65536,1)` 以及四维尾行 `(1,1,65539,1)`、128/256 线程、两个 dtype 组合共 8 组。每组检查 allocating/out 一致、packed 字节与 float32 scale、输入不变、输出哨兵、存储地址、版本计数及 N=1 的高半字节为零。 + +同一运行分别检查原生产版本与候选。非有限编码只与**同一旧 packed 路径**对照以观察回退一致性，不扩展 API 约定的有限值输入域；其他布局对非有限编码可能有不同结果。独立脚本未传入旧扩展时，会在报告中把这项旧版本比较标为 NOT_CHECKED，不能冒充独立验证。 + +```bash +TORCH_CUDA_ARCH_LIST=8.9 MAX_JOBS=1 python scripts/verify_singleton_quantization.py \ + --build-directory /tmp/hadamard-singleton-check \ + --report /tmp/hadamard-singleton-check.json +``` + +## 未采用的候选 + +候选对 packed 的 N=1、Transform=false 分支使用符号直接确定有限非零输入的 ±7 量化码，零得到 0；scale 仍沿用原公式，非有限输入回退到原浮点运算。该候选通过完整穷举，以及原 700 组 packed 检查、16 组流/Graph、372 正例/63 负例/4 项 mutation 检查。 + +性能以**当前已经优化的 packed 实现**为基线，使用与正式测量分开的正确性阶段。RTX4090D、CUDA 12.8.61、Torch 2.6.0a0+ecf3bae40a.nv25.01、NumPy 1.26.4： + +- 目标为 65,536 / 262,147 / 1,048,579 行，N=1，FP16/BF16，128/256 线程。 +- 对照为小行数、其他 packed 小维度、原量化、Hadamard 及融合入口。 +- 三轮共 **132 条设备、132 条普通调用**成对记录；每条保留七个配对样本。设备为 64 节点 Graph × 10 次重放的 CUDA event 时间，主机调用另外同步计时。 +- 12 个 dtype/线程/轮次分组的目标几何平均只有 **1.0266–1.0437×**；单配置目标范围 **0.9987–1.0820×**。普通 allocating/out 目标整体几何平均约 **1.0151 / 1.0144×**。 +- 每场景“不退化超过 5%”通过，最大设备/主机退化分别为 **0.466% / 2.667%**；但全部目标分组都未达到预先设定的 **1.10×** 门槛，因此判为 **REJECT**，保持现有生产内核。 + +此处没有将 2.7%–4.4% 的局部设备收益写成全面加速，也没有在看到结果后降低门槛。[候选源码、原始数据、原生产版本穷举结果和复现工具](https://github.com/a962695448-rgb/Learning-CUDA/tree/5e80eadd156c82a299e7cc7991bf46d1904243e6/03_hadamard_tc/a962695448-rgb/results/singleton-quantization-20260917) 已固定归档；本轮新增验证工具不依赖采用该候选。 diff --git a/03_hadamard_tc/a962695448-rgb/reports/wide-pairs-20260918.md b/03_hadamard_tc/a962695448-rgb/reports/wide-pairs-20260918.md new file mode 100644 index 00000000..db5f424f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/reports/wide-pairs-20260918.md @@ -0,0 +1,36 @@ +# N32–256 半warp候选：未采用 + +基线bde0256（生产计算b49fff4）。候选四文件改为每16lane处理一行、每lane相邻两值，保持逻辑蝶形顺序、存储类型舍入和INT4约定。协议、源码和驱动已在首次实机采样前固定于95b13fa0f9519936e004b3ac6f120ab0a04e84ef的preregistered目录；本次未覆盖该登记。 + +## 正确性与入口覆盖 + +两版本CLI编译及候选七种模式的同一1876输入集合通过；基线original128/packed256也通过，两版各15非法CLI检查通过。七次执行不算更多独立输入。改动后的tuner通过96形状/dtype/scale/mode、384启动配置、1920原始样本；其计时仅作诊断，不参与采用判定。 + +候选通过416个选择分量的全有限FP16/BF16编码配置、512个有限输入下溢出/抵消兼容配置、64个新增流/Graph配置及120个原有执行上下文配置。还通过旧小维度变换、成对量化、单元素编码、packed量化、out接口和metadata回归。分量验证不穷尽所有向量；非有限中间值只说明与基线字节兼容，不宣称其INT4数学含义。 + +## 三轮完整结果 + +三轮均 **REJECT**，每轮70项门槛未过，共210项：159设备点、48普通调用点、3目标分组。全部678设备与1356主机记录完成，18个同源校准通过。独立复算与远端审计相同，没有删点、提高阈值或补跑到通过。 + +设备目标分组几何平均：Hadamard为1.097118–1.216922倍，融合INT4为1.366486–1.380550倍。虽然部分目标有收益，多个变换或控制配置退化超过5%，故不采用整个候选，生产内核保持b49fff4。上述数字只对应未采用候选在本轮RTX4090D的Graph设备执行，不能当作生产、普通调用、跨卡或模型端到端加速。 + +普通调用同时覆盖128/256线程及allocating/out。大尺寸采用采样前定义的max(128,min(5000,2**32//(rows*dim)))调用数，原新各组相同；18平衡组、5%双中位比限制、配对IQR<=10%及1.10倍目标门槛均保持登记原样。绝对离散度保留为诊断。 + +## 静态资源诊断 + +计时完成后，对实际受测.so执行只读cuobjdump，前后SHA与运行记录一致。下表为每线程寄存器数： + +| N | FP16变换 原/新 | BF16变换 原/新 | FP16融合 原/新 | BF16融合 原/新 | +|---|---|---|---|---| +| 32 | 11 / 14 | 11 / 14 | 19 / 23 | 19 / 23 | +| 64 | 16 / 21 | 16 / 21 | 21 / 23 | 21 / 23 | +| 128 | 21 / 25 | 21 / 25 | 22 / 26 | 22 / 26 | +| 256 | 23 / 36 | 23 / 36 | 27 / 34 | 27 / 37 | + +这些函数的静态LOCAL/STACK报告均为0。寄存器增加是可核对的变化，不等于已测得occupancy下降，也不能独自解释退化。后续可研究向量化读写和不同批量的并行度，必须用新源码和独立验证决定，不从本次胜点推导全面采用。 + +## 数据与状态 + +38份原始记录已完整回收：raw66034082字节，SHA256 e790a8873f9c2d8958dd4561fdec44483f6293e0c50711431107e2a22749b946。252输入文件前后不变。静态资源诊断另附31704字节原文及收据，不混入原始计时包。平台已确认本轮实例关机。 + +[固定预登记和完整结果](https://github.com/a962695448-rgb/Learning-CUDA/tree/c0a105c725343e3332c3465e125384bbd815d8eb/03_hadamard_tc/a962695448-rgb/results/wide-pairs-20260918) 提供全部驱动、源指纹、压缩原始数据、恢复工具与独立审计。 diff --git a/03_hadamard_tc/a962695448-rgb/scripts/build_torch_extension.py b/03_hadamard_tc/a962695448-rgb/scripts/build_torch_extension.py new file mode 100644 index 00000000..8883aa54 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/build_torch_extension.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Build/load the forward-only PyTorch adapter using the current PyTorch ABI.""" +import argparse +import json +import os +from pathlib import Path + + +def load_extension(verbose=False, build_directory=None): + import torch + from torch.utils.cpp_extension import load + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA-enabled PyTorch and a visible NVIDIA GPU are required") + root = Path(__file__).resolve().parents[1] + build = Path(build_directory) if build_directory else root / "build" / "torch_extension" + build.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("MAX_JOBS", "1") + # PyTorch selects the visible device's architecture unless the caller explicitly + # sets TORCH_CUDA_ARCH_LIST (e.g. 8.9 for RTX4090, 8.0 for A100). + return load( + name="infinitensor_hadamard_cuda", + sources=[str(root / "src" / "torch_binding.cu")], + extra_include_paths=[str(root / "include")], + extra_cflags=["-O3", "-std=c++17"], + extra_cuda_cflags=[ + "-O3", "-std=c++17", "-lineinfo", + "-U__CUDA_NO_HALF_OPERATORS__", "-U__CUDA_NO_HALF_CONVERSIONS__", + "-U__CUDA_NO_BFLOAT16_OPERATORS__", "-U__CUDA_NO_BFLOAT16_CONVERSIONS__", + "--expt-relaxed-constexpr", + ], + build_directory=str(build), + with_cuda=True, + verbose=verbose, + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--build-directory") + args = parser.parse_args() + try: + import torch + extension = load_extension(args.verbose, args.build_directory) + sample = torch.tensor([[3.0, 1.0]], device="cuda", dtype=torch.float16) + actual = extension.hadamard(sample) + torch.cuda.synchronize() + if actual.cpu().tolist() != [[4.0, 2.0]]: + raise RuntimeError(f"extension smoke failed: {actual.cpu().tolist()}") + print(json.dumps({"status": "PASS", "torch": torch.__version__, + "torch_cuda": torch.version.cuda, + "module": str(Path(extension.__file__).resolve()), + "gpu": torch.cuda.get_device_name(), "smoke": [[4.0, 2.0]]})) + return 0 + except Exception as error: + print(json.dumps({"status": "ERROR", "error": f"{type(error).__name__}: {error}"})) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/scripts/compare_reference.py b/03_hadamard_tc/a962695448-rgb/scripts/compare_reference.py new file mode 100644 index 00000000..392acd61 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/compare_reference.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Compare against the real, commit-pinned Dao-AILab CUDA implementation. + +No local oracle is substituted. Missing dependencies, GPU, or provenance return +nonzero and write an ERROR report. Numerical failures write raw metrics and +reproducible CPU tensor witnesses before returning nonzero. +""" +import argparse +import hashlib +import importlib.metadata +import json +import math +import platform +from pathlib import Path +import statistics +import subprocess +import sys +import time +import urllib.parse + +from build_torch_extension import load_extension + +REFERENCE_COMMIT = "e7706faf8d1c3b9f241e36860640ad1dac644ede" +REFERENCE_REPO = "https://github.com/Dao-AILab/fast-hadamard-transform" + + +def hash_file(path): + digest = hashlib.sha256() + with Path(path).open("rb") as file: + for block in iter(lambda: file.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def provenance(package, backend, repository): + direct = {} + try: + distribution = importlib.metadata.distribution("fast_hadamard_transform") + direct = json.loads(distribution.read_text("direct_url.json") or "{}") + except importlib.metadata.PackageNotFoundError: + pass + vcs = direct.get("vcs_info", {}) + url = urllib.parse.urlsplit(direct.get("url", "")) + official_url = url.hostname == "github.com" and url.path.rstrip("/").removesuffix(".git").lower() == "/dao-ailab/fast-hadamard-transform" + verified = official_url and vcs.get("commit_id") == REFERENCE_COMMIT + evidence = "PEP610 VCS installation" if verified else None + if repository is not None: + repository = Path(repository).resolve() + def git(*arguments): + return subprocess.check_output(["git", "-C", str(repository), *arguments], text=True, timeout=20).strip() + commit = git("rev-parse", "HEAD") + if commit != REFERENCE_COMMIT: + raise RuntimeError(f"reference checkout is {commit}, expected {REFERENCE_COMMIT}") + if git("status", "--porcelain", "--untracked-files=no"): + raise RuntimeError("reference checkout has modified tracked files") + module_local = Path(package.__file__).resolve().is_relative_to(repository) + backend_local = Path(backend.__file__).resolve().is_relative_to(repository) + direct_local = url.scheme == "file" and Path(urllib.parse.unquote(url.path)).resolve() == repository + if (module_local and backend_local) or direct_local: + verified = True + evidence = "clean pinned source checkout and local module/install provenance" + if not verified: + raise RuntimeError("Cannot verify the reference commit. Install the pinned VCS URL with " + "FAST_HADAMARD_TRANSFORM_FORCE_BUILD=TRUE, or pass --reference-repo " + "for a clean pinned checkout used for an in-place/local build.") + return {"repository": REFERENCE_REPO, "commit": REFERENCE_COMMIT, + "verification": evidence, "package_version": getattr(package, "__version__", None), + "python_module": str(Path(package.__file__).resolve()), + "cuda_module": str(Path(backend.__file__).resolve()), + "cuda_module_sha256": hash_file(backend.__file__)} + + +def make_input(torch, shape, dtype, pattern, seed, device): + generator = torch.Generator(device="cpu").manual_seed(seed) + if pattern == "uniform": + values = torch.rand(shape, generator=generator, dtype=torch.float32) * 2 - 1 + elif pattern == "normal": + values = torch.randn(shape, generator=generator, dtype=torch.float32) * 0.5 + elif pattern == "outlier": + values = (torch.rand(shape, generator=generator, dtype=torch.float32) * 2 - 1) * 0.001 + rows = values.reshape(-1, shape[-1]) + rows[torch.arange(rows.shape[0]), torch.arange(rows.shape[0]) * 7 % shape[-1]] = 8 + elif pattern == "zeros": + values = torch.zeros(shape, dtype=torch.float32) + else: + raise ValueError(pattern) + return values.to(device=device, dtype=dtype).contiguous() + + +def metrics(torch, actual, expected, tolerance): + if actual.shape != expected.shape or actual.dtype != expected.dtype or actual.device != expected.device: + raise RuntimeError("reference/ours output shape, dtype, or device mismatch") + actual_cpu, expected_cpu = actual.float().cpu(), expected.float().cpu() + errors = (actual_cpu - expected_cpu).abs() + finite = bool(torch.isfinite(errors).all()) + worst = int(torch.nan_to_num(errors, nan=float("inf")).reshape(-1).argmax()) + maximum = float(errors.reshape(-1)[worst]) if finite else None + failed = int((~torch.isfinite(errors) | (errors >= tolerance)).sum()) + return {"pass": finite and failed == 0, "max_abs_error": maximum, + "mean_abs_error": float(errors.mean()) if finite else None, + "mismatching_elements": int((actual_cpu != expected_cpu).sum()), + "elements_at_or_above_strict_limit": failed, "strict_abs_limit": tolerance, + "worst_flat_index": worst, + "ours_at_worst": float(actual_cpu.reshape(-1)[worst]) if finite else None, + "reference_at_worst": float(expected_cpu.reshape(-1)[worst]) if finite else None} + + +def check_rejections(torch, extension, device): + cases = { + "cpu": lambda: extension.hadamard(torch.ones((2, 16), dtype=torch.float16)), + "dtype": lambda: extension.hadamard(torch.ones((2, 16), device=device, dtype=torch.float32)), + "rank": lambda: extension.hadamard(torch.ones((1, 2, 16), device=device, dtype=torch.float16)), + "noncontiguous": lambda: extension.hadamard(torch.ones((16, 16), device=device, dtype=torch.float16).t()), + "non_power_of_two": lambda: extension.hadamard(torch.ones((2, 3), device=device, dtype=torch.float16)), + "dimension_too_large": lambda: extension.hadamard(torch.ones((2, 512), device=device, dtype=torch.float16)), + "empty": lambda: extension.hadamard(torch.empty((0, 16), device=device, dtype=torch.float16)), + "scale_nan": lambda: extension.hadamard(torch.ones((2, 16), device=device, dtype=torch.float16), float("nan")), + "scale_zero": lambda: extension.hadamard(torch.ones((2, 16), device=device, dtype=torch.float16), 0), + "autograd": lambda: extension.hadamard(torch.ones((2, 16), device=device, dtype=torch.float16, requires_grad=True)), + } + results = [] + for name, operation in cases.items(): + try: + operation() + except (RuntimeError, ValueError) as error: + results.append({"case": name, "pass": True, "error": str(error).splitlines()[0]}) + else: + raise RuntimeError(f"extension accepted invalid input: {name}") + return results + + +def benchmark_pair(torch, ours, reference, values, scale, groups, repetitions, warmup): + functions = {"ours": lambda: ours(values, scale), "dao": lambda: reference(values, scale)} + for function in functions.values(): + for _ in range(warmup): + function() + torch.cuda.synchronize(values.device) + samples = {name: [] for name in functions} + for group in range(groups): + order = ("ours", "dao") if group % 2 == 0 else ("dao", "ours") + for name in order: + begin, finish = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + begin.record() + for _ in range(repetitions): + functions[name]() + finish.record() + finish.synchronize() + samples[name].append(begin.elapsed_time(finish) * 1000 / repetitions) + medians = {name: statistics.median(times) for name, times in samples.items()} + return {"method": "eager_api", "samples_us": samples, "median_us": medians, + "dao_over_ours": medians["dao"] / medians["ours"], + "groups": groups, "repetitions_per_group": repetitions, "warmup": warmup, + "scope": "CUDA-event interval around allocating PyTorch API calls, including GPU idle gaps from host dispatch; no H2D, D2H, build, or validation"} + + +def benchmark_graph_pair(torch, ours, reference, values, scale, groups, captured_calls, + replays, warmup, graph_warmup): + """Measure captured GPU work with Python dispatch amortized over many calls.""" + functions = {"ours": lambda: ours(values, scale), "dao": lambda: reference(values, scale)} + graphs, outputs, expected = {}, {}, {} + capture_stream = torch.cuda.Stream(device=values.device) + capture_stream.wait_stream(torch.cuda.current_stream(values.device)) + with torch.cuda.stream(capture_stream): + for name, function in functions.items(): + for _ in range(warmup): + function() + expected[name] = function() + capture_stream.synchronize() + for name, function in functions.items(): + graph = torch.cuda.CUDAGraph() + # Retain every output for both implementations. Each captured call gets + # its own allocation, and replays reuse those fixed allocations. Separate + # private graph pools allow the alternating measurement order below. + with torch.cuda.graph(graph, stream=capture_stream): + outputs[name] = [function() for _ in range(captured_calls)] + graphs[name] = graph + torch.cuda.synchronize(values.device) + for graph in graphs.values(): + # Always execute once before validating, even when warmup is disabled. + graph.replay() + torch.cuda.synchronize(values.device) + for name, tensors in outputs.items(): + for index, output in enumerate(tensors): + if not torch.equal(output, expected[name]): + raise RuntimeError(f"CUDA graph output differs from eager output: {name}, call {index}") + + for graph in graphs.values(): + for _ in range(graph_warmup): + graph.replay() + torch.cuda.synchronize(values.device) + + intervals = {name: [] for name in functions} + samples = {name: [] for name in functions} + orders = [] + for group in range(groups): + order = ("ours", "dao") if group % 2 == 0 else ("dao", "ours") + orders.append(list(order)) + for name in order: + begin, finish = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + begin.record() + for _ in range(replays): + graphs[name].replay() + finish.record() + finish.synchronize() + elapsed_ms = begin.elapsed_time(finish) + intervals[name].append(elapsed_ms) + samples[name].append(elapsed_ms * 1000 / (replays * captured_calls)) + medians = {name: statistics.median(times) for name, times in samples.items()} + return {"method": "cuda_graph", "samples_us": samples, "median_us": medians, + "raw_event_intervals_ms": intervals, + "dao_over_ours": medians["dao"] / medians["ours"], + "groups": groups, "captured_calls_per_graph": captured_calls, + "replays_per_group": replays, "calls_per_group": replays * captured_calls, + "api_warmup_calls": warmup, "graph_warmup_replays": graph_warmup, + "validation_replays": 1, "all_captured_outputs_equal_eager": True, + "group_order": orders, + "output_lifetime": "One output per captured call retained through measurement; separate private graph pools; fixed input and output addresses on replay", + "scope": "CUDA-event interval around graph replays divided by captured calls; includes captured GPU work and amortized replay scheduling, excludes per-call Python dispatch and allocation, capture, validation, H2D and D2H; not standalone kernel latency"} + + +def run(args, report): + import torch + if not torch.cuda.is_available(): + raise RuntimeError("No CUDA-enabled PyTorch/GPU is available; reference comparison was not run") + device = torch.device(args.device) + if device.type != "cuda": + raise ValueError("--device must name a CUDA device") + torch.cuda.set_device(device) + # The checkout verifies provenance; imports must resolve to the installed + # package. Prepending the source tree can shadow its PEP 610 metadata with + # setup.py's source egg-info and incorrectly reject a verified local build. + import fast_hadamard_transform as reference_package + import fast_hadamard_transform_cuda as reference_backend + reference = reference_package.hadamard_transform + report["reference"] = provenance(reference_package, reference_backend, args.reference_repo) + extension = load_extension(args.verbose, args.build_directory) + report["environment"] = {"python": platform.python_version(), "torch": torch.__version__, + "torch_cuda": torch.version.cuda, "device": str(device), + "gpu": torch.cuda.get_device_name(device), + "compute_capability": list(torch.cuda.get_device_capability(device)), + "extension": str(Path(extension.__file__).resolve()), + "extension_sha256": hash_file(extension.__file__)} + report["rejected_inputs"] = check_rejections(torch, extension, device) + report["cases"] = [] + failures = 0 + with torch.inference_mode(): + for dtype_name, dtype, limit in (("fp16", torch.float16, 1e-2), ("bf16", torch.bfloat16, 5e-2)): + for dim in (1, 2, 4, 8, 16, 32, 64, 128, 256): + for shape in ((1, dim), (3, dim), (17, dim), (1, 3, 7, dim), (2, 5, 13, dim)): + for pattern in ("uniform", "normal", "outlier", "zeros"): + seeds = (2026,) if pattern == "zeros" else (2026, 95811, 314159) + for seed in seeds: + values = make_input(torch, shape, dtype, pattern, seed, device) + for normalized in (False, True): + scale = float(torch.tensor(1 / math.sqrt(dim) if normalized else 1, dtype=torch.float32)) + actual = extension.hadamard(values, scale) + expected = reference(values, scale) + result = metrics(torch, actual, expected, limit) + packed, scales = extension.hadamard_int4(values, scale) + split_packed, split_scales = extension.quantize_int4(actual) + quant_exact = bool(torch.equal(packed, split_packed) and torch.equal(scales, split_scales)) + result.update({"dtype": dtype_name, "shape": list(shape), "pattern": pattern, + "seed": seed, "normalized": normalized, "scale": scale, + "fused_vs_split_int4_exact": quant_exact}) + result["pass"] = result["pass"] and quant_exact + if not result["pass"]: + failures += 1 + witness_directory = Path(args.json).with_suffix("").with_name(Path(args.json).stem + "_failures") + witness_directory.mkdir(parents=True, exist_ok=True) + witness = witness_directory / f"case_{len(report['cases']):05d}.pt" + torch.save({"input": values.cpu(), "ours": actual.cpu(), "dao": expected.cpu(), + "absolute_error": (actual.float() - expected.float()).abs().cpu(), + "metadata": result}, witness) + result["witness"] = str(witness) + report["cases"].append(result) + print(f"CHECKED dtype={dtype_name} dim={dim} cases={len(report['cases'])} failures={failures}", flush=True) + # Data production and the wrapper execute on a non-default stream; the + # reference is evaluated only after this stream completes. + stream = torch.cuda.Stream(device=device) + with torch.cuda.stream(stream): + stream_input = torch.arange(17 * 256, device=device, dtype=torch.float32).remainder(29).div(32).to(torch.float16).reshape(17, 256) + stream_output = extension.hadamard(stream_input, 1.0) + stream.synchronize() + report["non_default_stream"] = metrics(torch, stream_output, reference(stream_input, 1.0), 1e-2) + if not report["non_default_stream"]["pass"]: + failures += 1 + report["multi_device_guard"] = "not_exercised: requires a second visible GPU" + if torch.cuda.device_count() > 1: + original = torch.cuda.current_device() + other = (original + 1) % torch.cuda.device_count() + values = torch.ones((3, 16), device=f"cuda:{other}", dtype=torch.float16) + guarded_output = extension.hadamard(values) + if guarded_output.device.index != other or torch.cuda.current_device() != original: + raise RuntimeError("device guard failed to preserve caller device/output device") + report["multi_device_guard"] = "PASS" + report["benchmarks"] = [] + report["graph_benchmarks"] = [] + if args.benchmark: + for dtype_name, dtype in (("fp16", torch.float16), ("bf16", torch.bfloat16)): + for dim in (16, 64, 256): + for shape in ((17, dim), (4, 128, 8, dim)): + values = make_input(torch, shape, dtype, "normal", 2026, device) + entry = benchmark_pair(torch, extension.hadamard, reference, values, 1.0, + args.groups, args.repetitions, args.warmup) + entry.update({"dtype": dtype_name, "shape": list(shape), "scale": 1.0}) + report["benchmarks"].append(entry) + graph_entry = benchmark_graph_pair( + torch, extension.hadamard, reference, values, 1.0, + args.groups, args.graph_calls, args.graph_replays, + args.warmup, args.graph_warmup) + graph_entry.update({"dtype": dtype_name, "shape": list(shape), "scale": 1.0}) + report["graph_benchmarks"].append(graph_entry) + maxima = {} + for dtype in ("fp16", "bf16"): + errors = [case["max_abs_error"] for case in report["cases"] if case["dtype"] == dtype] + maxima[dtype] = None if any(error is None for error in errors) else max(errors) + report["summary"] = {"cases": len(report["cases"]), "failures": failures, + "max_abs_error_by_dtype": maxima} + report["status"] = "PASS" if failures == 0 else "FAIL" + return 0 if failures == 0 else 1 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--json", default="results/third_party_reference.json") + parser.add_argument("--reference-repo", help="Optional clean pinned checkout used to build the installed reference") + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--benchmark", action="store_true", help="Run both eager API and CUDA Graph benchmarks") + parser.add_argument("--groups", type=int, default=5) + parser.add_argument("--repetitions", type=int, default=200) + parser.add_argument("--warmup", type=int, default=25) + parser.add_argument("--graph-calls", type=int, default=64, help="Independent API calls captured in each CUDA graph") + parser.add_argument("--graph-replays", type=int, default=20, help="CUDA graph replays per timing group") + parser.add_argument("--graph-warmup", type=int, default=5, help="Untimed graph warmup replays after validating one replay") + parser.add_argument("--build-directory") + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + if min(args.groups, args.repetitions, args.graph_calls, args.graph_replays) < 1 or min(args.warmup, args.graph_warmup) < 0: + parser.error("groups/repetitions/graph-calls/graph-replays must be positive and warmups nonnegative") + output = Path(args.json) + output.parent.mkdir(parents=True, exist_ok=True) + report = {"status": "RUNNING", "reference_commit_required": REFERENCE_COMMIT, + "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + start = time.monotonic() + try: + code = run(args, report) + except Exception as error: + report.update({"status": "ERROR", "error": f"{type(error).__name__}: {error}"}) + code = 2 + report["elapsed_seconds"] = time.monotonic() - start + output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + print(json.dumps({"status": report["status"], "json": str(output), + "summary": report.get("summary"), "error": report.get("error")}), flush=True) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/scripts/run_validation.py b/03_hadamard_tc/a962695448-rgb/scripts/run_validation.py new file mode 100644 index 00000000..c309ce66 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/run_validation.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Reproduce correctness, rejected-CLI, and optional measured benchmark logs.""" +import argparse +from pathlib import Path +import subprocess +import time + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--benchmark", action="store_true") + parser.add_argument("--label", default="local") + args = parser.parse_args() + if not args.label or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for c in args.label): + parser.error("--label must contain only letters, digits, '-' or '_'") + root = Path(__file__).resolve().parents[1] + executable = root / "build/hadamard" + results = root / "results" + results.mkdir(exist_ok=True) + with (results / f"validation_{args.label}.log").open("w") as log: + def run(arguments, expected=0): + command = [str(executable), *map(str, arguments)] + log.write("COMMAND " + repr(command) + "\n") + log.flush() + start = time.monotonic() + result = subprocess.run(command, cwd=root, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=180) + log.write(result.stdout) + log.write(f"EXIT_CODE {result.returncode}; ELAPSED_SECONDS {time.monotonic()-start:.3f}\n\n") + log.flush() + if result.returncode != expected: + raise RuntimeError(f"unexpected exit {result.returncode}, expected {expected}: {command}\n{result.stdout}") + print("PASS", " ".join(map(str, arguments)), flush=True) + + run(["--self-test"]) + for invalid in ( + ["--benchmark", "--dim", "0"], + ["--benchmark", "--dim", "3"], + ["--benchmark", "--dim", "512"], + ["--benchmark", "--batch", "-1"], + ["--benchmark", "--seq", "0"], + ["--benchmark", "--heads", "abc"], + ["--benchmark", "--dtype", "fp32"], + ["--benchmark", "--scale", "nan"], + ["--benchmark", "--scale", "2"], + ["--benchmark", "--batch", "18446744073709551615", "--seq", "2"], + ["--benchmark", "--batch", "18446744073709551616"], + ["--benchmark", "--repetitions", "0"], + ["--benchmark", "--warmup", "-1"], + ["--benchmark", "--dim"], + ["--unsupported"], + ): + run(invalid, expected=2) + if args.benchmark: + csv = results / f"benchmark_{args.label}.csv" + # Do not mix independent runs in one CSV silently. + if csv.exists(): + raise FileExistsError(f"Choose a fresh --label; CSV exists: {csv}") + shapes = [(1, 1, 1, 1), (1, 1, 1, 16), (1, 1, 17, 256), + (4, 128, 8, 16), (4, 128, 8, 64), (4, 128, 8, 256), + (4, 512, 8, 256)] + for dtype in ("fp16", "bf16"): + for batch, seq, heads, dim in shapes: + run(["--benchmark", "--batch", batch, "--seq", seq, "--heads", heads, + "--dim", dim, "--dtype", dtype, "--repetitions", 300, + "--warmup", 30, "--csv", csv]) + run(["--benchmark", "--batch", 1, "--seq", 257, "--heads", 1, + "--dim", 256, "--dtype", dtype, "--normalize", "--repetitions", 300, + "--warmup", 30, "--csv", csv]) + + +if __name__ == "__main__": + main() diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_block_threads.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_block_threads.py new file mode 100644 index 00000000..0da6570b --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_block_threads.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""复用原1,800组Dao矩阵，核查线程兼容性及N256显式融合布局。""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import time + +import compare_reference + + +def load_for_validation(build_directory): + from torch.utils.cpp_extension import load + root = Path(__file__).resolve().parents[1] + build = Path(build_directory) + build.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("MAX_JOBS", "1") + return load(name="infinitensor_hadamard_block_threads_check", sources=[str(root / "src/torch_binding.cu")], + extra_include_paths=[str(root / "include")], extra_cflags=["-O3", "-std=c++17"], + extra_cuda_cflags=["-O3", "-std=c++17", "-lineinfo", + "-U__CUDA_NO_HALF_OPERATORS__", "-U__CUDA_NO_HALF_CONVERSIONS__", + "-U__CUDA_NO_BFLOAT16_OPERATORS__", "-U__CUDA_NO_BFLOAT16_CONVERSIONS__", + "--expt-relaxed-constexpr"], build_directory=str(build), verbose=True) + + +def bitwise_equal(torch, actual, expected, name): + a = actual if isinstance(actual, tuple) else (actual,) + b = expected if isinstance(expected, tuple) else (expected,) + if len(a) != len(b) or any(x.shape != y.shape or x.dtype != y.dtype or + not torch.equal(x.view(torch.uint8), y.view(torch.uint8)) for x, y in zip(a, b)): + raise RuntimeError("default/128/256 bitwise mismatch: " + name) + + +class CheckedInterface: + """同一输入核查三种调用，向原Dao对照器返回显式256结果。""" + def __init__(self, torch, extension): + self.torch, self.extension, self.__file__ = torch, extension, extension.__file__ + + def hadamard(self, values, scale=1.0): + default = self.extension.hadamard(values, scale) + explicit128 = self.extension.hadamard(values, scale, block_threads=128) + explicit256 = self.extension.hadamard(values, scale, block_threads=256) + bitwise_equal(self.torch, default, explicit128, "transform default vs128") + bitwise_equal(self.torch, explicit256, explicit128, "transform 256 vs128") + return explicit256 + + def hadamard_int4(self, values, scale=1.0): + default = self.extension.hadamard_int4(values, scale) + explicit128 = self.extension.hadamard_int4(values, scale, block_threads=128) + explicit256 = self.extension.hadamard_int4(values, scale, block_threads=256) + bitwise_equal(self.torch, default, explicit128, "fused default vs128") + bitwise_equal(self.torch, explicit256, explicit128, "fused 256 vs128") + if values.shape[-1] == 256: + contiguous = self.extension.hadamard_int4(values, scale, block_threads=128, + fused_layout="contiguous256") + bitwise_equal(self.torch, contiguous, explicit128, "fused contiguous256 vs original128") + return explicit256 + + def quantize_int4(self, values): + default = self.extension.quantize_int4(values) + explicit128 = self.extension.quantize_int4(values, block_threads=128) + explicit256 = self.extension.quantize_int4(values, block_threads=256) + bitwise_equal(self.torch, default, explicit128, "quantize default vs128") + bitwise_equal(self.torch, explicit256, explicit128, "quantize 256 vs128") + return explicit256 + + +def check_thread_rejections(torch, extension): + values = torch.ones((3, 16), device="cuda", dtype=torch.float16) + results = [] + for name in ("hadamard", "hadamard_int4", "quantize_int4"): + for invalid in (-1, 0, 32, 64, 127, 129, 512, 128.5, "256"): + try: + getattr(extension, name)(values, block_threads=invalid) + except (RuntimeError, TypeError, ValueError) as error: + results.append({"method": name, "value": invalid, "pass": True, "error": str(error).splitlines()[0]}) + else: + raise RuntimeError(f"invalid block_threads accepted: {name} {invalid!r}") + return results + + +def check_fused_layout_rejections(torch, extension): + values = torch.ones((3, 256), device="cuda", dtype=torch.float16) + cases = [("unknown", values, {"fused_layout": "unknown"}), + ("empty", values, {"fused_layout": ""}), + ("wrong_type", values, {"fused_layout": 1}), + ("wrong_threads", values, {"fused_layout": "contiguous256", "block_threads": 256}), + ("wrong_dimension", values[:, :128].contiguous(), {"fused_layout": "contiguous256"})] + results = [] + for name, data, options in cases: + try: + extension.hadamard_int4(data, **options) + except (RuntimeError, TypeError, ValueError) as error: + results.append({"case": name, "pass": True, "error": str(error).splitlines()[0]}) + else: + raise RuntimeError("invalid fused layout accepted: " + name) + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reference-repo", type=Path, required=True) + parser.add_argument("--build-directory", type=Path, required=True, help="独立扩展构建目录") + parser.add_argument("--json", type=Path, required=True, help="新的结果文件，禁止覆盖") + args = parser.parse_args() + if args.json.exists(): + parser.error("result already exists; choose a fresh output") + args.json.parent.mkdir(parents=True, exist_ok=True) + report = {"status": "RUNNING", "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "reference_commit_required": compare_reference.REFERENCE_COMMIT, + "scope": "Original 1800 input/shape/dtype/scale cases; each checks default==128==256 bitwise. Repeated calls are not new distinct cases. No performance measurement here."} + code = 1 + try: + import torch + extension = load_for_validation(args.build_directory) + sample = torch.tensor([[3.0, 1.0]], device="cuda", dtype=torch.float16) + bitwise_equal(torch, extension.hadamard(sample), extension.hadamard(sample, 1.0, 128), "legacy one-argument transform") + bitwise_equal(torch, extension.hadamard_int4(sample), extension.hadamard_int4(sample, 1.0, 128), "legacy one-argument fused") + report["legacy_one_argument_signatures"] = "PASS" + checked = CheckedInterface(torch, extension) + # 只在本验证进程替换装载入口；不修改原脚本、原oracle或安装环境。 + compare_reference.load_extension = lambda *unused, **unused_kwargs: checked + existing_args = argparse.Namespace(device="cuda:0", reference_repo=str(args.reference_repo), + verbose=False, build_directory=str(args.build_directory), json=str(args.json), benchmark=False) + code = compare_reference.run(existing_args, report) + if code: + raise RuntimeError("original Dao correctness matrix failed") + report["thread_value_rejections"] = check_thread_rejections(torch, extension) + report["fused_layout_rejections"] = check_fused_layout_rejections(torch, extension) + report["optional_fused_layout"] = { + "layout": "contiguous256", "block_threads": 128, + "original_matrix_cases_checked": sum(case["shape"][-1] == 256 for case in report["cases"]), + "bitwise_equal_to_original_fused": True, + "scope": "N256 subset of the same 1800 inputs, not additional distinct inputs; other dimensions and ordinary transforms use the original kernel."} + report["default_and_explicit128_and_256_bitwise_equal"] = True + root = Path(__file__).resolve().parents[1] + report["source_sha256"] = {name: hashlib.sha256((root / name).read_bytes()).hexdigest() + for name in ("include/kernels.cuh", "include/contiguous256.cuh", "include/reference.hpp", "src/torch_binding.cu", "scripts/compare_reference.py", "scripts/verify_block_threads.py")} + report["status"] = "PASS" + code = 0 + except Exception as error: + import traceback + report.update(status="FAIL", error=repr(error), traceback=traceback.format_exc()) + print(report["traceback"], flush=True) + code = 1 + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + args.json.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "summary": report.get("summary"), + "invalid_thread_cases": len(report.get("thread_value_rejections", []))}), flush=True) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_execution_context.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_execution_context.py new file mode 100644 index 00000000..c9034a55 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_execution_context.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Check stream ordering and changing-input CUDA Graph replay against a CPU oracle. + +Negative controls deliberately use the default stream or retain a stale graph +output. A run passes only if both faults are detected for every configuration. +This is correctness coverage on one device, not a performance measurement. +""" + +import argparse +import hashlib +import json +import math +from pathlib import Path +import time +import traceback + +from build_torch_extension import load_extension + +DELAY_CYCLES = 50_000_000 +SHAPES = ((3, 1), (17, 8), (17, 64), (17, 256), (2, 1, 3, 256)) + + +def configurations(): + for shape in SHAPES: + for threads in (128, 256): + common = {"shape": shape, "block_threads": threads} + yield dict(common, method="quantize_int4") + layouts = ("original", "auto", "packed") if shape[-1] <= 16 else ("original", "auto") + for method in ("hadamard", "hadamard_int4"): + for layout in layouts: + yield dict(common, method=method, row_layout=layout) + if shape[-1] == 256 and threads == 128: + yield dict(common, method="hadamard_int4", fused_layout="contiguous256") + + +def invoke(op, x, case): + keywords = {key: value for key, value in case.items() if key not in ("shape", "method")} + if case["method"] != "quantize_int4": + keywords["scale"] = 1 / math.sqrt(case["shape"][-1]) + return getattr(op, case["method"])(x, **keywords) + + +def fixture(torch, shape, dtype, phase): + index = torch.arange(math.prod(shape), dtype=torch.float32).reshape(shape) + if phase == 0: + values = index * 0 + elif phase == 1: + values = (index.remainder(29) - 14) / 16 + elif phase == 2: + values = (index.remainder(17) - 8) / 8 + 1 / 16 + else: + values = (index.remainder(11) - 5) / 4 - 1 / 8 + return values.to(dtype) + + +def reference(torch, np, cpu_input, case): + values = cpu_input.float().numpy().copy() + n = values.shape[-1] + if case["method"] != "quantize_int4": + stride = 1 + while stride < n: + blocks = values.reshape(-1, n // (2 * stride), 2 * stride) + left, right = blocks[..., :stride].copy(), blocks[..., stride:].copy() + blocks[..., :stride], blocks[..., stride:] = left + right, left - right + stride *= 2 + # The fused contract quantizes the rounded FP16/BF16 transform output. + rounded = torch.from_numpy(values * np.float32(1 / math.sqrt(n))).to(cpu_input.dtype) + if case["method"] == "hadamard": + return (rounded,) + values = rounded.float().numpy() + maximum = np.max(np.abs(values), axis=-1) + scales = np.where(maximum == 0, np.float32(1), maximum / np.float32(7)).astype(np.float32) + quantized = np.clip(np.rint(values / scales[..., None]), -7, 7).astype(np.int32) + padded = np.zeros((*quantized.shape[:-1], (n + 1) // 2 * 2), dtype=np.int32) + padded[..., :n] = quantized + packed = ((padded[..., ::2] & 15) | ((padded[..., 1::2] & 15) << 4)).astype(np.uint8) + return torch.from_numpy(packed), torch.from_numpy(scales) + + +def outputs(value): + return value if isinstance(value, tuple) else (value,) + + +def compare(torch, actual, expected): + actual = outputs(actual) + if len(actual) != len(expected): + raise AssertionError("output tuple length differs") + for index, (left, right) in enumerate(zip(actual, expected)): + left = left.detach().cpu() + if left.shape != right.shape or left.dtype != right.dtype: + raise AssertionError(f"output {index} shape/dtype differs") + if not torch.equal(left.contiguous().view(torch.uint8), right.contiguous().view(torch.uint8)): + raise AssertionError(f"output {index} bits differ from the CPU oracle") + + +def require_rejection(torch, bad_output, expected, fault): + try: + compare(torch, bad_output, expected) + except AssertionError as error: + return {"fault": fault, "detected": True, "reason": str(error)} + raise AssertionError(f"negative control was not detected: {fault}") + + +def stream_probe(torch, op, case, cpu_input, wrong_stream): + x, payload = torch.zeros_like(cpu_input, device="cuda"), cpu_input.cuda() + side, default = torch.cuda.Stream(), torch.cuda.default_stream() + # Warm allocations on both streams before the delayed producer. All tensors + # remain alive until synchronization, so allocator reuse cannot race a free. + for stream in (default, side): + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + warm = invoke(op, x, case) + del warm + torch.cuda.synchronize() + with torch.cuda.stream(side): + torch.cuda._sleep(DELAY_CYCLES) + x.copy_(payload, non_blocking=True) + if wrong_stream: + # Test-only fault: remove the producer/consumer stream ordering. + with torch.cuda.stream(default): + result = invoke(op, x, case) + else: + result = invoke(op, x, case) + torch.cuda.synchronize() + compare(torch, x, (cpu_input,)) + return result + + +def check_case(torch, np, op, case, dtype): + cpu_inputs = [fixture(torch, case["shape"], dtype, phase) for phase in range(4)] + expected = [reference(torch, np, value, case) for value in cpu_inputs] + good = stream_probe(torch, op, case, cpu_inputs[1], False) + compare(torch, good, expected[1]) + bad = stream_probe(torch, op, case, cpu_inputs[1], True) + stream_control = require_rejection(torch, bad, expected[1], "consumer_on_default_stream") + + x = cpu_inputs[0].cuda() + payloads = [value.cuda() for value in cpu_inputs] + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + for _ in range(3): + warm = invoke(op, x, case) + del warm + side.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=side): + result = invoke(op, x, case) + stale = None + graph_control = None + replays = [] + for phase, payload in enumerate(payloads): + with torch.cuda.stream(side): + x.copy_(payload, non_blocking=True) + graph.replay() + side.synchronize() + compare(torch, result, expected[phase]) + compare(torch, x, (cpu_inputs[phase],)) + if phase == 0: + stale = tuple(value.cpu().clone() for value in outputs(result)) + elif phase == 1: + graph_control = require_rejection(torch, stale, expected[phase], "stale_graph_output") + replays.append({"phase": phase, "oracle_bitwise_equal": True, "input_unchanged": True}) + return dict(case, dtype=str(dtype), status="PASS", stream_oracle_bitwise_equal=True, + stream_input_unchanged=True, graph_replays=replays, + negative_controls=[stream_control, graph_control]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-directory", type=Path, required=True) + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args() + try: + args.json.parent.mkdir(parents=True, exist_ok=True) + stream = args.json.open("x", encoding="utf-8") + except OSError as error: + parser.error(f"cannot create output: {error}; choose a new path") + report = {"status": "RUNNING", "cases": [], "delay_cycles": DELAY_CYCLES, + "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "limitations": ["Single-device correctness only; no speedup claim.", + "Delay uses the test-only torch.cuda._sleep API; unavailable or undetected controls fail.", + "Representative shapes/layouts, not every possible stream schedule."]} + + def save(): + stream.seek(0) + stream.write(json.dumps(report, indent=2, allow_nan=False) + "\n") + stream.truncate() + stream.flush() + + with stream: + save() + try: + import numpy as np + import torch + if not torch.cuda.is_available() or torch.version.hip: + raise RuntimeError("a CUDA-enabled NVIDIA GPU is required") + if not callable(getattr(torch.cuda, "_sleep", None)): + raise RuntimeError("torch.cuda._sleep is required for the negative control") + op = load_extension(verbose=True, build_directory=str(args.build_directory)) + report["environment"] = {"torch": torch.__version__, "torch_cuda": torch.version.cuda, + "numpy": np.__version__, "gpu": torch.cuda.get_device_name(), + "capability": list(torch.cuda.get_device_capability()), + "extension_sha256": hashlib.sha256(Path(op.__file__).read_bytes()).hexdigest()} + root = Path(__file__).resolve().parents[1] + report["source_sha256"] = {str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(root.rglob("*")) if path.is_file() + and path.suffix in (".cu", ".cuh") and not {"build", "results"}.intersection(path.relative_to(root).parts)} + for name in ("scripts/build_torch_extension.py", "scripts/verify_execution_context.py"): + report["source_sha256"][name] = hashlib.sha256((root / name).read_bytes()).hexdigest() + for dtype in (torch.float16, torch.bfloat16): + for case in configurations(): + report["active_case"] = dict(case, dtype=str(dtype)) + save() + report["cases"].append(check_case(torch, np, op, case, dtype)) + report.pop("active_case", None) + report["status"] = "PASS" + except KeyboardInterrupt: + report.update(status="INTERRUPTED", traceback=traceback.format_exc()) + except Exception: + report.update(status="FAIL", traceback=traceback.format_exc()) + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + save() + print(json.dumps({"status": report["status"], "configurations": len(report["cases"])})) + if report["status"] != "PASS": + print(report["traceback"]) + return 130 if report["status"] == "INTERRUPTED" else 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_hadamard_packed.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_hadamard_packed.py new file mode 100644 index 00000000..326b4112 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_hadamard_packed.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check paired Hadamard storage, rounded transforms and fused INT4 bytes.""" +import argparse +import json +import math +from pathlib import Path + +from build_torch_extension import load_extension +from verify_execution_context import check_case, compare, reference + + +def transform_reference(torch, np, cpu, scale): + values = cpu.float().numpy().copy() + size = values.shape[-1] + stride = 1 + while stride < size: + blocks = values.reshape(-1, size // (2 * stride), 2 * stride) + left = blocks[..., :stride].copy() + right = blocks[..., stride:].copy() + blocks[..., :stride] = left + right + blocks[..., stride:] = left - right + stride *= 2 + rounded = torch.from_numpy(values * np.float32(scale)).to(cpu.dtype) + assert torch.all(torch.isfinite(rounded.float())) + return rounded + + +def guarded_outputs(torch, x, method): + specs = [(tuple(x.shape), x.dtype)] if method == 'hadamard' else [ + ((*x.shape[:-1], x.shape[-1] // 2), torch.uint8), + (tuple(x.shape[:-1]), torch.float32), + ] + buffers, guards = [], [] + for shape, dtype in specs: + base = torch.full((math.prod(shape) + 17,), 37, dtype=dtype, device=x.device) + buffers.append(base[9:-8].reshape(shape)) + guards.append(base) + return tuple(buffers), guards + + +def verify(torch, np, candidate, control=None): + records = [] + raw = np.arange(65536, dtype=np.uint16).view(np.int16).copy() + for dtype in (torch.float16, torch.bfloat16): + values = torch.from_numpy(raw).view(dtype) + values = values[torch.isfinite(values.float())] + assert values.numel() == (63488 if dtype == torch.float16 else 65280) + # Small anchors keep every transform finite, even for the largest input. + anchors = torch.tensor([0, .5, -.5, .25, -.25, torch.finfo(dtype).tiny], dtype=dtype) + for dim in (2, 4, 8, 16): + for position in (0, dim - 1): + cpu = anchors[torch.arange(values.numel()) % anchors.numel()][:, None].expand(-1, dim).clone() + cpu[:, position] = values + for scale in (1., 1 / math.sqrt(dim)): + transformed = transform_reference(torch, np, cpu, scale) + expected = { + 'hadamard': (transformed,), + 'hadamard_int4': reference(torch, np, transformed, {'method': 'quantize_int4'}), + } + for offset in (0, 1): + pool = torch.full((cpu.numel() + offset + 1,), 37, dtype=dtype, device='cuda') + x = pool[offset:offset + cpu.numel()].reshape(cpu.shape) + x.copy_(cpu) + version = x._version + assert x.data_ptr() % 4 == offset * 2 + for threads in (128, 256): + options = dict(scale=scale, block_threads=threads, row_layout='packed') + for method in ('hadamard', 'hadamard_int4'): + if control is not None: + compare(torch, getattr(control, method)(x, **options), expected[method]) + compare(torch, getattr(candidate, method)(x, **options), expected[method]) + buffers, guards = guarded_outputs(torch, x, method) + pointers = [b.data_ptr() for b in buffers] + versions = [b._version for b in buffers] + assert getattr(candidate, method + '_out')(x, *buffers, **options) is None + compare(torch, buffers, expected[method]) + assert [b.data_ptr() for b in buffers] == pointers + assert [b._version for b in buffers] == [v + 1 for v in versions] + for guard in guards: + assert torch.all(guard[:9] == 37) and torch.all(guard[-8:] == 37) + compare(torch, x, (cpu,)) + assert x._version == version + if offset: + assert pool[0].item() == 37 + assert pool[-1].item() == 37 + records.append({'dtype': str(dtype), 'dim': dim, 'position': position, + 'scale': scale, 'offset': offset, 'threads': threads, + 'finite_component_encodings': values.numel(), + 'both_methods_allocating_and_out': 'PASS', 'guards_versions_and_input': 'PASS'}) + return {'status': 'PASS', 'records': records, 'unique_finite_component_encodings': 128768, + 'scope': 'Selected first/last components contain all finite storage encodings; remaining components use six bounded anchor classes. Not all vector combinations.'} + + +def contexts(torch, np, op): + records = [] + for dtype in (torch.float16, torch.bfloat16): + for shape in ((17, 2), (4097, 16), (1, 3, 7, 8)): + for method in ('hadamard', 'hadamard_int4'): + for threads in (128, 256): + case = dict(shape=shape, method=method, block_threads=threads, row_layout='packed') + records.append(check_case(torch, np, op, case, dtype)) + return records + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build-directory', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + args = parser.parse_args() + import torch + import numpy as np + op = load_extension(build_directory=args.build_directory) + report = verify(torch, np, op) + report['contexts'] = contexts(torch, np, op) + args.report.write_text(json.dumps(report, indent=2) + '\n') + print('PASS', len(report['records']), len(report['contexts'])) + + +if __name__ == '__main__': + main() diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_out_buffers.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_out_buffers.py new file mode 100644 index 00000000..b3bccd0f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_out_buffers.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Verify reusable output buffers, mutation tracking, and execution contexts.""" +import argparse +import hashlib +import json +import math +from pathlib import Path +import time +import traceback + +from build_torch_extension import load_extension +from verify_execution_context import compare, fixture, reference, require_rejection + +METHODS = ("hadamard", "hadamard_int4", "quantize_int4") + + +def buffers_for(torch, x, method, guarded=False): + specs = [(tuple(x.shape), x.dtype)] if method == "hadamard" else [ + ((*x.shape[:-1], (x.shape[-1] + 1) // 2), torch.uint8), + (tuple(x.shape[:-1]), torch.float32), + ] + buffers, guards = [], [] + for shape, dtype in specs: + if guarded: + base = torch.full((math.prod(shape) + 16,), 37, dtype=dtype, device=x.device) + buffers.append(base[8:-8].reshape(shape)) + guards.append(base) + else: + buffers.append(torch.empty(shape, dtype=dtype, device=x.device)) + return tuple(buffers), guards + + +def invoke(op, x, buffers, method, **options): + result = getattr(op, method + "_out")(x, *buffers, **options) + if result is not None: + raise AssertionError("out interfaces must return None") + + +def positive_cases(torch, np, op): + records = [] + for dtype in (torch.float16, torch.bfloat16): + for dim in (1, 2, 4, 8, 16, 32, 64, 128, 256): + for shape in ((17, dim), (1, 1, 3, dim)): + for method in METHODS: + options = [{"block_threads": n} for n in (128, 256)] + if method != "quantize_int4": + options += [{"row_layout": "auto"}] + if dim <= 16: + options += [{"row_layout": "packed", "block_threads": n} for n in (128, 256)] + if method == "hadamard_int4" and dim == 256: + options += [{"fused_layout": "contiguous256"}] + for kwargs in options: + x = torch.empty(shape, dtype=dtype, device="cuda") + buffers, guards = buffers_for(torch, x, method, True) + pointers = tuple(b.data_ptr() for b in buffers) + call_options = dict(kwargs) + if method != "quantize_int4": + call_options["scale"] = 1 / math.sqrt(dim) + for phase in (1, 2): + cpu = fixture(torch, shape, dtype, phase) + x.copy_(cpu) + expected = reference(torch, np, cpu, {"shape": shape, "method": method}) + versions = tuple(b._version for b in buffers) + invoke(op, x, buffers, method, **call_options) + compare(torch, buffers, expected) + compare(torch, x, (cpu,)) + if tuple(b.data_ptr() for b in buffers) != pointers: + raise AssertionError("output storage was replaced") + if tuple(b._version for b in buffers) != tuple(v + 1 for v in versions): + raise AssertionError("successful mutation was not tracked") + for base in guards: + if not torch.all(base[:8] == 37) or not torch.all(base[-8:] == 37): + raise AssertionError("output guard elements were overwritten") + records.append({"dtype": str(dtype), "shape": shape, "method": method, + "options": call_options, "two_inputs_passed": True, + "storage_preserved": True, "guards_preserved": True}) + return records + + +def reject_without_mutation(torch, op, x, buffers, method, options, label): + protected = (x, *buffers) + before = tuple(value.detach().clone() for value in protected) + versions = tuple(None if value.is_inference() else value._version for value in protected) + try: + invoke(op, x, buffers, method, **options) + except (RuntimeError, ValueError) as error: + reason = str(error).splitlines()[0] + else: + raise AssertionError("invalid call accepted: " + label) + for value, original, version in zip(protected, before, versions): + compare(torch, value.detach().resolve_conj().resolve_neg(), (original.cpu(),)) + if version is not None and value._version != version: + raise AssertionError("rejected call changed a version counter: " + label) + return {"method": method, "case": label, "rejected_before_mutation": True, "error": reason} + + +def negative_cases(torch, op): + records = [] + for method in METHODS: + labels = ["cpu_input", "noncontiguous_input", "input_grad", "input_lazy_negative", + "cpu_output", "wrong_output_dtype", "wrong_output_rank", "wrong_output_shape", + "noncontiguous_output", "bad_threads", "input_output_overlap"] + if method != "quantize_int4": + labels += ["bad_scale", "bad_layout"] + if method == "hadamard": + labels += ["output_grad", "output_lazy_negative", "partial_overlap", "dlpack_overlap"] + else: + labels += ["wrong_scales_dtype", "wrong_scales_shape", "wrong_scales_rank", "cpu_scales", "noncontiguous_scales", "scales_grad", "scales_lazy_negative", + "input_scales_overlap", "output_output_overlap", "inference_scales"] + if method == "hadamard_int4": labels += ["bad_fused_layout", "invalid_contiguous256"] + for label in labels: + x = torch.arange(192, device="cuda", dtype=torch.float16).reshape(3, 64) / 16 + buffers, _ = buffers_for(torch, x, method) + buffers = list(buffers) + for b in buffers: + b.fill_(3) + options = {} + if label == "cpu_input": x = x.cpu() + elif label == "noncontiguous_input": x = x.T.contiguous().T + elif label == "input_grad": x.requires_grad_(True) + elif label == "input_lazy_negative": x = torch._neg_view(x) + elif label == "cpu_output": buffers[0] = buffers[0].cpu() + elif label == "wrong_output_dtype": buffers[0] = buffers[0].float() if method != "hadamard" else buffers[0].bfloat16() + elif label == "wrong_output_rank": buffers[0] = buffers[0].reshape(-1) + elif label == "wrong_output_shape": buffers[0] = torch.full((3, buffers[0].shape[-1]+1), 3, device="cuda", dtype=buffers[0].dtype) + elif label == "noncontiguous_output": buffers[0] = buffers[0].T.contiguous().T + elif label == "bad_threads": options["block_threads"] = 64 + elif label == "bad_scale": options["scale"] = float("nan") + elif label == "bad_layout": options["row_layout"] = "invalid" + elif label == "output_grad": buffers[0].requires_grad_(True) + elif label == "output_lazy_negative": buffers[0] = torch._neg_view(buffers[0]) + elif label == "input_output_overlap": buffers[0] = x if method == "hadamard" else x.view(torch.uint8).reshape(-1)[:96].reshape(3,32) + elif label == "partial_overlap": + base = torch.arange(193, device="cuda", dtype=torch.float16) + x, buffers[0] = base[:192].reshape(3,64), base[1:].reshape(3,64) + elif label == "dlpack_overlap": + buffers[0] = torch.from_dlpack(x) + if buffers[0].data_ptr() != x.data_ptr(): raise AssertionError("DLPack fixture did not alias") + elif label == "wrong_scales_dtype": buffers[1] = buffers[1].half() + elif label == "wrong_scales_shape": buffers[1] = torch.ones(4, device="cuda") + elif label == "wrong_scales_rank": buffers[1] = buffers[1].reshape(3,1) + elif label == "cpu_scales": buffers[1] = buffers[1].cpu() + elif label == "noncontiguous_scales": buffers[1] = torch.ones(6,device="cuda")[::2] + elif label == "scales_grad": buffers[1].requires_grad_(True) + elif label == "scales_lazy_negative": buffers[1] = torch._neg_view(buffers[1]) + elif label == "input_scales_overlap": buffers[1] = x.view(torch.float32).reshape(-1)[:3] + elif label == "output_output_overlap": + pool = torch.full((96,), 3, device="cuda", dtype=torch.uint8) + buffers[0], buffers[1] = pool.reshape(3,32), pool[:12].view(torch.float32) + elif label == "inference_scales": + with torch.inference_mode(): buffers[1] = torch.ones(3, device="cuda") + elif label == "bad_fused_layout": options["fused_layout"] = "invalid" + elif label == "invalid_contiguous256": options["fused_layout"] = "contiguous256" + records.append(reject_without_mutation(torch, op, x, buffers, method, options, label)) + return records + + +def mutation_contract(torch, np, op): + records = [] + for method in METHODS: + x = torch.ones((3,64), device="cuda", dtype=torch.float16) + buffers, _ = buffers_for(torch, x, method) + watched = buffers[0] if method == "hadamard" else buffers[1] + watched.fill_(1) + grad_input = torch.ones_like(watched, requires_grad=True) + loss = (grad_input * watched).sum() + invoke(op, x, buffers, method) + try: loss.backward() + except RuntimeError as error: + if "modified by an inplace operation" not in str(error): raise + else: raise AssertionError("autograd failed to detect output mutation") + with torch.inference_mode(): + inference_buffers, _ = buffers_for(torch, x, method) + invoke(op, x, inference_buffers, method) + transformed = torch.zeros((3,64),dtype=torch.float16); transformed[:,0] = 64 + expected = (transformed,) if method == "hadamard" else reference( + torch,np,transformed if method == "hadamard_int4" else x.cpu(), + {"shape":x.shape,"method":"quantize_int4"}) + compare(torch,inference_buffers,expected) + # Allocation check starts after warmup, excluding caller-owned buffers. + for _ in range(10): invoke(op, x, buffers, method) + torch.cuda.synchronize(); live = torch.cuda.memory_allocated(); torch.cuda.reset_peak_memory_stats() + for _ in range(50): invoke(op, x, buffers, method) + torch.cuda.synchronize() + if torch.cuda.max_memory_allocated() != live: raise AssertionError("out path allocated tensor memory") + records.append({"method":method,"autograd_detected_mutation":True,"inference_mode_passed":True,"output_allocation_bytes":0}) + # Disjoint slices of one allocation are legal, even with a shared version counter. + pool = torch.zeros(800, device="cuda", dtype=torch.uint8) + x = pool[16:400].view(torch.float16).reshape(3,64) + out = pool[400:784].view(torch.float16).reshape(3,64) + x.fill_(1); expected = torch.zeros_like(x).cpu(); expected[:,0] = 64 + invoke(op, x, (out,), "hadamard"); compare(torch, out, (expected,)) + if not torch.all(x == 1): raise AssertionError("disjoint input slice changed") + records.append({"case":"disjoint_shared_storage","passed":True}) + return records + + +def execution_contexts(torch, np, op): + records = [] + for dtype in (torch.float16, torch.bfloat16): + for shape in ((17,8),(17,256),(1,1,3,64),(4097,8)): + for method in METHODS: + cpu = [fixture(torch,shape,dtype,phase) for phase in range(4)] + expected = [reference(torch,np,v,{"shape":shape,"method":method}) for v in cpu] + x = torch.zeros_like(cpu[0], device="cuda"); payloads = [v.cuda() for v in cpu] + buffers, _ = buffers_for(torch,x,method) + options = {} if method == "quantize_int4" else {"scale":1/math.sqrt(shape[-1]),"row_layout":"auto"} + side = torch.cuda.Stream() + for wrong in (False, True): + x.zero_(); side.wait_stream(torch.cuda.current_stream()) + for stream in (torch.cuda.default_stream(),side): + with torch.cuda.stream(stream): invoke(op,x,buffers,method,**options) + torch.cuda.synchronize() + with torch.cuda.stream(side): + torch.cuda._sleep(50_000_000); x.copy_(payloads[1]) + if wrong: + with torch.cuda.stream(torch.cuda.default_stream()): invoke(op,x,buffers,method,**options) + else: invoke(op,x,buffers,method,**options) + torch.cuda.synchronize() + if wrong: require_rejection(torch,buffers,expected[1],"consumer_on_default_stream") + else: compare(torch,buffers,expected[1]) + side.wait_stream(torch.cuda.current_stream()) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph,stream=side): invoke(op,x,buffers,method,**options) + stale = None + for phase in range(4): + with torch.cuda.stream(side): x.copy_(payloads[phase]); graph.replay() + side.synchronize(); compare(torch,buffers,expected[phase]); compare(torch,x,(cpu[phase],)) + if phase == 0: stale = tuple(v.cpu().clone() for v in buffers) + elif phase == 1: require_rejection(torch,stale,expected[phase],"stale_graph_output") + records.append({"dtype":str(dtype),"shape":shape,"method":method,"stream_passed":True,"graph_replays":4,"negative_controls_detected":2}) + return records + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build-directory',type=Path,required=True) + parser.add_argument('--json',type=Path,required=True) + args=parser.parse_args();args.json.parent.mkdir(parents=True,exist_ok=True) + with args.json.open('x') as stream: + report={'status':'RUNNING','created_utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())} + try: + import numpy as np + import torch + op=load_extension(verbose=True,build_directory=str(args.build_directory)) + report['environment']={'torch':torch.__version__,'cuda':torch.version.cuda,'gpu':torch.cuda.get_device_name(), + 'extension_sha256':hashlib.sha256(Path(op.__file__).read_bytes()).hexdigest()} + report['positive']=positive_cases(torch,np,op) + report['negative']=negative_cases(torch,op) + report['mutation']=mutation_contract(torch,np,op) + report['contexts']=execution_contexts(torch,np,op) + report['status']='PASS' + except Exception: report.update(status='FAIL',traceback=traceback.format_exc()) + root=Path(__file__).resolve().parents[1] + report['source_sha256']={name:hashlib.sha256((root/name).read_bytes()).hexdigest() for name in ['src/torch_binding.cu','scripts/verify_out_buffers.py']} + stream.write(json.dumps(report,indent=2)+'\n') + print(json.dumps({'status':report['status'],'positive':len(report.get('positive',[])),'negative':len(report.get('negative',[])),'contexts':len(report.get('contexts',[]))})) + if report['status']!='PASS': print(report['traceback']); return 1 + return 0 + + +if __name__=='__main__': raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_paired_quantization.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_paired_quantization.py new file mode 100644 index 00000000..f0249a5f --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_paired_quantization.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Verify packed pairs with all finite component encodings and offset storage.""" +import argparse +import json +from pathlib import Path + +from build_torch_extension import load_extension +from verify_execution_context import compare, reference + + +def guarded_outputs(torch, x): + shapes = [(*x.shape[:-1], x.shape[-1]//2), tuple(x.shape[:-1])] + buffers, guards = [], [] + for shape, dtype in zip(shapes, (torch.uint8, torch.float32)): + size = 1 + for extent in shape: size *= extent + base = torch.full((size+17,),37,dtype=dtype,device=x.device) + buffers.append(base[9:-8].reshape(shape));guards.append(base) + return tuple(buffers), guards + + +def verify(torch, np, candidate, control=None): + records=[] + raw=np.arange(65536,dtype=np.uint16).view(np.int16).copy() + for dtype in (torch.float16,torch.bfloat16): + values=torch.from_numpy(raw).view(dtype) + values=values[torch.isfinite(values.float())] + assert values.numel()==(63488 if dtype==torch.float16 else 65280) + anchors=torch.tensor([0,.5,1.5,7,-7,torch.finfo(dtype).tiny,torch.finfo(dtype).max],dtype=dtype) + for dim in (2,4,8,16): + for position in (0,dim-1): + cpu=anchors[torch.arange(values.numel())%anchors.numel()][:,None].expand(-1,dim).clone() + cpu[:,position]=values + expected=reference(torch,np,cpu,{'shape':cpu.shape,'method':'quantize_int4'}) + for offset in (0,1): + pool=torch.full((cpu.numel()+offset+1,),37,dtype=dtype,device='cuda') + x=pool[offset:offset+cpu.numel()].reshape(cpu.shape);x.copy_(cpu) + assert x.data_ptr()%4==offset*2 + for threads in (128,256): + compare(torch,candidate.quantize_int4_packed(x,threads),expected) + compare(torch,candidate.quantize_int4(x,threads),expected) + if control is not None:compare(torch,control.quantize_int4_packed(x,threads),expected) + buffers,guards=guarded_outputs(torch,x);pointers=[b.data_ptr() for b in buffers];versions=[b._version for b in buffers] + assert candidate.quantize_int4_packed_out(x,*buffers,threads) is None + compare(torch,buffers,expected);compare(torch,x,(cpu,)) + assert [b.data_ptr() for b in buffers]==pointers + assert [b._version for b in buffers]==[v+1 for v in versions] + for guard in guards: + assert torch.all(guard[:9]==37) and torch.all(guard[-8:]==37) + if offset:assert pool[0].item()==37 + assert pool[-1].item()==37 + records.append({'dtype':str(dtype),'dim':dim,'variable_position':position,'input_offset_elements':offset,'threads':threads, + 'finite_component_encodings':values.numel(),'cpu_oracle':'PASS','allocating_out_and_original':'PASS','guards_versions_and_input':'PASS'}) + return {'status':'PASS','unique_finite_component_encodings':128768,'records':records, + 'scope':'Each finite encoding is inserted into selected low/high components; other values use seven fixed anchor classes. This is not every vector combination.'} + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build-directory',type=Path,required=True) + parser.add_argument('--report',type=Path,required=True) + args=parser.parse_args() + import torch + import numpy as np + op=load_extension(build_directory=args.build_directory) + report=verify(torch,np,op) + args.report.write_text(json.dumps(report,indent=2)+'\n') + print('PASS:',len(report['records']),'configurations') + + +if __name__=='__main__':main() diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_quantize_packed.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_quantize_packed.py new file mode 100644 index 00000000..92b63eff --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_quantize_packed.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Independent byte-level checks for explicit packed quantization.""" +import argparse +import json +import math +from pathlib import Path + +from build_torch_extension import load_extension +from verify_execution_context import compare, fixture, reference, require_rejection +from verify_out_buffers import buffers_for + + +def pattern(torch, shape, dtype, name): + index = torch.arange(math.prod(shape), dtype=torch.int64).reshape(shape) + if name == 'zero': + return torch.zeros(shape, dtype=dtype) + if name == 'finite': + return fixture(torch, shape, dtype, 1) + if name == 'ties': + table = torch.tensor([-7, -6.5, -2.5, -.5, .5, 1.5, 6.5, 7], dtype=dtype) + elif name == 'extreme': + maximum = torch.finfo(dtype).max + table = torch.tensor([maximum, -maximum, maximum/2, -maximum/2, 0, 1, -1, 0], dtype=dtype) + else: + smallest = torch.finfo(dtype).tiny / (1024 if dtype == torch.float16 else 128) + table = torch.tensor([0, smallest, -smallest, smallest*3, -smallest*3, smallest*7, -smallest*7, 0], dtype=dtype) + values = table[index % table.numel()] + if name == 'ties' and shape[-1] > 1: + values[..., -1] = 7 # Exact scale=1 exposes round-to-nearest-even ties. + return values + + +def call(op, x, buffers, threads): + if buffers is None: + return op.quantize_int4_packed(x, threads) + assert op.quantize_int4_packed_out(x, *buffers, threads) is None + return buffers + + +def verify(torch, np, op, control=None): + records = [] + for dtype in (torch.float16, torch.bfloat16): + for dim in (1, 2, 4, 8, 16): + for shape in [*( (n, dim) for n in (1, 3, 17, 255, 257, 4097)), (1, 3, 7, dim)]: + for name in ('zero', 'finite', 'ties', 'extreme', 'subnormal'): + cpu = pattern(torch, shape, dtype, name) + expected = reference(torch, np, cpu, {'method':'quantize_int4','shape':shape}) + x = cpu.cuda() + for threads in (128, 256): + if control is not None: + compare(torch, control.quantize_int4(x,threads), expected) + compare(torch, op.quantize_int4(x,threads), expected) + compare(torch, call(op,x,None,threads), expected) + buffers,guards = buffers_for(torch,x,'quantize_int4',True) + versions = [b._version for b in buffers] + pointers = [b.data_ptr() for b in buffers] + compare(torch,call(op,x,buffers,threads),expected) + assert [b._version for b in buffers] == [v+1 for v in versions] + assert [b.data_ptr() for b in buffers] == pointers + for guard in guards: + assert torch.all(guard[:8] == 37) and torch.all(guard[-8:] == 37) + compare(torch,x,(cpu,)) + records.append({'dtype':str(dtype),'shape':shape,'pattern':name,'threads':threads,'allocating_and_out_passed':True,'guards_and_versions_passed':True}) + negatives = [] + for label,dim,options in [('large32',32,{}),('large256',256,{}),('bad_threads',8,{'block_threads':64}),('auto_unsupported',8,{'row_layout':'auto'})]: + x = torch.ones((17,dim),device='cuda',dtype=torch.float16) + buffers,_ = buffers_for(torch,x,'quantize_int4') + for b in buffers:b.fill_(3) + protected=(x,*buffers);before=[v.cpu().clone() for v in protected];versions=[v._version for v in protected] + for method in ('quantize_int4_packed','quantize_int4_packed_out'): + args=(x,*buffers) if method.endswith('_out') else (x,) + try:getattr(op,method)(*args,**options) + except (ValueError,RuntimeError,TypeError):pass + else:raise AssertionError('invalid call accepted: '+label) + for value,original,version in zip(protected,before,versions): + compare(torch,value,(original,));assert value._version==version + negatives.append({'case':label,'allocating_and_out_rejected_before_mutation':True}) + return {'status':'PASS','positive':records,'negative':negatives} + + +def contexts(torch,np,op): + records=[] + for dtype in (torch.float16,torch.bfloat16): + for shape in ((17,1),(4097,16)): + for threads in (128,256): + for out_api in (False,True): + cpu=[fixture(torch,shape,dtype,phase) for phase in range(4)] + expected=[reference(torch,np,c,{'method':'quantize_int4','shape':shape}) for c in cpu] + x=torch.zeros_like(cpu[0],device='cuda');payloads=[c.cuda() for c in cpu] + buffers=buffers_for(torch,x,'quantize_int4')[0] if out_api else None + side=torch.cuda.Stream() + for wrong in (False,True): + x.zero_();side.wait_stream(torch.cuda.current_stream()) + for stream in (torch.cuda.default_stream(),side): + with torch.cuda.stream(stream):warm=call(op,x,buffers,threads) + torch.cuda.synchronize() + with torch.cuda.stream(side): + torch.cuda._sleep(50_000_000);x.copy_(payloads[1]) + if wrong: + with torch.cuda.stream(torch.cuda.default_stream()):actual=call(op,x,buffers,threads) + else:actual=call(op,x,buffers,threads) + torch.cuda.synchronize() + if wrong:require_rejection(torch,actual,expected[1],'wrong_stream') + else:compare(torch,actual,expected[1]) + side.wait_stream(torch.cuda.current_stream());graph=torch.cuda.CUDAGraph() + with torch.cuda.graph(graph,stream=side):actual=call(op,x,buffers,threads) + stale=None + for phase in range(4): + with torch.cuda.stream(side):x.copy_(payloads[phase]);graph.replay() + side.synchronize();compare(torch,actual,expected[phase]);compare(torch,x,(cpu[phase],)) + if phase==0:stale=tuple(v.cpu().clone() for v in actual) + if phase==1:require_rejection(torch,stale,expected[phase],'stale_output') + records.append({'dtype':str(dtype),'shape':shape,'threads':threads,'out_api':out_api,'replays':4,'negative_controls':2}) + return records + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build-directory',type=Path,required=True) + parser.add_argument('--report',type=Path,required=True) + args=parser.parse_args() + import numpy as np + import torch + op=load_extension(build_directory=args.build_directory) + report=verify(torch,np,op);report['contexts']=contexts(torch,np,op) + args.report.write_text(json.dumps(report,indent=2)+'\n') + print('PASS',len(report['positive']),len(report['contexts'])) + + +if __name__=='__main__':main() diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_singleton_quantization.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_singleton_quantization.py new file mode 100644 index 00000000..660b6130 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_singleton_quantization.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Exhaust singleton quantization storage encodings without changing its domain.""" +import argparse +import json +from pathlib import Path + +from build_torch_extension import load_extension +from verify_execution_context import compare, reference +from verify_out_buffers import buffers_for + + +def verify(torch, np, candidate, control=None): + records = [] + words = np.arange(65536, dtype=np.uint16) + # Fixed permutation places different signs/exponents in adjacent warp lanes. + np.random.default_rng(9172030).shuffle(words) + for dtype in (torch.float16, torch.bfloat16): + expected_finite = 63488 if dtype == torch.float16 else 65280 + for shape in ((65536, 1), (1, 1, 65539, 1)): + raw = np.resize(words, int(np.prod(shape))).view(np.int16).copy() + cpu = torch.from_numpy(raw).view(dtype).reshape(shape) + mask = np.isfinite(cpu.float().numpy().reshape(-1)) + if len(mask) == 65536: + assert int(mask.sum()) == expected_finite + finite_input = cpu.reshape(-1, 1)[torch.from_numpy(mask)] + oracle = reference(torch, np, finite_input, {'shape':finite_input.shape, 'method':'quantize_int4'}) + x = cpu.cuda() + for threads in (128, 256): + # Compare nonfinite encodings only with the same legacy packed + # path; other layouts may choose a different unsupported fallback. + baseline_op = control if control is not None else candidate + baseline = tuple(v.cpu() for v in baseline_op.quantize_int4_packed(x, threads)) + actual = candidate.quantize_int4_packed(x, threads) + compare(torch, actual, baseline) + actual_finite = tuple(v.cpu().reshape(-1)[torch.from_numpy(mask)] for v in actual) + expected_finite_outputs = tuple(v.reshape(-1) for v in oracle) + compare(torch, actual_finite, expected_finite_outputs) + buffers, guards = buffers_for(torch, x, 'quantize_int4', True) + versions = [b._version for b in buffers] + pointers = [b.data_ptr() for b in buffers] + assert candidate.quantize_int4_packed_out(x, *buffers, threads) is None + compare(torch, buffers, baseline) + assert [b._version for b in buffers] == [v+1 for v in versions] + assert [b.data_ptr() for b in buffers] == pointers + for guard in guards: + assert torch.all(guard[:8] == 37) and torch.all(guard[-8:] == 37) + compare(torch, x, (cpu,)) + assert torch.all((buffers[0] & 0xF0) == 0) + records.append({'dtype':str(dtype),'shape':shape,'threads':threads, + 'storage_encodings':65536,'finite_rows_checked':int(mask.sum()), + 'independent_finite_oracle':'PASS', + 'legacy_fallback_bits':'PASS' if control is not None else 'NOT_CHECKED: control module required', + 'out_guards_versions_and_storage':'PASS','input_unchanged':True}) + return {'status':'PASS','unique_storage_encodings':131072,'unique_finite_values':128768,'records':records} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build-directory',type=Path,required=True) + parser.add_argument('--report',type=Path,required=True) + args = parser.parse_args() + import torch + import numpy as np + module = load_extension(build_directory=args.build_directory) + report = verify(torch,np,module) + args.report.write_text(json.dumps(report,indent=2)+'\n') + print('PASS:',report['unique_finite_values'],'finite values,',report['unique_storage_encodings'],'storage encodings') + + +if __name__=='__main__':main() diff --git a/03_hadamard_tc/a962695448-rgb/scripts/verify_tensor_metadata.py b/03_hadamard_tc/a962695448-rgb/scripts/verify_tensor_metadata.py new file mode 100644 index 00000000..a176dbec --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/scripts/verify_tensor_metadata.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""验证惰性负号视图明确拒绝，以及显式物化后的三种接口。""" +import argparse +import hashlib +import json +from pathlib import Path +import time +import traceback + +from build_torch_extension import load_extension + + +def check_metadata(torch, op): + results = [] + for dtype in (torch.float16, torch.bfloat16): + for shape in ((3, 256), (1, 1, 3, 256)): + base = torch.arange(768, device="cuda", dtype=torch.float32).remainder(29).sub(14).div(16).to(dtype).reshape(shape) + before = base.clone() + lazy = torch._neg_view(base) + if not lazy.is_neg() or not lazy.is_contiguous() or lazy.data_ptr() != base.data_ptr(): + raise RuntimeError("fixture does not exercise a contiguous lazy negative alias") + materialized, expected_input = lazy.resolve_neg(), -base + if materialized.is_neg() or not torch.equal(materialized.view(torch.uint8), expected_input.view(torch.uint8)): + raise RuntimeError("resolve_neg fixture did not materialize the logical values") + for threads in (128, 256): + for method in ("hadamard", "hadamard_int4", "quantize_int4"): + operation = getattr(op, method) + try: + operation(lazy, block_threads=threads) + except RuntimeError as error: + if "resolve_neg" not in str(error): + raise + reason = str(error).splitlines()[0] + else: + raise RuntimeError(f"accepted lazy negative alias: {method}/{dtype}/{shape}/{threads}") + actual = operation(materialized, block_threads=threads) + expected = operation(expected_input, block_threads=threads) + actual = actual if isinstance(actual, tuple) else (actual,) + expected = expected if isinstance(expected, tuple) else (expected,) + if len(actual) != len(expected) or any(a.shape != b.shape or a.dtype != b.dtype or + not torch.equal(a.view(torch.uint8), b.view(torch.uint8)) for a, b in zip(actual, expected)): + raise RuntimeError("materialized logical input output mismatch: " + method) + results.append({"dtype": str(dtype), "shape": list(shape), "block_threads": threads, + "method": method, "rejected_lazy_alias": True, + "materialized_output_bitwise_equal": True, "error": reason}) + try: + op.hadamard_int4(lazy, fused_layout="contiguous256") + except RuntimeError as error: + if "resolve_neg" not in str(error): + raise + reason = str(error).splitlines()[0] + else: + raise RuntimeError("contiguous256 fused accepted a lazy negative alias") + actual = op.hadamard_int4(materialized, fused_layout="contiguous256") + expected = op.hadamard_int4(expected_input) + if len(actual) != len(expected) or any(a.shape != b.shape or a.dtype != b.dtype or + not torch.equal(a.view(torch.uint8), b.view(torch.uint8)) for a, b in zip(actual, expected)): + raise RuntimeError("materialized contiguous256 fused differs from original layout") + results.append({"dtype": str(dtype), "shape": list(shape), "block_threads": 128, + "method": "hadamard_int4", "fused_layout": "contiguous256", + "rejected_lazy_alias": True, "materialized_output_bitwise_equal": True, + "error": reason}) + if not torch.equal(base.view(torch.uint8), before.view(torch.uint8)): + raise RuntimeError("underlying input was modified") + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-directory", type=Path, required=True) + parser.add_argument("--json", type=Path, required=True) + args = parser.parse_args() + if args.json.exists(): + parser.error("result already exists; choose a new output path") + args.json.parent.mkdir(parents=True, exist_ok=True) + report = {"status": "RUNNING", "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + try: + import torch + op = load_extension(verbose=True, build_directory=str(args.build_directory)) + report["environment"] = {"torch": torch.__version__, "torch_cuda": torch.version.cuda, + "gpu": torch.cuda.get_device_name(), + "extension_sha256": hashlib.sha256(Path(op.__file__).read_bytes()).hexdigest()} + report["cases"] = check_metadata(torch, op) + report["status"] = "PASS" + except Exception: + report.update(status="FAIL", traceback=traceback.format_exc()) + root = Path(__file__).resolve().parents[1] + report["source_sha256"] = {name: hashlib.sha256((root / name).read_bytes()).hexdigest() + for name in ("src/torch_binding.cu", "include/kernels.cuh", "include/contiguous256.cuh", + "scripts/verify_tensor_metadata.py")} + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + args.json.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "metadata_cases": len(report.get("cases", []))})) + if report["status"] != "PASS": + print(report["traceback"]) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/03_hadamard_tc/a962695448-rgb/src/main.cu b/03_hadamard_tc/a962695448-rgb/src/main.cu new file mode 100644 index 00000000..6433184a --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/src/main.cu @@ -0,0 +1,649 @@ +#include "kernels.cuh" +#include "contiguous256.cuh" +#include "packed_rows.cuh" +#include "row_policy.hpp" +#include "reference.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void check_cuda(cudaError_t result, const char* expression, int line) { + if (result != cudaSuccess) + throw std::runtime_error(std::string(expression) + " at line " + + std::to_string(line) + ": " + cudaGetErrorString(result)); +} +#define CUDA_CHECK(expression) check_cuda((expression), #expression, __LINE__) + +void cleanup_cuda(cudaError_t result, const char* name) noexcept { + if (result != cudaSuccess) + std::cerr << "CUDA cleanup error (" << name << "): " << cudaGetErrorString(result) << '\n'; +} + +std::size_t product(std::size_t a, std::size_t b) { + if (b != 0 && a > std::numeric_limits::max() / b) + throw std::invalid_argument("shape or allocation size overflows size_t"); + return a * b; +} + +std::size_t divide_up(std::size_t size, std::size_t divisor) { + return (size - 1) / divisor + 1; +} + +template class DeviceBuffer { +public: + explicit DeviceBuffer(std::size_t count) : count_(count) { + if (count_) CUDA_CHECK(cudaMalloc(reinterpret_cast(&data_), product(count_, sizeof(T)))); + } + ~DeviceBuffer() { if (data_) cleanup_cuda(cudaFree(data_), "cudaFree"); } + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; + T* data() { return data_; } + const T* data() const { return data_; } + std::size_t bytes() const { return product(count_, sizeof(T)); } + void upload(const std::vector& values) { + if (values.size() != count_) throw std::logic_error("upload size mismatch"); + CUDA_CHECK(cudaMemcpy(data_, values.data(), bytes(), cudaMemcpyHostToDevice)); + } + std::vector download() const { + std::vector values(count_); + CUDA_CHECK(cudaMemcpy(values.data(), data_, bytes(), cudaMemcpyDeviceToHost)); + return values; + } +private: + T* data_ = nullptr; + std::size_t count_; +}; + +class Event { +public: + Event() { CUDA_CHECK(cudaEventCreate(&event_)); } + ~Event() { cleanup_cuda(cudaEventDestroy(event_), "cudaEventDestroy"); } + Event(const Event&) = delete; + Event& operator=(const Event&) = delete; + operator cudaEvent_t() const { return event_; } +private: + cudaEvent_t event_{}; +}; + +enum class Method { Naive, Warp, TensorCore, SplitInt4, FusedInt4 }; +const char* method_name(Method method) { + switch (method) { + case Method::Naive: return "naive_global"; + case Method::Warp: return "warp"; + case Method::TensorCore: return "tensor_core"; + case Method::SplitInt4: return "split_int4"; + case Method::FusedInt4: return "fused_int4"; + } + throw std::logic_error("unknown method"); +} + +struct Options { + bool self_test = false, benchmark = false, normalized = false; + std::size_t batch = 4, seq = 128, heads = 8, dim = 256; + std::string dtype = "fp16", csv, fused_layout = "original", row_layout = "original"; + int repetitions = 200, warmup = 20, block_threads = 128; + std::size_t rows() const { return product(product(batch, seq), heads); } + std::size_t elements() const { return product(rows(), dim); } + float scale() const { return normalized ? 1.0f / std::sqrt(static_cast(dim)) : 1.0f; } +}; + +void help() { + std::cout << "Hadamard CUDA benchmark; shape [batch, seq, heads, dim], last-axis transform.\n" + << "Usage: hadamard [--self-test] [--benchmark] [options]\n" + << " --batch B --seq S --heads H Positive dimensions (defaults 4,128,8)\n" + << " --dim N Power of two in [1,256], default 256\n" + << " --dtype fp16|bf16 Default fp16\n" + << " --scale 1|normalized Default 1; normalized means 1/sqrt(N)\n" + << " --normalize Alias for --scale normalized\n" + << " --repetitions R --warmup W Default 200,20; R positive, W nonnegative\n" + << " --block-threads 128|256 Warp transform/quantize/split/fused; default 128\n" + << " Naive, Tensor Core and CPU paths are unchanged\n" + << " --fused-layout original|contiguous256 Fused INT4 only; default original\n" + << " contiguous256 requires N256 and block_threads=128\n" + << " Self-test: only N256 fused uses it; other N remain original\n" + << " --row-layout original|packed|auto N<=16 row mapping; default original\n" + << " auto may choose 256 threads on validated models\n" + << " self-test: other N use original; incompatible with contiguous256\n" + << " --csv FILE Append measured rows, with header if empty\n" + << "No mode selects --self-test. Self-tests always cover both dtypes/scales.\n" + << "INT4: rowwise symmetric [-7,7], scale=max(abs(y))/7 (zero row:1),\n" + << "ties-to-even; even element low nibble; y is rounded to output dtype first.\n"; +} + +std::size_t unsigned_number(const std::string& text, const std::string& argument, bool zero_ok = false) { + std::size_t value = 0; + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != text.data() + text.size() || (!zero_ok && !value)) + throw std::invalid_argument(argument + " requires a valid " + (zero_ok ? "nonnegative" : "positive") + " integer"); + return value; +} + +Options parse(int argc, char** argv) { + Options options; + for (int i = 1; i < argc; ++i) { + const std::string argument = argv[i]; + auto next = [&]() -> std::string { + if (++i == argc) throw std::invalid_argument("missing value for " + argument); + return argv[i]; + }; + if (argument == "--self-test") options.self_test = true; + else if (argument == "--benchmark") options.benchmark = true; + else if (argument == "--normalize") options.normalized = true; + else if (argument == "--batch") options.batch = unsigned_number(next(), argument); + else if (argument == "--seq") options.seq = unsigned_number(next(), argument); + else if (argument == "--heads") options.heads = unsigned_number(next(), argument); + else if (argument == "--dim") options.dim = unsigned_number(next(), argument); + else if (argument == "--dtype") options.dtype = next(); + else if (argument == "--csv") options.csv = next(); + else if (argument == "--fused-layout") options.fused_layout = next(); + else if (argument == "--row-layout") options.row_layout = next(); + else if (argument == "--block-threads") { + const auto value = unsigned_number(next(), argument); + if (value != 128 && value != 256) + throw std::invalid_argument("--block-threads must be 128 or 256"); + options.block_threads = static_cast(value); + } + else if (argument == "--scale") { + const auto value = next(); + if (value == "1" || value == "1.0") options.normalized = false; + else if (value == "normalized") options.normalized = true; + else throw std::invalid_argument("--scale must be 1 or normalized"); + } else if (argument == "--repetitions" || argument == "--warmup") { + const auto value = unsigned_number(next(), argument, argument == "--warmup"); + if (value > 100000) throw std::invalid_argument(argument + " must not exceed 100000"); + if (argument == "--warmup") options.warmup = static_cast(value); + else options.repetitions = static_cast(value); + } else throw std::invalid_argument("unknown option: " + argument); + } + if (!hadamard::power_of_two(options.dim) || options.dim > 256) + throw std::invalid_argument("--dim must be a power of two in [1,256]"); + if (options.dtype != "fp16" && options.dtype != "bf16") + throw std::invalid_argument("--dtype must be fp16 or bf16"); + if (options.fused_layout != "original" && options.fused_layout != "contiguous256") + throw std::invalid_argument("--fused-layout must be original or contiguous256"); + const auto row_layout = hadamard::parse_row_layout(options.row_layout); + if (row_layout != hadamard::RowLayout::Original && options.fused_layout != "original") + throw std::invalid_argument("--row-layout cannot be combined with contiguous256"); + if (row_layout == hadamard::RowLayout::Packed && options.benchmark && options.dim > 16) + throw std::invalid_argument("--benchmark with --row-layout packed requires --dim <= 16"); + if (options.fused_layout == "contiguous256") { + if (options.block_threads != 128) + throw std::invalid_argument("--fused-layout contiguous256 requires --block-threads 128"); + if (options.benchmark && options.dim != 256) + throw std::invalid_argument("--benchmark with --fused-layout contiguous256 requires --dim 256"); + } + // Validate all products before allocating or touching a CUDA device. + product(options.elements(), sizeof(float)); + if (!options.self_test && !options.benchmark) options.self_test = true; + return options; +} + +template class Runner { +public: + Runner(std::size_t row_count, int dimension, float transform_scale, int warp_block_threads = 128, + bool use_contiguous256_fused = false, + hadamard::RowLayout row_layout = hadamard::RowLayout::Original) + : rows(row_count), dim(dimension), scale(transform_scale), block_threads(warp_block_threads), + contiguous256_fused(use_contiguous256_fused), requested_row_layout(row_layout), count(product(rows, dim)), + input(count), output(count), scratch_a(count), scratch_b(count), + matrix(dim >= 16 ? dim * dim : 0), + packed(product(rows, (dim + 1) / 2)), quant_scales(rows) { + cudaDeviceProp properties{}; + CUDA_CHECK(cudaGetDeviceProperties(&properties, 0)); + row_device_family = hadamard::row_device(properties.name); + if (divide_up(count, 256) > static_cast(properties.maxGridSize[0]) || + divide_up(rows, block_threads / 32) > static_cast(properties.maxGridSize[0])) + throw std::invalid_argument("shape exceeds CUDA launch grid limits"); + if (dim >= 16) { + std::vector h(dim * dim); + for (int r = 0; r < dim; ++r) + for (int c = 0; c < dim; ++c) { + int bits = r & c, sign = 1; + while (bits) { sign = -sign; bits &= bits - 1; } + h[r * dim + c] = hadamard::as_storage(static_cast(sign)); + } + matrix.upload(h); + } + } + + void run(Method method) { + if (method == Method::Naive) { + const auto blocks = static_cast(divide_up(count, 256)); + hadamard::to_float_kernel<<>>(input.data(), scratch_a.data(), count); + CUDA_CHECK(cudaGetLastError()); + float* current = scratch_a.data(); + float* next = scratch_b.data(); + for (int stride = 1; stride < dim; stride *= 2) { + hadamard::butterfly_stage<<>>(current, next, count, stride); + CUDA_CHECK(cudaGetLastError()); + std::swap(current, next); + } + hadamard::from_float_kernel<<>>(current, output.data(), count, scale); + CUDA_CHECK(cudaGetLastError()); + return; + } + switch (dim) { +#define DIM_CASE(n) case n: specialized(method); break + DIM_CASE(1); DIM_CASE(2); DIM_CASE(4); DIM_CASE(8); DIM_CASE(16); + DIM_CASE(32); DIM_CASE(64); DIM_CASE(128); DIM_CASE(256); +#undef DIM_CASE + default: throw std::logic_error("unsupported dimension"); + } + } + + std::size_t rows; + int dim; + float scale; + int block_threads; + bool contiguous256_fused; + hadamard::RowLayout requested_row_layout; + hadamard::RowDevice row_device_family = hadamard::RowDevice::Unknown; + std::size_t count; + DeviceBuffer input, output; + DeviceBuffer scratch_a, scratch_b; + DeviceBuffer matrix; + DeviceBuffer packed; + DeviceBuffer quant_scales; + +private: + template void specialized(Method method) { + const auto blocks = static_cast(divide_up(rows, block_threads / 32)); + if constexpr (N <= 16) { + const auto choice = hadamard::choose_rows(requested_row_layout, row_device_family, rows, N, + method == Method::FusedInt4, block_threads); + if (choice.packed && method != Method::TensorCore) { + const auto packed_blocks = static_cast(divide_up(rows, choice.threads / N)); + if (method == Method::FusedInt4) { + hadamard::packed_rows_kernel<<>>( + input.data(), nullptr, packed.data(), quant_scales.data(), rows, scale); + CUDA_CHECK(cudaGetLastError()); + } else { + hadamard::packed_rows_kernel<<>>( + input.data(), output.data(), nullptr, nullptr, rows, scale); + CUDA_CHECK(cudaGetLastError()); + if (method == Method::SplitInt4) { + hadamard::warp_kernel<<>>( + output.data(), nullptr, packed.data(), quant_scales.data(), rows, 1); + CUDA_CHECK(cudaGetLastError()); + } + } + return; + } + } + if (method == Method::TensorCore) { + if constexpr (N >= 16) { + const dim3 grid(static_cast(divide_up(rows, 16)), N / 16); + hadamard::tensor_core_kernel<<>>(input.data(), matrix.data(), output.data(), rows, scale); + CUDA_CHECK(cudaGetLastError()); + } else throw std::invalid_argument("tensor_core requires dim >= 16"); + } else if (method == Method::FusedInt4) { + if constexpr (N == 256) { + if (contiguous256_fused) { + hadamard::contiguous256_kernel<<>>( + input.data(), nullptr, packed.data(), quant_scales.data(), rows, scale); + CUDA_CHECK(cudaGetLastError()); + return; + } + } + hadamard::warp_kernel<<>>(input.data(), nullptr, packed.data(), quant_scales.data(), rows, scale); + CUDA_CHECK(cudaGetLastError()); + } else { + hadamard::warp_kernel<<>>(input.data(), output.data(), nullptr, nullptr, rows, scale); + CUDA_CHECK(cudaGetLastError()); + if (method == Method::SplitInt4) { + hadamard::warp_kernel<<>>(output.data(), nullptr, packed.data(), quant_scales.data(), rows, 1); + CUDA_CHECK(cudaGetLastError()); + } + } + } +}; + +template std::vector to_float(const std::vector& input) { + std::vector result(input.size()); + for (std::size_t i = 0; i < input.size(); ++i) result[i] = hadamard::as_float(input[i]); + return result; +} + +template std::vector make_input(std::size_t rows, int dim, const std::string& pattern, std::uint32_t seed) { + std::mt19937 generator(seed); + std::uniform_int_distribution distribution(-128, 128); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::normal_distribution normal(0.0f, 0.5f); + std::vector result(product(rows, dim)); + for (std::size_t row = 0; row < rows; ++row) + for (int j = 0; j < dim; ++j) { + float value = 0; + if (pattern == "random") value = static_cast(distribution(generator)) / 128.0f; + else if (pattern == "uniform") value = uniform(generator); + else if (pattern == "normal") value = normal(generator); + else if (pattern == "outlier") + value = j == static_cast((row * 7) % dim) ? 8.0f : uniform(generator) * 0.001f; + else if (pattern == "impulse") value = j == static_cast((row * 7) % dim) ? 0.75f : 0.0f; + else if (pattern == "alternating") value = (j % 2 ? -1.0f : 1.0f) * (row % 2 ? 0.5f : 1.0f); + else if (pattern != "zeros") throw std::logic_error("unknown input pattern"); + result[row * dim + j] = hadamard::as_storage(value); + } + return result; +} + +struct Validation { + double max_error = 0; + std::size_t dense_rows = 0; + std::size_t rounded_warp_mismatches = 0; + std::size_t dense_quant_byte_mismatches = 0; + std::size_t dense_quant_scale_mismatches = 0; +}; + +template +Validation validate(Runner& runner, const std::vector& input, bool all_rows) { + runner.input.upload(input); + const auto floats = to_float(input); + std::vector indices; + if (all_rows || runner.rows <= 32) { + for (std::size_t row = 0; row < runner.rows; ++row) indices.push_back(row); + } else { + for (std::size_t i = 0; i < 32; ++i) indices.push_back(i * (runner.rows - 1) / 31); + } + std::vector sample; + sample.reserve(product(indices.size(), runner.dim)); + for (const auto row : indices) + sample.insert(sample.end(), floats.begin() + row * runner.dim, floats.begin() + (row + 1) * runner.dim); + const auto dense = hadamard::dense_reference(sample, runner.dim, runner.scale); + std::vector expected(dense.size()); + for (std::size_t i = 0; i < dense.size(); ++i) + expected[i] = hadamard::as_float(hadamard::as_storage(static_cast(dense[i]))); + Validation validation{0, indices.size()}; + const double tolerance = std::is_same::value ? 1e-2 : 5e-2; + std::vector warp_values; + std::vector warp_row_exact(indices.size(), true); + for (const auto method : {Method::Naive, Method::Warp, Method::TensorCore}) { + if (method == Method::TensorCore && runner.dim < 16) continue; + runner.run(method); + const auto actual = to_float(runner.output.download()); + for (std::size_t s = 0; s < indices.size(); ++s) + for (int j = 0; j < runner.dim; ++j) { + const auto index = indices[s] * runner.dim + j; + const double difference = std::abs(static_cast(actual[index]) - expected[s * runner.dim + j]); + validation.max_error = std::max(validation.max_error, difference); + if (method == Method::Warp && difference != 0) { + ++validation.rounded_warp_mismatches; + warp_row_exact[s] = false; + } + if (!std::isfinite(actual[index]) || !(difference < tolerance)) { + std::ostringstream message; + message << method_name(method) << " oracle mismatch row=" << indices[s] << " column=" << j + << " actual=" << actual[index] << " expected=" << expected[s * runner.dim + j] + << " abs_error=" << difference << " tolerance(strict)=" << tolerance; + throw std::runtime_error(message.str()); + } + } + if (method == Method::Warp) warp_values = actual; + } + // Check the full quantization contract, even when dense transform checking is sampled. + const auto cpu_quant = hadamard::quantize_int4(warp_values, runner.dim); + const auto dense_quant = hadamard::quantize_int4(expected, runner.dim); + runner.run(Method::SplitInt4); + const auto split_packed = runner.packed.download(); + const auto split_scales = runner.quant_scales.download(); + runner.run(Method::FusedInt4); + const auto fused_packed = runner.packed.download(); + const auto fused_scales = runner.quant_scales.download(); + if (split_packed != fused_packed || split_packed != cpu_quant.packed) + throw std::runtime_error("CPU/split/fused INT4 packed bytes mismatch"); + for (std::size_t row = 0; row < runner.rows; ++row) + if (split_scales[row] != fused_scales[row] || split_scales[row] != cpu_quant.scales[row]) + throw std::runtime_error("CPU/split/fused INT4 scales mismatch at row " + std::to_string(row)); + const int bytes = (runner.dim + 1) / 2; + for (std::size_t s = 0; s < indices.size(); ++s) { + if (split_scales[indices[s]] != dense_quant.scales[s]) { + if (warp_row_exact[s]) throw std::runtime_error("dense-oracle INT4 scale mismatch for exact transform row"); + ++validation.dense_quant_scale_mismatches; + } + for (int j = 0; j < bytes; ++j) + if (split_packed[indices[s] * bytes + j] != dense_quant.packed[s * bytes + j]) { + if (warp_row_exact[s]) throw std::runtime_error("dense-oracle INT4 packed bytes mismatch for exact transform row"); + ++validation.dense_quant_byte_mismatches; + } + } + return validation; +} + +template void self_test_dtype(const char* dtype, std::size_t& cases, Validation& totals, + int block_threads, bool contiguous256_fused, hadamard::RowLayout row_layout) { + auto record = [&](const Validation& validation) { + totals.max_error = std::max(totals.max_error, validation.max_error); + totals.dense_rows += validation.dense_rows; + totals.rounded_warp_mismatches += validation.rounded_warp_mismatches; + totals.dense_quant_byte_mismatches += validation.dense_quant_byte_mismatches; + totals.dense_quant_scale_mismatches += validation.dense_quant_scale_mismatches; + ++cases; + }; + for (int dim = 1; dim <= 256; dim *= 2) { + for (const bool normalized : {false, true}) + for (const std::size_t rows : {1u, 3u, 17u, 65u}) { + const float scale = normalized ? 1.0f / std::sqrt(static_cast(dim)) : 1.0f; + Runner runner(rows, dim, scale, block_threads, contiguous256_fused && dim == 256, row_layout); + auto test = [&](const char* pattern, std::uint32_t seed) { + const auto input = make_input(rows, dim, pattern, seed); + try { + record(validate(runner, input, true)); + } catch (const std::exception& error) { + throw std::runtime_error(std::string(dtype) + " dim=" + std::to_string(dim) + + " rows=" + std::to_string(rows) + " pattern=" + pattern + + " seed=" + std::to_string(seed) + + " normalized=" + std::to_string(normalized) + ": " + error.what()); + } + }; + for (const auto* pattern : {"random", "zeros", "impulse", "alternating"}) + test(pattern, 2026 + dim * 17 + rows); + for (const auto* pattern : {"uniform", "normal", "outlier"}) + for (const std::uint32_t seed : {2026u, 95811u, 314159u}) test(pattern, seed); + } + std::cout << "SELF_TEST " << dtype << " dim=" << dim << " PASS fused_layout=" + << (contiguous256_fused && dim == 256 ? "contiguous256" : "original") << std::endl; + } + // Larger, non-multiple batch: full split/fused/CPU-quant comparison, 32 dense rows. + for (const bool normalized : {false, true}) { + Runner runner(1025, 256, normalized ? 1.0f / 16.0f : 1.0f, block_threads, contiguous256_fused, row_layout); + record(validate(runner, make_input(1025, 256, "random", 95811), false)); + } +} + +double event_microseconds(const std::function& launch, int warmup, int repetitions) { + for (int i = 0; i < warmup; ++i) launch(); + CUDA_CHECK(cudaDeviceSynchronize()); + Event start, stop; + CUDA_CHECK(cudaEventRecord(start)); + for (int i = 0; i < repetitions; ++i) launch(); + CUDA_CHECK(cudaEventRecord(stop)); + CUDA_CHECK(cudaEventSynchronize(stop)); + float milliseconds = 0; + CUDA_CHECK(cudaEventElapsedTime(&milliseconds, start, stop)); + return static_cast(milliseconds) * 1000.0 / repetitions; +} + +std::string csv_quote(const std::string& value) { + std::string result = "\""; + for (const char c : value) { if (c == '\"') result += '\"'; result += c; } + return result + '\"'; +} + +std::string timestamp() { + const auto now = std::time(nullptr); + const std::tm* utc = std::gmtime(&now); + std::ostringstream value; + value << std::put_time(utc, "%Y-%m-%dT%H:%M:%SZ"); + return value.str(); +} + +struct Measurement { std::string method, scope; int repetitions; double microseconds; }; + +void report(const Options& options, const cudaDeviceProp& gpu, const std::vector& measurements, + double max_error, std::size_t dense_rows) { + int runtime = 0, driver = 0; + CUDA_CHECK(cudaRuntimeGetVersion(&runtime)); + CUDA_CHECK(cudaDriverGetVersion(&driver)); + std::ofstream csv; + std::string csv_header = "timestamp_utc,gpu,compute_capability,cuda_runtime,cuda_driver,batch,seq,heads,dim,dtype,scale,method,scope,repetitions,mean_us,input_elements_per_second,max_abs_error,dense_oracle_rows,warp_block_threads,mean_ms,fused_layout"; + const bool detailed_rows = options.row_layout != "original"; + if (detailed_rows) csv_header += ",requested_row_layout,resolved_row_layout,quantize_block_threads"; + bool header = false; + if (!options.csv.empty()) { + const std::filesystem::path path(options.csv); + if (path.has_parent_path()) std::filesystem::create_directories(path.parent_path()); + header = !std::filesystem::exists(path) || std::filesystem::file_size(path) == 0; + if (!header) { + std::ifstream existing(path); + std::string first_line; + std::getline(existing, first_line); + if (!first_line.empty() && first_line.back() == '\r') first_line.pop_back(); + if (first_line != csv_header) + throw std::runtime_error("CSV header differs; choose a new file to preserve existing results"); + } + csv.open(path, std::ios::app); + if (!csv) throw std::runtime_error("cannot open CSV file: " + options.csv); + if (header) + csv << csv_header << '\n'; + } + std::cout << "BENCHMARK shape=[" << options.batch << ',' << options.seq << ',' << options.heads << ',' + << options.dim << "] dtype=" << options.dtype << " scale=" << options.scale() + << " rows=" << options.rows() << " dense_oracle_rows=" << dense_rows + << (options.row_layout == "auto" ? " fallback_block_threads=" : " warp_block_threads=") << options.block_threads + << " fused_layout=" << options.fused_layout << " row_layout=" << options.row_layout << '\n'; + std::cout << "Timing: kernel_only=CUDA events, allocations/H2D/matrix setup excluded;\n" + << "cpu_compute=FP32 FWHT host wall time, input reset excluded;\n" + << "host_e2e=pageable H2D + warp transform + D2H, preallocated buffers.\n" + << "Throughput=input elements / elapsed seconds; it is not FLOP/s or physical memory bandwidth.\n"; + for (const auto& measured : measurements) { + const double throughput = static_cast(options.elements()) * 1e6 / measured.microseconds; + std::cout << std::left << std::setw(20) << measured.method << std::setw(16) << measured.scope + << std::right << std::fixed << std::setprecision(3) << std::setw(12) << measured.microseconds + << " us (" << std::setprecision(6) << measured.microseconds / 1000.0 << " ms) " + << std::scientific << std::setprecision(4) << throughput << " elements/s\n"; + if (csv) { + const bool uses_warp = measured.method == "warp" || measured.method == "split_int4" || + measured.method == "fused_int4" || measured.method == "warp_h2d_d2h"; + csv << timestamp() << ',' << csv_quote(gpu.name) << ',' << gpu.major << gpu.minor << ',' + << runtime << ',' << driver << ',' << options.batch << ',' << options.seq << ',' << options.heads + << ',' << options.dim << ',' << options.dtype << ',' << std::setprecision(9) << options.scale() + << ',' << measured.method << ',' << measured.scope << ',' << measured.repetitions << ',' + << std::setprecision(12) << measured.microseconds << ',' << throughput << ',' << max_error << ',' << dense_rows << ','; + const auto choice = hadamard::choose_rows(hadamard::parse_row_layout(options.row_layout), + hadamard::row_device(gpu.name), options.rows(), static_cast(options.dim), + measured.method == "fused_int4", options.block_threads); + if (uses_warp) csv << choice.threads; + csv << ',' << measured.microseconds / 1000.0 << ','; + if (measured.method == "fused_int4") csv << options.fused_layout; + if (detailed_rows) { + csv << ',' << options.row_layout << ','; + if (uses_warp) csv << (choice.packed ? "packed" : "original"); + csv << ','; + if (measured.method == "split_int4") csv << options.block_threads; + } + csv << '\n'; + } + } + if (csv) { csv.flush(); if (!csv) throw std::runtime_error("failed writing CSV"); } +} + +template void benchmark(const Options& options, const cudaDeviceProp& gpu) { + Runner runner(options.rows(), static_cast(options.dim), options.scale(), options.block_threads, + options.fused_layout == "contiguous256", hadamard::parse_row_layout(options.row_layout)); + const auto input = make_input(runner.rows, runner.dim, "random", 20260905); + const auto validation = validate(runner, input, false); + std::vector measurements; + for (const auto method : {Method::Naive, Method::Warp, Method::TensorCore, Method::SplitInt4, Method::FusedInt4}) { + if (method == Method::TensorCore && options.dim < 16) continue; + const double duration = event_microseconds([&]() { runner.run(method); }, options.warmup, options.repetitions); + measurements.push_back({method_name(method), "kernel_only", options.repetitions, duration}); + } + using Clock = std::chrono::steady_clock; + const int host_repetitions = std::min(options.repetitions, 20); + const auto source = to_float(input); + std::vector cpu(source.size()); + double cpu_microseconds = 0; + volatile float cpu_result = 0; + for (int i = 0; i < host_repetitions + 1; ++i) { + cpu = source; + const auto begin = Clock::now(); + hadamard::fwht(cpu.data(), runner.rows, options.dim, options.scale()); + const auto finish = Clock::now(); + cpu_result = cpu.front(); + if (i) cpu_microseconds += std::chrono::duration(finish - begin).count(); + } + (void)cpu_result; + measurements.push_back({"cpu_fp32_fwht", "cpu_compute", host_repetitions, cpu_microseconds / host_repetitions}); + std::vector host_output(input.size()); + double e2e_microseconds = 0; + for (int i = 0; i < host_repetitions + 1; ++i) { + const auto begin = Clock::now(); + runner.input.upload(input); + runner.run(Method::Warp); + CUDA_CHECK(cudaMemcpy(host_output.data(), runner.output.data(), runner.output.bytes(), cudaMemcpyDeviceToHost)); + const auto finish = Clock::now(); + if (i) e2e_microseconds += std::chrono::duration(finish - begin).count(); + } + measurements.push_back({"warp_h2d_d2h", "host_e2e", host_repetitions, e2e_microseconds / host_repetitions}); + report(options, gpu, measurements, validation.max_error, validation.dense_rows); +} + +} // namespace + +int main(int argc, char** argv) { + try { + for (int i = 1; i < argc; ++i) + if (std::string(argv[i]) == "--help" || std::string(argv[i]) == "-h") { help(); return 0; } + const auto options = parse(argc, argv); + int devices = 0; + CUDA_CHECK(cudaGetDeviceCount(&devices)); + if (!devices) throw std::runtime_error("no CUDA device is available"); + CUDA_CHECK(cudaSetDevice(0)); + cudaDeviceProp gpu{}; + CUDA_CHECK(cudaGetDeviceProperties(&gpu, 0)); + if (gpu.major < 8) throw std::runtime_error("this build requires an sm80+ GPU for BF16 Tensor Cores"); + std::cout << "GPU=" << gpu.name << " sm=" << gpu.major << gpu.minor << '\n'; + if (options.self_test) { + std::size_t cases = 0; + Validation totals; + const bool contiguous256_fused = options.fused_layout == "contiguous256"; + self_test_dtype<__half>("fp16", cases, totals, options.block_threads, contiguous256_fused, hadamard::parse_row_layout(options.row_layout)); + self_test_dtype<__nv_bfloat16>("bf16", cases, totals, options.block_threads, contiguous256_fused, hadamard::parse_row_layout(options.row_layout)); + std::cout << "SELF_TEST PASS cases=" << cases << " max_abs_error=" << totals.max_error + << " CPU/split/fused_INT4_bytes=exact scales=exact" + << " rounded_warp_vs_dense_elements=" << totals.rounded_warp_mismatches + << " dense_quant_differing_bytes=" << totals.dense_quant_byte_mismatches + << " dense_quant_differing_scales=" << totals.dense_quant_scale_mismatches + << (options.row_layout == "auto" ? " fallback_block_threads=" : " warp_block_threads=") << options.block_threads + << " row_layout=" << options.row_layout + << " fused_layout=" << options.fused_layout + << " fused_layout_scope=" << (contiguous256_fused ? "N256_only_other_N_original" : "all_N_original") << '\n'; + } + if (options.benchmark) { + if (options.dtype == "fp16") benchmark<__half>(options, gpu); + else benchmark<__nv_bfloat16>(options, gpu); + } + CUDA_CHECK(cudaDeviceSynchronize()); + return 0; + } catch (const std::invalid_argument& error) { + std::cerr << "Argument error: " << error.what() << "\nUse --help for usage.\n"; + return 2; + } catch (const std::exception& error) { + std::cerr << "ERROR: " << error.what() << '\n'; + return 1; + } +} diff --git a/03_hadamard_tc/a962695448-rgb/src/torch_binding.cu b/03_hadamard_tc/a962695448-rgb/src/torch_binding.cu new file mode 100644 index 00000000..ca22740b --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/src/torch_binding.cu @@ -0,0 +1,303 @@ +#include +#include +#include +#include +#include +#include + +#include "kernels.cuh" +#include "contiguous256.cuh" +#include "packed_rows.cuh" +#include "packed_pairs.cuh" +#include "row_policy.hpp" +#include + +#include +#include +#include +#include +#include + +namespace { + +void validate(const at::Tensor& input, double scale, int block_threads) { + TORCH_CHECK(block_threads == 128 || block_threads == 256, "block_threads must be 128 or 256"); + TORCH_CHECK(input.is_cuda(), "input must be a CUDA tensor"); + TORCH_CHECK(input.scalar_type() == at::kHalf || input.scalar_type() == at::kBFloat16, + "input dtype must be float16 or bfloat16"); + TORCH_CHECK(input.dim() == 2 || input.dim() == 4, "input must have 2 or 4 dimensions"); + TORCH_CHECK(input.is_contiguous(), "input must be contiguous"); + TORCH_CHECK(!input.is_neg(), "input must not have a lazy negative bit; call resolve_neg() first"); + TORCH_CHECK(input.numel() > 0, "input must be nonempty"); + const auto dim = input.size(-1); + TORCH_CHECK(dim >= 1 && dim <= 256 && !(dim & (dim - 1)), + "last dimension must be a power of two in [1,256]"); + TORCH_CHECK(!input.requires_grad(), "this forward-only extension does not support requires_grad=True"); + TORCH_CHECK(std::isfinite(scale) && scale > 0 && + std::isfinite(static_cast(scale)) && static_cast(scale) > 0, + "scale must be finite, positive, and representable as float32"); + const auto rows = input.numel() / dim; + TORCH_CHECK((rows - 1) / (block_threads / 32) + 1 <= std::numeric_limits::max(), + "input has too many rows for the CUDA launch grid"); +} + +template +void launch(const at::Tensor& input, at::Tensor& output, at::Tensor& packed, + at::Tensor& scales, float scale, cudaStream_t stream, int block_threads, + bool contiguous256_fused, bool packed_rows) { + const auto rows = static_cast(input.numel() / N); + const auto blocks = static_cast((rows - 1) / (block_threads / 32) + 1); + const auto* source = reinterpret_cast(input.data_ptr()); + auto* destination = output.defined() ? reinterpret_cast(output.data_ptr()) : nullptr; + auto* bytes = packed.defined() ? packed.data_ptr() : nullptr; + auto* row_scales = scales.defined() ? scales.data_ptr() : nullptr; + if constexpr (N <= 16) { + if (packed_rows) { + const auto packed_blocks = static_cast((rows - 1) / (block_threads / N) + 1); + hadamard::packed_rows_kernel<<>>( + source, destination, bytes, row_scales, rows, scale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return; + } + } + if constexpr (N == 256 && Transform && Quantize) { + if (contiguous256_fused) { + hadamard::contiguous256_kernel<<>>( + source, destination, bytes, row_scales, rows, scale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return; + } + } + hadamard::warp_kernel<<>>( + source, destination, bytes, row_scales, rows, scale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +template +void dispatch_dim(const at::Tensor& input, at::Tensor& output, at::Tensor& packed, + at::Tensor& scales, float scale, cudaStream_t stream, int block_threads, + bool contiguous256_fused, bool packed_rows) { + switch (input.size(-1)) { +#define DIM_CASE(N) case N: launch(input, output, packed, scales, scale, stream, block_threads, contiguous256_fused, packed_rows); break + DIM_CASE(1); DIM_CASE(2); DIM_CASE(4); DIM_CASE(8); DIM_CASE(16); + DIM_CASE(32); DIM_CASE(64); DIM_CASE(128); DIM_CASE(256); +#undef DIM_CASE + default: TORCH_CHECK(false, "unsupported dimension"); + } +} + +template +void dispatch(const at::Tensor& input, at::Tensor& output, at::Tensor& packed, + at::Tensor& scales, double scale, int block_threads, bool contiguous256_fused = false, bool packed_rows = false) { + const auto stream = c10::cuda::getCurrentCUDAStream(input.get_device()).stream(); + if (input.scalar_type() == at::kHalf) + dispatch_dim<__half, Transform, Quantize>(input, output, packed, scales, static_cast(scale), stream, block_threads, contiguous256_fused, packed_rows); + else + dispatch_dim<__nv_bfloat16, Transform, Quantize>(input, output, packed, scales, static_cast(scale), stream, block_threads, contiguous256_fused, packed_rows); +} + +template +void launch_pairs(const at::Tensor& input, at::Tensor& packed, at::Tensor& scales, + cudaStream_t stream, int block_threads) { + const auto rows = static_cast(input.numel() / N); + const auto blocks = static_cast((rows - 1) / (block_threads / (N / 2)) + 1); + hadamard::quantize_pairs_kernel<<>>( + reinterpret_cast(input.data_ptr()), packed.data_ptr(), + scales.data_ptr(), rows); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +template +void dispatch_pairs(const at::Tensor& input, at::Tensor& packed, at::Tensor& scales, + cudaStream_t stream, int block_threads) { + // Select the dimension before grid arithmetic so power-of-two divisions + // stay compile-time specializations, matching the existing launch path. + switch (input.size(-1)) { +#define PAIR_CASE(N) case N: launch_pairs(input, packed, scales, stream, block_threads); break + PAIR_CASE(2); PAIR_CASE(4); PAIR_CASE(8); PAIR_CASE(16); +#undef PAIR_CASE + default: TORCH_CHECK(false, "paired quantization requires dimension 2, 4, 8, or 16"); + } +} + +// Specialize only quantize-only dispatch. Hadamard launch/dispatch templates +// retain their existing definitions instead of inheriting the new routing body. +template <> +void dispatch(const at::Tensor& input, at::Tensor& output, at::Tensor& packed, + at::Tensor& scales, double scale, int block_threads, + bool contiguous256_fused, bool packed_rows) { + const auto stream = c10::cuda::getCurrentCUDAStream(input.get_device()).stream(); + if (packed_rows && input.size(-1) >= 2) { + if (input.scalar_type() == at::kHalf) + dispatch_pairs<__half>(input, packed, scales, stream, block_threads); + else + dispatch_pairs<__nv_bfloat16>(input, packed, scales, stream, block_threads); + return; + } + if (input.scalar_type() == at::kHalf) + dispatch_dim<__half, false, true>(input, output, packed, scales, static_cast(scale), stream, block_threads, contiguous256_fused, packed_rows); + else + dispatch_dim<__nv_bfloat16, false, true>(input, output, packed, scales, static_cast(scale), stream, block_threads, contiguous256_fused, packed_rows); +} + +hadamard::RowChoice row_choice(const at::Tensor& input, const std::string& name, + bool fused, int fallback_threads) { + const auto layout = hadamard::parse_row_layout(name); + TORCH_CHECK(layout != hadamard::RowLayout::Packed || input.size(-1) <= 16, + "row_layout='packed' requires last dimension at most 16"); + auto device = hadamard::RowDevice::Unknown; + if (layout == hadamard::RowLayout::Auto) + device = hadamard::row_device(at::cuda::getDeviceProperties(input.get_device())->name); + return hadamard::choose_rows(layout, device, input.numel() / input.size(-1), + static_cast(input.size(-1)), fused, fallback_threads); +} + +at::Tensor transform(const at::Tensor& input, double scale, int block_threads, + const std::string& row_layout = "original") { + validate(input, scale, block_threads); + const c10::cuda::CUDAGuard device_guard(input.device()); + auto output = at::empty_like(input); + at::Tensor packed, scales; + const auto choice = row_choice(input, row_layout, false, block_threads); + dispatch(input, output, packed, scales, scale, choice.threads, false, choice.packed); + return output; +} + +template +std::tuple quantized(const at::Tensor& input, double scale, int block_threads, + const std::string& fused_layout = "original", + const std::string& row_layout = "original") { + validate(input, scale, block_threads); + TORCH_CHECK(fused_layout == "original" || fused_layout == "contiguous256", + "fused_layout must be 'original' or 'contiguous256'"); + TORCH_CHECK(fused_layout == "original" || row_layout == "original", + "row_layout cannot be combined with contiguous256"); + const bool contiguous256_fused = fused_layout == "contiguous256"; + if (contiguous256_fused) { + TORCH_CHECK(Transform, "contiguous256 is only supported for fused Hadamard INT4"); + TORCH_CHECK(input.size(-1) == 256, "fused_layout='contiguous256' requires last dimension 256"); + TORCH_CHECK(block_threads == 128, "fused_layout='contiguous256' requires block_threads=128"); + } + const c10::cuda::CUDAGuard device_guard(input.device()); + auto packed_shape = input.sizes().vec(); + packed_shape.back() = (packed_shape.back() + 1) / 2; + auto scale_shape = input.sizes().vec(); + scale_shape.pop_back(); + auto packed = at::empty(packed_shape, input.options().dtype(at::kByte)); + auto scales = at::empty(scale_shape, input.options().dtype(at::kFloat)); + at::Tensor output; + const auto choice = row_choice(input, row_layout, true, block_threads); + dispatch(input, output, packed, scales, scale, choice.threads, contiguous256_fused, choice.packed); + return {packed, scales}; +} + +std::tuple quantize_only(const at::Tensor& input, int block_threads) { + return quantized(input, 1.0, block_threads); +} + +std::tuple quantize_only_packed(const at::Tensor& input, int block_threads) { + return quantized(input, 1.0, block_threads, "original", "packed"); +} + +void validate_buffer(const at::Tensor& input, const at::Tensor& buffer, + at::ScalarType dtype, int rank, const char* name) { + TORCH_CHECK(buffer.is_cuda() && buffer.device() == input.device(), name, " must be on the input CUDA device"); + TORCH_CHECK(buffer.scalar_type() == dtype, name, " has incorrect dtype"); + TORCH_CHECK(buffer.dim() == rank, name, " has incorrect rank"); + TORCH_CHECK(buffer.is_contiguous(), name, " must be contiguous"); + TORCH_CHECK(!buffer.is_neg() && !buffer.is_conj(), name, " must not have lazy negative/conjugate metadata"); + TORCH_CHECK(!buffer.requires_grad(), name, " must not require gradients"); + TORCH_CHECK(!buffer.is_inference() || c10::InferenceMode::is_enabled(), + name, " is an inference tensor; update it inside inference_mode"); +} + +void require_disjoint(const at::Tensor& first, const at::Tensor& second) { + // Both tensors have validated contiguous shapes on one device. Compare the + // active byte intervals, including distinct storage wrappers (e.g. DLPack). + const auto a = reinterpret_cast(first.data_ptr()); + const auto b = reinterpret_cast(second.data_ptr()); + const auto a_size = static_cast(first.numel()) * first.element_size(); + const auto b_size = static_cast(second.numel()) * second.element_size(); + TORCH_CHECK(a < b ? b - a >= a_size : a - b >= b_size, + "input and output buffers must have disjoint active byte ranges"); +} + +void transform_out(const at::Tensor& input, at::Tensor output, double scale, + int block_threads, const std::string& row_layout) { + validate(input, scale, block_threads); + validate_buffer(input, output, input.scalar_type(), input.dim(), "output"); + TORCH_CHECK(output.sizes() == input.sizes(), "output has incorrect shape"); + require_disjoint(input, output); + const c10::cuda::CUDAGuard device_guard(input.device()); + const auto choice = row_choice(input, row_layout, false, block_threads); + at::Tensor packed, scales; + torch::autograd::increment_version(output); + dispatch(input, output, packed, scales, scale, choice.threads, false, choice.packed); +} + +template +void quantized_out(const at::Tensor& input, at::Tensor packed, at::Tensor scales, + double scale, int block_threads, const std::string& fused_layout, + const std::string& row_layout) { + validate(input, scale, block_threads); + TORCH_CHECK(fused_layout == "original" || fused_layout == "contiguous256", + "fused_layout must be 'original' or 'contiguous256'"); + TORCH_CHECK(fused_layout == "original" || row_layout == "original", + "row_layout cannot be combined with contiguous256"); + const bool contiguous256 = fused_layout == "contiguous256"; + if (contiguous256) { + TORCH_CHECK(Transform && input.size(-1) == 256 && block_threads == 128, + "contiguous256 requires fused Hadamard, dimension 256, and 128 threads"); + } + validate_buffer(input, packed, at::kByte, input.dim(), "packed output"); + validate_buffer(input, scales, at::kFloat, input.dim() - 1, "scales output"); + TORCH_CHECK(packed.size(-1) == (input.size(-1) + 1) / 2, "packed output has incorrect shape"); + for (int i = 0; i < input.dim() - 1; ++i) { + TORCH_CHECK(packed.size(i) == input.size(i), "packed output has incorrect shape"); + TORCH_CHECK(scales.size(i) == input.size(i), "scales output has incorrect shape"); + } + require_disjoint(input, packed); + require_disjoint(input, scales); + require_disjoint(packed, scales); + const c10::cuda::CUDAGuard device_guard(input.device()); + const auto choice = row_choice(input, row_layout, true, block_threads); + at::Tensor output; + torch::autograd::increment_version(packed); + torch::autograd::increment_version(scales); + dispatch(input, output, packed, scales, scale, choice.threads, contiguous256, choice.packed); +} + +void quantize_only_out(const at::Tensor& input, at::Tensor packed, at::Tensor scales, int block_threads) { + quantized_out(input, packed, scales, 1.0, block_threads, "original", "original"); +} + +void quantize_only_packed_out(const at::Tensor& input, at::Tensor packed, at::Tensor scales, int block_threads) { + quantized_out(input, packed, scales, 1.0, block_threads, "original", "packed"); +} + +} // namespace + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, module) { + module.def("hadamard", &transform, pybind11::arg("input"), pybind11::arg("scale") = 1.0, pybind11::arg("block_threads") = 128, pybind11::arg("row_layout") = "original", + "Forward-only last-axis Hadamard; finite CUDA FP16/BF16 input, 2D or 4D contiguous. block_threads=128 (default) or 256; row_layout=original (default), packed (N<=16), or auto (verified model/range rules; may select 256 threads)."); + module.def("hadamard_int4", &quantized, pybind11::arg("input"), pybind11::arg("scale") = 1.0, pybind11::arg("block_threads") = 128, + pybind11::arg("fused_layout") = "original", pybind11::arg("row_layout") = "original", + "Fused transform and rowwise symmetric INT4; returns (uint8 packed, float32 scales). fused_layout='original' (default) supports block_threads=128 or 256; explicit 'contiguous256' requires N256 and block_threads=128. row_layout=packed/auto selects small-N row packing and cannot be combined with contiguous256."); + module.def("quantize_int4", &quantize_only, pybind11::arg("input"), pybind11::arg("block_threads") = 128, + "Quantize an already-rounded FP16/BF16 tensor; even values occupy the low nibble. block_threads=128 (default) or 256."); + module.def("quantize_int4_packed", &quantize_only_packed, pybind11::arg("input"), pybind11::arg("block_threads") = 128, + "Opt-in row-packed INT4 quantization for N<=16. Same rounding and packed output contract as quantize_int4; no automatic device policy."); + module.def("hadamard_out", &transform_out, pybind11::arg("input"), pybind11::arg("output"), + pybind11::arg("scale") = 1.0, pybind11::arg("block_threads") = 128, pybind11::arg("row_layout") = "original", + "Write Hadamard into a preallocated, disjoint same-shape/dtype output; returns None. No resizing or output allocation."); + module.def("hadamard_int4_out", &quantized_out, pybind11::arg("input"), pybind11::arg("packed"), pybind11::arg("scales"), + pybind11::arg("scale") = 1.0, pybind11::arg("block_threads") = 128, + pybind11::arg("fused_layout") = "original", pybind11::arg("row_layout") = "original", + "Write fused INT4 into preallocated uint8 packed and float32 scales buffers; returns None. All active byte ranges must be disjoint."); + module.def("quantize_int4_out", &quantize_only_out, pybind11::arg("input"), pybind11::arg("packed"), pybind11::arg("scales"), + pybind11::arg("block_threads") = 128, + "Write INT4 quantization into preallocated disjoint buffers; returns None. No resizing or output allocation."); + module.def("quantize_int4_packed_out", &quantize_only_packed_out, pybind11::arg("input"), pybind11::arg("packed"), pybind11::arg("scales"), + pybind11::arg("block_threads") = 128, + "Write row-packed INT4 quantization for N<=16 into disjoint caller buffers. Returns None; no resizing or output allocation."); +} diff --git a/03_hadamard_tc/a962695448-rgb/src/tune_launch.cu b/03_hadamard_tc/a962695448-rgb/src/tune_launch.cu new file mode 100644 index 00000000..942b15c7 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/src/tune_launch.cu @@ -0,0 +1,233 @@ +// 独立发射配置实验；不改变产品接口和默认派发。 +#include "kernels.cuh" +#include "reference.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +constexpr std::array threads{32, 64, 128, 256}; +constexpr int captured_calls = 64, replays = 20, groups = 5, warmup = 5; +std::string context; +void check(cudaError_t error, const char* call) { + if (error != cudaSuccess) + throw std::runtime_error(context + " " + call + ": " + cudaGetErrorString(error)); +} +#define CHECK(call) check((call), #call) +void cleanup(cudaError_t error) noexcept { + if (error != cudaSuccess) std::cerr << "cleanup: " << cudaGetErrorString(error) << '\n'; +} +template struct Buffer { + T* ptr = nullptr; + std::size_t count; + explicit Buffer(std::size_t n) : count(n) { + CHECK(cudaMalloc(reinterpret_cast(&ptr), count * sizeof(T))); + } + ~Buffer() { cleanup(cudaFree(ptr)); } + Buffer(const Buffer&) = delete; + std::vector read() const { + std::vector host(count); + CHECK(cudaMemcpy(host.data(), ptr, count * sizeof(T), cudaMemcpyDeviceToHost)); + return host; + } +}; +struct Stream { + cudaStream_t handle{}; + Stream() { CHECK(cudaStreamCreateWithFlags(&handle, cudaStreamNonBlocking)); } + ~Stream() { cleanup(cudaStreamDestroy(handle)); } +}; +struct Event { + cudaEvent_t handle{}; + Event() { CHECK(cudaEventCreate(&handle)); } + ~Event() { cleanup(cudaEventDestroy(handle)); } +}; +struct Graph { + cudaGraph_t graph{}; + cudaGraphExec_t exec{}; + ~Graph() { + if (exec) cleanup(cudaGraphExecDestroy(exec)); + if (graph) cleanup(cudaGraphDestroy(graph)); + } +}; +template void equal(const std::vector& actual, const std::vector& expected, + const char* name) { + if (actual.size() != expected.size()) throw std::runtime_error(context + " size mismatch"); + if (std::memcmp(actual.data(), expected.data(), actual.size() * sizeof(T)) == 0) return; + std::size_t i = 0; + while (std::memcmp(&actual[i], &expected[i], sizeof(T)) == 0) ++i; + std::ostringstream message; + message << context << " exact comparison failed: " << name << " first_index=" << i; + for (std::size_t j = i > 1 ? i - 2 : 0; j < std::min(i + 3, actual.size()); ++j) { + std::uint64_t a = 0, b = 0; + std::memcpy(&a, &actual[j], sizeof(T)); std::memcpy(&b, &expected[j], sizeof(T)); + message << " [" << j << ": actual=" << static_cast(actual[j]) + << " expected=" << static_cast(expected[j]) << " bits=0x" << std::hex + << a << "/0x" << b << std::dec << ']'; + } + throw std::runtime_error(message.str()); +} + +template +void run(int rows, float scale, const cudaDeviceProp& gpu, int runtime) { + const char* dtype = std::is_same_v ? "fp16" : "bf16"; + const std::string case_name = std::string(dtype) + " rows=" + std::to_string(rows) + + " N=" + std::to_string(N) + " scale=" + std::to_string(scale) + + " mode=" + (Quantize ? "fused_int4" : "transform"); + context = case_name + " baseline threads=128"; + const unsigned seed = 20260905u + rows + N; + std::mt19937 rng(seed); + std::uniform_real_distribution uniform(-1.0f, 1.0f); + std::vector input(static_cast(rows) * N); + for (std::size_t i = 0; i < input.size(); ++i) { + float x = uniform(rng); + if (i % 257 == 0) x *= 8; + if (i < N && rows > 1) x = 0; // 全零行覆盖量化 scale=1。 + input[i] = hadamard::as_storage(x); + } + Buffer x(input.size()), y(input.size()); + Buffer packed(static_cast(rows) * ((N + 1) / 2)); + Buffer scales(rows); + Stream stream; + // pageable H2D 的 host 返回不保证设备复制完成；必须和 nonblocking kernel 同 stream。 + CHECK(cudaMemcpyAsync(x.ptr, input.data(), input.size() * sizeof(T), + cudaMemcpyHostToDevice, stream.handle)); + auto launch = [&](int block_threads) { + const int blocks = (rows + block_threads / 32 - 1) / (block_threads / 32); + hadamard::warp_kernel<<>>( + x.ptr, y.ptr, packed.ptr, scales.ptr, rows, scale); + CHECK(cudaGetLastError()); + }; + // 独立取得原版变换结果，以其实际舍入后的值验证融合量化。 + hadamard::warp_kernel<<<(rows + 3) / 4, 128, 0, stream.handle>>>( + x.ptr, y.ptr, nullptr, nullptr, rows, scale); + CHECK(cudaGetLastError()); + CHECK(cudaStreamSynchronize(stream.handle)); + const auto transformed = y.read(); + std::vector first_rows(std::min(rows, 4) * N); + for (std::size_t i = 0; i < first_rows.size(); ++i) + first_rows[i] = hadamard::as_float(input[i]); + const auto dense = hadamard::dense_reference(first_rows, N, scale); + const double tolerance = std::is_same_v ? 1e-2 : 5e-2; + for (std::size_t i = 0; i < dense.size(); ++i) { + const auto rounded = hadamard::as_storage(static_cast(dense[i])); + const double error = std::abs(static_cast(hadamard::as_float(transformed[i])) - + hadamard::as_float(rounded)); + if (!(error < tolerance)) + throw std::runtime_error(context + " dense reference failed index=" + std::to_string(i) + + " actual=" + std::to_string(hadamard::as_float(transformed[i])) + + " expected=" + std::to_string(hadamard::as_float(rounded))); + } + launch(128); + CHECK(cudaStreamSynchronize(stream.handle)); + std::vector baseline_y; + std::vector baseline_packed; + std::vector baseline_scales; + if constexpr (Quantize) { + baseline_packed = packed.read(); + baseline_scales = scales.read(); + std::vector rounded(transformed.size()); + for (std::size_t i = 0; i < rounded.size(); ++i) + rounded[i] = hadamard::as_float(transformed[i]); + const auto cpu = hadamard::quantize_int4(rounded, N); + equal(baseline_packed, cpu.packed, "baseline packed vs CPU quantization"); + equal(baseline_scales, cpu.scales, "baseline scales vs CPU quantization"); + } else baseline_y = transformed; + auto validate = [&](const char* phase, int block_threads) { + context = case_name + " phase=" + phase + " threads=" + std::to_string(block_threads); + CHECK(cudaStreamSynchronize(stream.handle)); + if constexpr (Quantize) { + equal(packed.read(), baseline_packed, "packed"); + equal(scales.read(), baseline_scales, "scales"); + } else equal(y.read(), baseline_y, "transform output"); + }; + // 先验证每个配置的全部元素；写入哨兵防止复用旧输出掩盖漏写。 + for (int block_threads : threads) { + CHECK(cudaMemsetAsync(y.ptr, 0xa5, y.count * sizeof(T), stream.handle)); + CHECK(cudaMemsetAsync(packed.ptr, 0xa5, packed.count, stream.handle)); + CHECK(cudaMemsetAsync(scales.ptr, 0xa5, scales.count * sizeof(float), stream.handle)); + launch(block_threads); + validate("direct", block_threads); + } + std::array graphs; + for (int i = 0; i < 4; ++i) { + CHECK(cudaStreamBeginCapture(stream.handle, cudaStreamCaptureModeThreadLocal)); + for (int call = 0; call < captured_calls; ++call) launch(threads[i]); + CHECK(cudaStreamEndCapture(stream.handle, &graphs[i].graph)); + CHECK(cudaGraphInstantiate(&graphs[i].exec, graphs[i].graph, nullptr, nullptr, 0)); + CHECK(cudaGraphLaunch(graphs[i].exec, stream.handle)); + validate("graph_before_timing", threads[i]); + } + for (int w = 0; w < warmup; ++w) + for (int i = 0; i < 4; ++i) CHECK(cudaGraphLaunch(graphs[(i + w) % 4].exec, stream.handle)); + CHECK(cudaStreamSynchronize(stream.handle)); + Event start, stop; + std::array, groups> samples{}; + std::array, groups> order{}; + for (int g = 0; g < groups; ++g) { + for (int position = 0; position < 4; ++position) { + // 每组轮换起点并反转方向，避免所有组都按同一线程数顺序执行。 + const int i = (g + (g % 2 ? -position : position) + 4) % 4; + order[g][i] = position; + CHECK(cudaEventRecord(start.handle, stream.handle)); + for (int repeat = 0; repeat < replays; ++repeat) + CHECK(cudaGraphLaunch(graphs[i].exec, stream.handle)); + CHECK(cudaEventRecord(stop.handle, stream.handle)); + CHECK(cudaEventSynchronize(stop.handle)); + float elapsed_ms = 0; + CHECK(cudaEventElapsedTime(&elapsed_ms, start.handle, stop.handle)); + samples[g][i] = elapsed_ms * 1000.0 / (replays * captured_calls); + } + } + // 计时后仍验证捕获路径；所有检查与分配/复制均在 event 区间外。 + for (int i = 0; i < 4; ++i) { + CHECK(cudaGraphLaunch(graphs[i].exec, stream.handle)); + validate("graph_after_timing", threads[i]); + } + for (int g = 0; g < groups; ++g) + for (int i = 0; i < 4; ++i) + std::cout << '"' << gpu.name << "\"," << gpu.major * 10 + gpu.minor << ',' + << runtime << ',' << seed << ',' << (Quantize ? "fused_int4" : "transform") + << ',' << rows << ',' << N << ',' << dtype << ',' << scale << ',' << threads[i] + << ',' << g << ',' << order[g][i] << ',' << captured_calls << ',' << replays + << ',' << samples[g][i] << ",PASS\n"; + std::cout.flush(); +} +template void cases(const cudaDeviceProp& gpu, int runtime) { + for (int rows : {1, 17, 4096, 16384}) + for (float scale : {1.0f, 1.0f / std::sqrt(static_cast(N))}) { + run(rows, scale, gpu, runtime); + run(rows, scale, gpu, runtime); + } +} +} // namespace + +int main(int argc, char**) { + try { + if (argc != 1) throw std::invalid_argument("Usage: tune_launch > new-results.csv"); + int device = 0, runtime = 0; + CHECK(cudaGetDevice(&device)); + cudaDeviceProp gpu{}; + CHECK(cudaGetDeviceProperties(&gpu, device)); + CHECK(cudaRuntimeGetVersion(&runtime)); + if (gpu.major < 8) throw std::runtime_error("This experiment requires sm80 or newer"); + std::cerr << "Timing: fixed input/output buffers, 64 serial kernels per graph, 20 replays, " + "5 groups; per-call amortized graph interval, not isolated kernel latency.\n"; + std::cout << std::setprecision(10) + << "gpu,sm,cuda_runtime,seed,mode,rows,dim,dtype,scale,threads,group,order," + "captured_calls,replays,mean_us,check_status\n"; + cases<__half, 16>(gpu, runtime); cases<__half, 64>(gpu, runtime); cases<__half, 256>(gpu, runtime); + cases<__nv_bfloat16, 16>(gpu, runtime); cases<__nv_bfloat16, 64>(gpu, runtime); + cases<__nv_bfloat16, 256>(gpu, runtime); + CHECK(cudaDeviceSynchronize()); + std::cerr << "PASS: 96 shape/dtype/scale/mode cases; 384 launch configurations; 1920 raw samples.\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/03_hadamard_tc/a962695448-rgb/tests/cpu_reference_test.cpp b/03_hadamard_tc/a962695448-rgb/tests/cpu_reference_test.cpp new file mode 100644 index 00000000..9895de71 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/tests/cpu_reference_test.cpp @@ -0,0 +1,34 @@ +#include "reference.hpp" +#include +#include + +int main() { + using namespace hadamard; + std::mt19937 rng(2026); + std::uniform_real_distribution dist(-1, 1); + std::size_t cases = 0; + for (std::size_t n : {1, 2, 4, 8, 16, 32, 64, 128, 256}) { + std::vector x(3 * n); + for (float& v : x) v = dist(rng); + auto oracle = dense_reference(x, n); + std::vector actual(x.begin(), x.end()); + fwht(actual.data(), 3, n); + for (std::size_t i = 0; i < actual.size(); ++i) + if (std::abs(actual[i] - oracle[i]) > 1e-10) return 1; + fwht(actual.data(), 3, n, 1.0 / n); + for (std::size_t i = 0; i < x.size(); ++i) + if (std::abs(actual[i] - x[i]) > 1e-10) return 2; + ++cases; + } + if (nearest_even(0.5f) != 0 || nearest_even(1.5f) != 2 || + nearest_even(-0.5f) != 0 || nearest_even(-1.5f) != -2) return 3; + auto quant = quantize_int4({-7, 7, 0, 1}, 4); + if (quant.scales != std::vector{1} || + quant.packed != std::vector{0x79, 0x10}) return 4; + auto zero = quantize_int4({0}, 1); + if (zero.scales[0] != 1 || zero.packed[0] != 0) return 5; + bool rejected = false; + try { float x[3]{}; fwht(x, 1, 3); } catch (const std::invalid_argument&) { rejected = true; } + if (!rejected) return 6; + std::cout << "PASS: " << cases << " matrix-oracle/involution cases, rounding, packing, zero, invalid shape\n"; +} diff --git a/03_hadamard_tc/a962695448-rgb/tests/row_policy_test.cpp b/03_hadamard_tc/a962695448-rgb/tests/row_policy_test.cpp new file mode 100644 index 00000000..26207f53 --- /dev/null +++ b/03_hadamard_tc/a962695448-rgb/tests/row_policy_test.cpp @@ -0,0 +1,41 @@ +#include "row_policy.hpp" +#include +#include + +int main() { + using namespace hadamard; + unsigned checks = 0; + auto expect = [&](RowChoice actual, bool packed, int threads) { + ++checks; + if (actual.packed != packed || actual.threads != threads) + throw std::runtime_error("row layout policy regression"); + }; + const auto model = row_device("NVIDIA GeForce RTX 4090 D"); + if (model != RowDevice::RTX4090D || + row_device("NVIDIA GeForce RTX 4090") != RowDevice::RTX4090 || + row_device("unverified 4090 D") != RowDevice::Unknown) + throw std::runtime_error("device identities must match exactly"); + for (int threads : {128, 256}) { + for (bool fused : {false, true}) { + for (int dim : {1, 2, 4, 8, 16}) { + for (std::size_t rows : {4096u, 4097u, 32767u, 65536u}) + expect(choose_rows(RowLayout::Auto, model, rows, dim, fused, threads), true, 256); + for (std::size_t rows : {0u, 17u, 4095u, 65537u, 1000000u}) + expect(choose_rows(RowLayout::Auto, model, rows, dim, fused, threads), false, threads); + expect(choose_rows(RowLayout::Original, model, 4096, dim, fused, threads), false, threads); + expect(choose_rows(RowLayout::Packed, model, 17, dim, fused, threads), true, threads); + expect(choose_rows(RowLayout::Auto, RowDevice::Unknown, 4096, dim, fused, threads), false, threads); + } + for (int dim : {0, 3, 32, 64, 128, 256}) + expect(choose_rows(RowLayout::Auto, model, 4096, dim, fused, threads), false, threads); + } + } + // Representative existing-device boundaries remain part of the contract. + expect(choose_rows(RowLayout::Auto, RowDevice::RTX4090, 63, 8, true, 128), false, 128); + expect(choose_rows(RowLayout::Auto, RowDevice::RTX4090, 64, 8, true, 128), true, 256); + expect(choose_rows(RowLayout::Auto, RowDevice::A100, 255, 2, false, 128), false, 128); + expect(choose_rows(RowLayout::Auto, RowDevice::A100, 256, 2, false, 128), true, 256); + expect(choose_rows(RowLayout::Auto, RowDevice::A800, 4096, 16, false, 128), true, 256); + expect(choose_rows(RowLayout::Auto, RowDevice::A800, 4096, 16, true, 128), false, 128); + std::cout << "PASS: " << checks << " row-policy checks\n"; +}