diff --git a/pom.xml b/pom.xml index f518c905f..5cea51450 100644 --- a/pom.xml +++ b/pom.xml @@ -136,9 +136,12 @@ under the License. 2.0.12 generate_java_files + check_java_files check_cpp_files check_go_files check_cpp_historical_files + ${project.basedir} + ${project.basedir} -Xmx4g @@ -373,6 +376,7 @@ under the License. ${maven-javadoc-plugin.version} en_US + ${project.basedir}/src/main/javadoc/overview.html ${project.reporting.outputDirectory} true public @@ -480,6 +484,10 @@ under the License. 0 + + ${test.data.root} + ${project.root} + @@ -728,7 +736,33 @@ under the License. maven-surefire-plugin ${testng.generate-java-files} - ${testng.check-cpp-files},${testng.check-go-files},${testng.check-cpp-historical-files} + + ${testng.check-java-files}, + ${testng.check-cpp-files}, + ${testng.check-go-files}, + ${testng.check-cpp-historical-files} + + + + + + + + + check-java-files + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${testng.check-java-files} + + ${testng.generate-java-files}, + ${testng.check-cpp-files}, + ${testng.check-go-files}, + ${testng.check-cpp-historical-files} + @@ -744,7 +778,12 @@ under the License. maven-surefire-plugin ${testng.check-cpp-files} - ${testng.generate-java-files},${testng.check-go-files},${testng.check-cpp-historical-files} + + ${testng.generate-java-files}, + ${testng.check-java-files}, + ${testng.check-go-files}, + ${testng.check-cpp-historical-files} + @@ -760,7 +799,12 @@ under the License. maven-surefire-plugin ${testng.check-go-files} - ${testng.generate-java-files},${testng.check-cpp-files},${testng.check-cpp-historical-files} + + ${testng.generate-java-files}, + ${testng.check-java-files}, + ${testng.check-cpp-files}, + ${testng.check-cpp-historical-files} + @@ -776,7 +820,12 @@ under the License. maven-surefire-plugin ${testng.check-cpp-historical-files} - ${testng.generate-java-files},${testng.check-go-files},${testng.check-cpp-files} + + ${testng.generate-java-files}, + ${testng.check-java-files}, + ${testng.check-go-files}, + ${testng.check-cpp-files} + diff --git a/src/main/javadoc/overview.html b/src/main/javadoc/overview.html index 9124aa98a..536f0f16a 100644 --- a/src/main/javadoc/overview.html +++ b/src/main/javadoc/overview.html @@ -23,7 +23,7 @@ -

Sketching Core Library

+

Sketching Core Library For Java

Overview

The Sketching Core Library provides a range of stochastic streaming algorithms and closely @@ -33,9 +33,9 @@

Overview

This library is divided into packages that constitute distinct groups of functionality:

-Note: In general, if the requirements or promises of any method's contract are not fulfilled +

Note: In general, if the requirements or promises of any method's contract are not fulfilled (that is, if there is a bug in either the method or its caller), then an unchecked exception will be thrown. -The precise type of such an unchecked exception does not form part of any method's contract. +The precise type of such an unchecked exception does not form part of any method's contract.

diff --git a/src/test/java/org/apache/datasketches/common/TestUtil.java b/src/test/java/org/apache/datasketches/common/TestUtil.java deleted file mode 100644 index fbe39a856..000000000 --- a/src/test/java/org/apache/datasketches/common/TestUtil.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.common; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Objects; - -/** - * Utilities common to testing - */ -public final class TestUtil { - - private static final String userDir = System.getProperty("user.dir"); - - /** - * TestNG group constants - */ - public static final String GENERATE_JAVA_FILES = "generate_java_files"; - public static final String CHECK_CPP_FILES = "check_cpp_files"; - public static final String CHECK_GO_FILES = "check_go_files"; - public static final String CHECK_RUST_FILES = "check_rust_files"; - public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files"; - - /** - * The project relative Path for Java serialized sketches to be tested by other languages. - */ - public static final Path javaPath = Path.of(userDir, "serialization_test_data", "java_generated_files").normalize(); - - /** - * The project relative Path for C++ serialized sketches to be tested by Java. - */ - public static final Path cppPath = Path.of(userDir, "serialization_test_data", "cpp_generated_files").normalize(); - - /** - * The project relative Path for Go serialized sketches to be tested by Java. - */ - public static final Path goPath = Path.of(userDir, "serialization_test_data", "go_generated_files").normalize(); - - /** - * The project relative Path for Rust serialized sketches to be tested by Java. - */ - public static final Path rustPath = Path.of(userDir, "serialization_test_data", "rust_generated_files").normalize(); - - /** - * The project relative Path for /src/test/resources - */ - public static final Path resPath = Path.of(userDir,"src","test","resources").normalize(); - - public enum Existence { MUST_EXIST, WARNING } - - /** - * Gets all the bytes of a file as a byte array. - * If the file is missing, this either throws an exception or writes a warning message to the console - * based on the state of the optional {@link #Existence Existence}. - * @param basePath the base directory path where the file is located - * @param fileName the simple file name of the file - * @param option an optional parameter. If option == Existence.MUST_EXIST and the file does not exist an exception will be thrown. - * If option == Existence.WARNING, or not given, and the file does not exist, it writes a warning message - * to {@link System.err System.err}. - * If option has more than one argument an exception will be thrown. - * @return a byte array. It may be empty. - * @throws RuntimeException for IO errors, or if resolved path is not a file or not readable or optionally not found. - */ - public static byte[] getFileBytes(final Path basePath, final String fileName, Existence... option) { - Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); - Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); - if (option.length > 1) { throw new IllegalArgumentException("Existence option has a maximum of one argument"); } - Existence status = (option.length == 1) ? option[0] : Existence.WARNING; - - Path path = Path.of(basePath.toString(), fileName); - Path absPath = path.toAbsolutePath(); //for error output - if (Files.notExists(path)) { - if (status == Existence.MUST_EXIST) { - throw new RuntimeException("File disappeared or not found: " + absPath); - } else { - System.err.println("WARNING: File disappeared or not found: " + absPath); - return new byte[0]; - } - } - if (!Files.isRegularFile(path) || !Files.isReadable(path)) { - throw new RuntimeException("Path is not a regular file or not readable: " + absPath); - } - try { - byte[] bytes = Files.readAllBytes(path); - return bytes; - } catch (IOException e) { - throw new RuntimeException("System IO Error reading file: " + absPath + " " + e); - } - } - - /** - * Puts all the bytes of the given byte array to a file with the given fileName. - * This assumes that the base directory path is {@link #javaPath javaPath}. - * @param fileName the name of the target file - * @param bytes the given byte array - */ - public static void putBytesToJavaPath(final String fileName, final byte[] bytes) { - putBytesToFile(javaPath, fileName, bytes); - } - - /** - * Puts all the bytes of the given byte array to a basePath file with the given fileName. - * If the file exists it will be overwritten. - * @param basePath the directory path for the given fileName - * @param fileName the name of the target file - * @param bytes the given byte array - * @throws RuntimeException for IO errors, - */ - public static void putBytesToFile(final Path basePath, final String fileName, final byte[] bytes) { - Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); - Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); - Objects.requireNonNull(bytes, "input parameter 'byte[] bytes' cannot be null."); - Path filePath = null; - try { - Files.createDirectories(basePath); //create the directory if it doesn't exist. - filePath = basePath.resolve(fileName); - Files.write(filePath, bytes); - } catch (IOException e) { - final String filePathDesc = String.valueOf(filePath); - throw new RuntimeException("System IO Error writing file: " + filePathDesc + " " + e); - } - } -} diff --git a/src/test/java/org/apache/datasketches/common/UtilTest.java b/src/test/java/org/apache/datasketches/common/UtilTest.java index da8858ca7..f9f0b09ae 100644 --- a/src/test/java/org/apache/datasketches/common/UtilTest.java +++ b/src/test/java/org/apache/datasketches/common/UtilTest.java @@ -20,8 +20,6 @@ package org.apache.datasketches.common; import static java.lang.Math.pow; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.javaPath; import static org.apache.datasketches.common.Util.bytesToInt; import static org.apache.datasketches.common.Util.bytesToLong; import static org.apache.datasketches.common.Util.bytesToString; @@ -56,7 +54,6 @@ import static org.apache.datasketches.common.Util.zeroPad; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -436,11 +433,7 @@ static void checkCeilingMultiple2expK() { assertEquals(v, v2); } - @Test - public void checkDirCreation() { - assertNotNull(javaPath); - assertNotNull(cppPath); - } + @Test public void printlnTest() { diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java new file mode 100644 index 000000000..1353ea28c --- /dev/null +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.Objects; + +/** + * Utilities common to testing + */ +public final class UtilityIO { + private static final String LS = System.getProperty("line.separator"); + private static final Class clazz = UtilityIO.class; + private static final ClassLoader CL = clazz.getClassLoader(); + private static final Path testDataRoot = getTestDataRoot(); + public static final Path projectRoot = getProjectRoot(); + private static final String TEST_DATA_ROOT_PROP = "test.data.root"; + private static final String PROJECT_ROOT_PROP = "project.root"; + + /** + * TestNG group constants + */ + public static final String GENERATE_JAVA_FILES = "generate_java_files"; + public static final String CHECK_JAVA_FILES = "check_java_files"; + public static final String CHECK_CPP_FILES = "check_cpp_files"; + public static final String CHECK_GO_FILES = "check_go_files"; + public static final String CHECK_RUST_FILES = "check_rust_files"; + public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files"; + + public enum Existence { MUST_EXIST, WARNING } + + /** + * For passing variables to a multi-language test method leveraging TestNG Groups. + * + *

Usage in a method would look like this:

+ * {@snippet : + * public void testMethod(GpLanguage lang) { + * String suffix = lang.sfx; + * Path path = lang.pth; + * ... + * } + * } + */ + public enum GroupLanguage { + JAVA("_java", getJavaPath()), + CPP("_cpp", getCppPath()), + GO("_go", getGoPath()); + + public final String sfx; + public final Path pth; + + GroupLanguage(String suffix, Path path) { + this.sfx = suffix; + this.pth = path; + } + } + + /** + * The project relative Path for Java serialized sketches to be tested by other languages. + */ + public static Path getJavaPath() { + return testDataRoot.resolve("serialization_test_data", "java_generated_files"); + } + + /** + * The project relative Path for C++ serialized sketches to be tested by Java. + */ + public static Path getCppPath() { + return testDataRoot.resolve("serialization_test_data", "cpp_generated_files"); + } + + /** + * The project relative Path for Go serialized sketches to be tested by Java. + */ + public static Path getGoPath() { + return testDataRoot.resolve("serialization_test_data", "go_generated_files"); + } + + /** + * The project relative Path for Rust serialized sketches to be tested by Java. + */ + public static Path getRustPath() { + return testDataRoot.resolve("serialization_test_data", "rust_generated_files"); + } + + /** + * Gets all the bytes of a file as a byte array. + * If the file is missing this issues a Warning message to the console. + * @param basePath the directory where fileName is located. + * @param fileName the simple file name of the file. + * @return a byte array. It will be empty if file not found. + * @throws RuntimeException for IO errors. + */ + public static byte[] getFileBytes(final Path basePath, final String fileName) { + return getFileBytes(basePath, fileName, Existence.WARNING); + } + + /** + * Gets all the bytes of a file as a byte array. + * If the file is missing this issues a Warning message to the console or throws a RuntimeException + * based on the state of Existence. + * @param basePath the directory where fileName is located. + * @param fileName the simple file name of the file. + * @return a byte array. It will be empty if file not found and Existence is WARNING + * @throws RuntimeException for IO errors, + */ + public static byte[] getFileBytes(final Path basePath, final String fileName, final Existence existence) { + Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); + Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); + Objects.requireNonNull(existence, "input parameter 'Existence existence' cannot be null."); + + Path path = basePath.resolve(fileName).normalize(); + String pathDisplay; + try { + pathDisplay = path.toAbsolutePath().toString(); + } catch (SecurityException e) { + pathDisplay = path.toString(); + } + + try { + return Files.readAllBytes(path); + } catch (NoSuchFileException e) { + if (existence == Existence.MUST_EXIST) { + throw new RuntimeException("File not found: " + pathDisplay, e); + } + System.err.println("WARNING: File not found: " + pathDisplay); + return new byte[0]; + } catch (AccessDeniedException e) { + throw new RuntimeException("Permission denied reading file: " + pathDisplay, e); + } catch (IOException e) { + throw new RuntimeException("IO Error reading file: " + pathDisplay, e); + } + } + + /** + * This gets a byte[] from a resource file in /src/test/resources whether called on an install or a jar. + * These files must exist. + * @param fileName the desired filename + * @return the bytes of the file as a byte[] + * @throws IllegalArgumentException if file not found. + */ + public static byte[] getTestResourceBytes(final String fileName) { + Objects.requireNonNull(fileName, "fileName cannot be null"); + // Strip leading slash if present, as ClassLoader doesn't accept leading slashes + String resourcePath = fileName.startsWith("/") ? fileName.substring(1) : fileName; + try (InputStream is = CL.getResourceAsStream(resourcePath)) { + if (is == null) { + throw new IllegalArgumentException("Resource not found on classpath: " + resourcePath); + } + return is.readAllBytes(); + } catch (final IOException e){ + throw new RuntimeException("Error reading resource: " + resourcePath, e); + } + } + + /** + * Puts all the bytes of the given byte array to a file with the given fileName. + * This assumes that the base directory path is {@link #javaPath javaPath}. + * @param fileName the name of the target file + * @param bytes the given byte array + */ + public static void putBytesToJavaPath(final String fileName, final byte[] bytes) { + putBytesToFile(getJavaPath(), fileName, bytes); + } + + /** + * Puts all the bytes of the given byte array to a basePath file with the given fileName. + * If the file exists it will be overwritten. Called from putBytesToJavaPath() and TestUtilTest. + * + *

Used by putBytesToJavaPath(...) + * @param basePath the directory path for the given fileName + * @param fileName the name of the target file + * @param bytes the given byte array + * @throws RuntimeException for IO errors, + */ + static void putBytesToFile(final Path basePath, final String fileName, final byte[] bytes) { + Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); + Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); + Objects.requireNonNull(bytes, "input parameter 'byte[] bytes' cannot be null."); + Path filePath = null; + try { + Files.createDirectories(basePath); //create the directory if it doesn't exist. + filePath = basePath.resolve(fileName); + Files.write(filePath, bytes); + } catch (IOException e) { + final String filePathDesc = String.valueOf(filePath); + throw new RuntimeException("System IO Error writing file: " + filePathDesc + " " + e); + } + } + + /** + * Resolves the target directory for test data generation/reading. + * The default is the project root, but it can be overridden like this: + * + *

mvn clean test -Pcheck-cpp-files -Dtest.data.root="/tmp/custom_path

+ * + *

Calling the binary test jar: see src/test/resources/testng.xml

+ * + * Used by getJavaPath(), getCppPath(), getGoPath(), getRustPath() + * + * @return Path to the test.data.root directory. + * @throws IllegalStateException if running from a JAR without -Dtest.data.root defined. + */ + private static Path getTestDataRoot() { + // Explicit user override or Maven-injected property + String sysProp = System.getProperty(TEST_DATA_ROOT_PROP); + if (sysProp != null && !sysProp.isBlank()) { + return Path.of(sysProp).toAbsolutePath().normalize(); + } + + checkExecutionFromJar(TEST_DATA_ROOT_PROP); + + // Fallback for local source checkout / IDE runs (walk up to top-level .mvn) + return getProjectRoot(); + } + + /** + * Resolves the project root + * The default is the project root, but it can be overridden like this: + * + *

mvn clean test -Dproject.root="/tmp/custom_path

+ * + *

Calling the binary test jar: see src/test/resources/testng.xml

+ * + * Used by getTestDataRoot(), projectRoot + * + * @return Path to the project.root directory. + * @throws IllegalStateException if running from a JAR without -Dproject.root defined. + */ + private static Path getProjectRoot() { + // Explicit user override or Maven-injected property + String sysProp = System.getProperty(PROJECT_ROOT_PROP); + if (sysProp != null && !sysProp.isBlank()) { + return Path.of(sysProp).toAbsolutePath().normalize(); + } + + // Strict JAR Execution Rule: Must explicitly provide -Dproject.root = String path + checkExecutionFromJar(PROJECT_ROOT_PROP); + + // Fallback for local source checkout / IDE runs (walk up to top-level .mvn) + Path current = Path.of(System.getProperty("user.dir")).toAbsolutePath(); + while (current != null) { + if (Files.exists(current.resolve(".mvn"))) { + return current.normalize(); + } + current = current.getParent(); + } + throw new IllegalStateException("Could not find project root containing '.mvn' directory"); + } + + private static void checkExecutionFromJar(final String root) { + var url = clazz.getResource(clazz.getSimpleName() + ".class"); + if (url != null && "jar".equals(url.getProtocol())) { + throw new IllegalStateException( + "Executing from a packaged JAR file requires an explicit target directory." + LS + + "Please specify the output directory using: -D" + root + "=/path/to/directory" + ); + } + } + +} diff --git a/src/test/java/org/apache/datasketches/common/TestUtilTest.java b/src/test/java/org/apache/datasketches/common/UtilityIOTest.java similarity index 50% rename from src/test/java/org/apache/datasketches/common/TestUtilTest.java rename to src/test/java/org/apache/datasketches/common/UtilityIOTest.java index e8051fe88..6d6b80523 100644 --- a/src/test/java/org/apache/datasketches/common/TestUtilTest.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIOTest.java @@ -20,25 +20,38 @@ package org.apache.datasketches.common; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToFile; -import static org.apache.datasketches.common.TestUtil.resPath; -import static org.apache.datasketches.common.TestUtil.Existence.MUST_EXIST; -import static org.apache.datasketches.common.TestUtil.Existence.WARNING; +import static org.apache.datasketches.common.UtilityIO.getCppPath; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.getJavaPath; +import static org.apache.datasketches.common.UtilityIO.projectRoot; +import static org.apache.datasketches.common.UtilityIO.putBytesToFile; +import static org.apache.datasketches.common.UtilityIO.Existence.MUST_EXIST; +import static org.apache.datasketches.common.UtilityIO.Existence.WARNING; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -public class TestUtilTest { +public class UtilityIOTest { + + @Test + public void checkDirCreation() { + assertNotNull(getJavaPath()); + assertNotNull(getCppPath()); + } @Test public void testGetFileBytes_Success() {// throws IOException { - byte[] resultBytes = getFileBytes(resPath, "GettysburgAddress.txt"); + byte[] resultBytes = UtilityIO.getTestResourceBytes("GettysburgAddress.txt"); assertNotNull(resultBytes); String resultString = new String(resultBytes, UTF_8); assertTrue(resultString.startsWith("Abraham Lincoln's Gettysburg Address:")); @@ -46,29 +59,47 @@ public void testGetFileBytes_Success() {// throws IOException { @Test public void testGetFileBytes_MissingFile_Warning() { - byte[] resultBytes = getFileBytes(resPath, "Test_NonExistentFile_OK", WARNING); //WARNING is the default + byte[] resultBytes = getFileBytes(projectRoot, "Test_NonExistentFile_ThisIsOK", WARNING); assertNotNull(resultBytes); assertEquals(resultBytes.length, 0, "Should return empty array for missing file."); } @Test(expectedExceptions = RuntimeException.class) public void testGetFileBytes_MissingFile_MustExist() { - getFileBytes(resPath, "Test_NonExistentFile_OK", MUST_EXIST); + getFileBytes(projectRoot, "Test_NonExistentFile_ThisIsOK", MUST_EXIST); } @Test(expectedExceptions = RuntimeException.class) public void testGetFileBytes_NotRegular_NotReadable() throws IOException { - getFileBytes(resPath, ""); + getFileBytes(projectRoot, ""); } - private static final Path testPath = Path.of(".", "target", "testDir"); + private Path tempDir; + + @BeforeMethod + public void setUp() throws IOException { + // Creates a unique temporary directory in the OS temp location + tempDir = Files.createTempDirectory("testDir_"); + } @Test public void testPutBytesToFile() { - byte[] gettysBytes = getFileBytes(resPath, "GettysburgAddress.txt"); - putBytesToFile(testPath, "GettysburgAddressCopy.txt", gettysBytes); - byte[] gettysBytes2 = getFileBytes(testPath, "GettysburgAddressCopy.txt"); + byte[] gettysBytes = UtilityIO.getTestResourceBytes("GettysburgAddress.txt"); + putBytesToFile(tempDir, "GettysburgAddressCopy.txt", gettysBytes); + byte[] gettysBytes2 = getFileBytes(tempDir, "GettysburgAddressCopy.txt"); assertEquals(gettysBytes, gettysBytes2); } + @AfterMethod + public void tearDown() throws IOException { + // Recursively deletes the temp directory and all enclosed files after the test runs + if (tempDir != null && Files.exists(tempDir)) { + try (var stream = Files.walk(tempDir)) { + stream.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + } + } diff --git a/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java index b0f3ef77f..72fc81ccb 100644 --- a/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java @@ -19,18 +19,18 @@ package org.apache.datasketches.cpc; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_GO_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.goPath; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -69,30 +69,46 @@ void negativeIntEquivalence() throws Exception { putBytesToJavaPath("cpc_negative_one_java.sk", sk.toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + allFlavors(GroupLanguage.JAVA); + negativeIntEquivalence(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void allFlavors() throws IOException { - final int[] nArr = {0, 100, 200, 2000, 20000}; - final Flavor[] flavorArr = {Flavor.EMPTY, Flavor.SPARSE, Flavor.HYBRID, Flavor.PINNED, Flavor.SLIDING}; - int flavorIdx = 0; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "cpc_n" + n + "_cpp.sk"); - final CpcSketch sketch = CpcSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getFlavor(), flavorArr[flavorIdx++]); - assertEquals(sketch.getEstimate(), n, n * 0.02); - } + public void checkCpp() { + allFlavors(GroupLanguage.CPP); + negativeIntEquivalence(GroupLanguage.CPP); } @Test(groups = {CHECK_GO_FILES}) - public void checkAllFlavorsGo() throws IOException { + public void checkGo() { + allFlavors(GroupLanguage.GO); + negativeIntEquivalence(GroupLanguage.GO); + } + + private static void allFlavors(final GroupLanguage lang) { final int[] nArr = {0, 100, 200, 2000, 20000}; final Flavor[] flavorArr = {Flavor.EMPTY, Flavor.SPARSE, Flavor.HYBRID, Flavor.PINNED, Flavor.SLIDING}; int flavorIdx = 0; for (final int n: nArr) { - final byte[] bytes = getFileBytes(goPath, "cpc_n" + n + "_go.sk"); + String fileName = "cpc_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); final CpcSketch sketch = CpcSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getFlavor(), flavorArr[flavorIdx++]); assertEquals(sketch.getEstimate(), n, n * 0.02); } } + private static void negativeIntEquivalence(final GroupLanguage lang) { + String fileName = "cpc_negative_one" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final CpcSketch sketch = CpcSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sketch.getEstimate(), 1, 0.02); + } + } diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java index 4decdb91c..a48b240a4 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java @@ -19,17 +19,18 @@ package org.apache.datasketches.filters.bloomfilter; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -39,8 +40,8 @@ public class BloomFilterCrossLanguageTest { @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBloomFilterBinariesForCompatibilityTesting() throws IOException { - final int[] nArr = {0, 10_000, 2_000_000, 300_000_00}; + public void generateBloomFilterBinaries() { + final int[] nArr = {0, 10_000, 2_000_000, 30_000_000}; final short[] hArr = {3, 5}; for (final int n : nArr) { for (final short numHashes : hArr) { @@ -57,13 +58,30 @@ public void generateBloomFilterBinariesForCompatibilityTesting() throws IOExcept } } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + readBloomFilterBinaries(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void readBloomFilterBinariesForCompatibilityTesting() throws IOException { - final int[] nArr = {0, 10_000, 2_000_000, 300_000_00}; + public void checkCpp() { + readBloomFilterBinaries(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + readBloomFilterBinaries(GroupLanguage.GO); + } + + private static void readBloomFilterBinaries(final GroupLanguage lang) { + final int[] nArr = {0, 10_000, 2_000_000, 30_000_000}; final short[] hArr = {3, 5}; for (final int n : nArr) { for (final short numHashes : hArr) { - final byte[] bytes = getFileBytes(cppPath,"bf_n" + n + "_h" + numHashes + "_cpp.sk"); + final String fileName = "bf_n" + n + "_h" + numHashes + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); final BloomFilter bf = BloomFilter.heapify(MemorySegment.ofArray(bytes)); assertEquals(bf.isEmpty(), n == 0); assertTrue(bf.isEmpty() || (bf.getBitsUsed() > (n / 10))); @@ -77,4 +95,5 @@ public void readBloomFilterBinariesForCompatibilityTesting() throws IOException } } } + } diff --git a/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java index 49a735df7..c6f57f482 100644 --- a/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.frequencies; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -32,6 +33,7 @@ import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -93,11 +95,37 @@ public void generateBinariesForCompatibilityTestingStringsSketchUtf8() throws IO putBytesToJavaPath("frequent_string_utf8_java.sk", sk.toByteArray(new ArrayOfStringsSerDe())); } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + longs(GroupLanguage.JAVA); + strings(GroupLanguage.JAVA); + stringsAscii(GroupLanguage.JAVA); + stringsUtf8(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void longs() throws IOException { + public void checkCpp() { + longs(GroupLanguage.CPP); + strings(GroupLanguage.CPP); + stringsAscii(GroupLanguage.JAVA); + stringsUtf8(GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + longs(GroupLanguage.GO); + strings(GroupLanguage.GO); + stringsAscii(GroupLanguage.GO); + stringsUtf8(GroupLanguage.GO); + } + + private static void longs(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "frequent_long_n" + n + "_cpp.sk"); + final String fileName = "frequent_long_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final FrequentLongsSketch sketch = FrequentLongsSketch.getInstance(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); if (n > 10) { @@ -109,11 +137,13 @@ public void longs() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void strings() throws IOException { + private static void strings(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "frequent_string_n" + n + "_cpp.sk"); + final String fileName = "frequent_string_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final FrequentItemsSketch sketch = FrequentItemsSketch.getInstance(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); if (n > 10) { @@ -125,9 +155,11 @@ public void strings() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void stringsAscii() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "frequent_string_ascii_cpp.sk"); + private static void stringsAscii(final GroupLanguage lang) { + final String fileName = "frequent_string_ascii" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final FrequentItemsSketch sketch = FrequentItemsSketch.getInstance(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertFalse(sketch.isEmpty()); assertEquals(sketch.getMaximumError(), 0); @@ -138,9 +170,11 @@ public void stringsAscii() throws IOException { assertEquals(sketch.getEstimate("ddddddddddddddddddddddddddddd"), 4); } - @Test(groups = {CHECK_CPP_FILES}) - public void stringsUtf8() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "frequent_string_utf8_cpp.sk"); + private static void stringsUtf8(final GroupLanguage lang) { + final String fileName = "frequent_string_utf8" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final FrequentItemsSketch sketch = FrequentItemsSketch.getInstance(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertFalse(sketch.isEmpty()); assertEquals(sketch.getMaximumError(), 0); diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java index 51932d554..95d538ca2 100644 --- a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.hll; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.hll.TgtHllType.HLL_4; import static org.apache.datasketches.hll.TgtHllType.HLL_6; import static org.apache.datasketches.hll.TgtHllType.HLL_8; @@ -33,6 +34,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -63,39 +65,35 @@ public void generateBinariesForCompatibilityTesting() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void hll4() throws IOException { - final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "hll4_n" + n + "_cpp.sk"); - final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getLgConfigK(), 12); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertEquals(sketch.getEstimate(), n, n * 0.02); - } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + deserializeHll(GroupLanguage.JAVA); } @Test(groups = {CHECK_CPP_FILES}) - public void hll6() throws IOException { - final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "hll6_n" + n + "_cpp.sk"); - final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getLgConfigK(), 12); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertEquals(sketch.getEstimate(), n, n * 0.02); - } + public void checkCpp() { + deserializeHll(GroupLanguage.CPP); } - @Test(groups = {CHECK_CPP_FILES}) - public void hll8() throws IOException { + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeHll(GroupLanguage.GO); + } + + private static void deserializeHll(final GroupLanguage lang) { + final String[] sArr = {"hll4", "hll6", "hll8"}; final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "hll8_n" + n + "_cpp.sk"); - final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getLgConfigK(), 12); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertEquals(sketch.getEstimate(), n, n * 0.02); + for (final String s: sArr) { + for (final int n: nArr) { + final String fileName = s + "_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sketch.getLgConfigK(), 12); + assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); + assertEquals(sketch.getEstimate(), n, n * 0.02); + } } } diff --git a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java index 3ce30ec1e..a178b0c2f 100644 --- a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java @@ -19,13 +19,13 @@ package org.apache.datasketches.kll; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_HISTORICAL_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; -import static org.apache.datasketches.common.TestUtil.resPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_HISTORICAL_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -35,8 +35,9 @@ import java.util.Comparator; import org.apache.datasketches.common.ArrayOfStringsSerDe; -import org.apache.datasketches.common.TestUtil; import org.apache.datasketches.common.Util; +import org.apache.datasketches.common.UtilityIO; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIteratorAPI; import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; import org.apache.datasketches.quantilescommon.QuantilesGenericSketchIteratorAPI; @@ -90,35 +91,38 @@ public void generateKllItemsSketchBinaries() throws IOException { } } - @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) - public void checkCppKllDoublesSketchOneItemVersion1() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "kll_sketch_double_one_item_v1.sk"); - final KllDoublesSketch sk = KllDoublesSketch.heapify(MemorySegment.ofArray(byteArr)); - assertFalse(sk.isEmpty()); - assertFalse(sk.isEstimationMode()); - assertEquals(sk.getN(), 1); - assertEquals(sk.getNumRetained(), 1); - assertEquals(sk.getMinItem(), 1.0); - assertEquals(sk.getMaxItem(), 1.0); + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + kllFloat(GroupLanguage.JAVA); + kllDouble(GroupLanguage.JAVA); + kllString(GroupLanguage.JAVA); + kllLong(GroupLanguage.JAVA); } - @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) - public void checkCppKllFloatsSketchOneItemVersion1() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "kll_sketch_float_one_item_v1.sk"); - final KllFloatsSketch sk = KllFloatsSketch.heapify(MemorySegment.ofArray(byteArr)); - assertFalse(sk.isEmpty()); - assertFalse(sk.isEstimationMode()); - assertEquals(sk.getN(), 1); - assertEquals(sk.getNumRetained(), 1); - assertEquals(sk.getMinItem(), 1.0F); - assertEquals(sk.getMaxItem(), 1.0F); + @Test(groups = {CHECK_CPP_FILES}) + public void checkCpp() { + kllFloat(GroupLanguage.CPP); + kllDouble(GroupLanguage.CPP); + kllString(GroupLanguage.CPP); + kllLong(GroupLanguage.CPP); } - @Test(groups = {CHECK_CPP_FILES}) - public void kllFloat() throws IOException { + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + kllFloat(GroupLanguage.GO); + kllDouble(GroupLanguage.GO); + kllString(GroupLanguage.GO); + kllLong(GroupLanguage.GO); + + } + + private static void kllFloat(final GroupLanguage lang) { final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "kll_float_n" + n + "_cpp.sk"); + final String fileName = "kll_float_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllFloatsSketch sketch = KllFloatsSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getK(), 200); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); @@ -139,11 +143,13 @@ public void kllFloat() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void kllDouble() throws IOException { + private static void kllDouble(final GroupLanguage lang) { final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "kll_double_n" + n + "_cpp.sk"); + final String fileName = "kll_double_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllDoublesSketch sketch = KllDoublesSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getK(), 200); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); @@ -164,15 +170,14 @@ public void kllDouble() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void kllString() throws IOException { + private static void kllString(final GroupLanguage lang) { // sketch contains numbers in strings to make meaningful assertions final Comparator numericOrder = new Comparator() { @Override public int compare(final String s1, final String s2) { try { - final int i1 = Integer.parseInt(s1); - final int i2 = Integer.parseInt(s2); + final int i1 = Integer.parseInt(s1.trim()); + final int i2 = Integer.parseInt(s2.trim()); return Integer.compare(i1, i2); } catch (final NumberFormatException e) { throw new RuntimeException(e); @@ -181,7 +186,10 @@ public int compare(final String s1, final String s2) { }; final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "kll_string_n" + n + "_cpp.sk"); + final String fileName = "kll_string_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllHeapItemsSketch sketch = new KllHeapItemsSketch<>( MemorySegment.ofArray(bytes), numericOrder, @@ -192,8 +200,8 @@ public int compare(final String s1, final String s2) { assertTrue(n > 100 ? sketch.isEstimationMode() : !sketch.isEstimationMode()); assertEquals(sketch.getN(), n); if (n > 0) { - assertEquals(sketch.getMinItem(), Integer.toString(1)); - assertEquals(sketch.getMaxItem(), Integer.toString(n)); + assertEquals(sketch.getMinItem().trim(), Integer.toString(1)); + assertEquals(sketch.getMaxItem().trim(), Integer.toString(n)); long weight = 0; final QuantilesGenericSketchIteratorAPI it = sketch.iterator(); while (it.next()) { @@ -206,11 +214,13 @@ public int compare(final String s1, final String s2) { } } - @Test(groups = {CHECK_CPP_FILES}) - public void kllLong() throws IOException { + private static void kllLong(final GroupLanguage lang) { final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath,"kll_long_n" + n + "_cpp.sk"); + final String fileName = "kll_long_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllLongsSketch sketch = KllLongsSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getK(), 200); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); @@ -230,4 +240,31 @@ public void kllLong() throws IOException { } } } + + @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) + public void checkCppKllDoublesSketchOneItemVersion1() { + final String fileName = "kll_sketch_double_one_item_v1.sk"; + final byte[] byteArr = UtilityIO.getTestResourceBytes(fileName); + final KllDoublesSketch sk = KllDoublesSketch.heapify(MemorySegment.ofArray(byteArr)); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertEquals(sk.getMinItem(), 1.0); + assertEquals(sk.getMaxItem(), 1.0); + } + + @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) + public void checkCppKllFloatsSketchOneItemVersion1() { + final String fileName = "kll_sketch_float_one_item_v1.sk"; + final byte[] byteArr = UtilityIO.getTestResourceBytes(fileName); + final KllFloatsSketch sk = KllFloatsSketch.heapify(MemorySegment.ofArray(byteArr)); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertEquals(sk.getMinItem(), 1.0F); + assertEquals(sk.getMaxItem(), 1.0F); + } + } diff --git a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java index 53746c260..b7b53f154 100644 --- a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java @@ -19,13 +19,13 @@ package org.apache.datasketches.quantiles; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_HISTORICAL_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; -import static org.apache.datasketches.common.TestUtil.resPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_HISTORICAL_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -35,7 +35,8 @@ import java.util.Comparator; import org.apache.datasketches.common.ArrayOfStringsSerDe; -import org.apache.datasketches.common.TestUtil; +import org.apache.datasketches.common.UtilityIO; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIteratorAPI; import org.apache.datasketches.quantilescommon.QuantilesGenericSketchIteratorAPI; import org.testng.Assert; @@ -56,7 +57,7 @@ public void generateDoublesSketch() throws IOException { for (int i = 1; i <= n; i++) { sk.update(i); } - putBytesToJavaPath("quantiles_double_n" + n + "_java.sk", sk.toByteArray()); + putBytesToJavaPath("quantiles_double_n" + n + "_java.sk", sk.toByteArray(true)); } } @@ -87,12 +88,32 @@ public int compare(final String s1, final String s2) { } } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + checkDoublesSketch(GroupLanguage.JAVA); + checkItemsSketchWithStrings(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void checkDoublesSketch() throws IOException { + public void checkCpp() { + checkDoublesSketch(GroupLanguage.CPP); + checkItemsSketchWithStrings(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + checkDoublesSketch(GroupLanguage.GO); + checkItemsSketchWithStrings(GroupLanguage.GO); + } + + private static void checkDoublesSketch(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] byteArr = getFileBytes(cppPath, "quantiles_double_n" + n + "_cpp.sk"); - final QuantilesDoublesSketch sk = QuantilesDoublesSketch.wrap(MemorySegment.ofArray(byteArr)); + final String fileName = "quantiles_double_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final QuantilesDoublesSketch sk = QuantilesDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); assertTrue(n > 128 ? sk.isEstimationMode() : !sk.isEstimationMode()); assertEquals(sk.getN(), n); @@ -111,8 +132,7 @@ public void checkDoublesSketch() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void checkItemsSketchWithStrings() throws IOException { + private static void checkItemsSketchWithStrings(final GroupLanguage lang) { // sketch contains numbers in strings to make meaningful assertions final Comparator numericOrder = new Comparator() { @Override @@ -128,10 +148,13 @@ public int compare(final String s1, final String s2) { }; final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] byteArr = getFileBytes(cppPath, "quantiles_string_n" + n + "_cpp.sk"); + final String fileName = "quantiles_string_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final QuantilesItemsSketch sk = QuantilesItemsSketch.heapify( String.class, - MemorySegment.ofArray(byteArr), + MemorySegment.ofArray(bytes), numericOrder, new ArrayOfStringsSerDe() ); @@ -242,7 +265,7 @@ private static void getAndCheck(final String ver, final int n, final double quan println("fullName: "+ fileName); println("Old Median: " + quantile); //Read File bytes - final byte[] byteArr = TestUtil.getFileBytes(resPath, fileName); + final byte[] byteArr = UtilityIO.getTestResourceBytes(fileName); final MemorySegment srcSeg = MemorySegment.ofArray(byteArr); // heapify as update sketch diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java index 8e38b9b16..980275bb7 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java @@ -19,17 +19,19 @@ package org.apache.datasketches.req; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; import org.testng.annotations.Test; @@ -68,18 +70,41 @@ public void generateMixedBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {1, 10}; for (final int n: nArr) { final ReqSketch sk = ReqSketch.builder().build(); - for (int i = -n; i <= n; i++) { + for (int i = -n; i <= n; i++) { //produces sets of {-1,0,1}, {-10...,0,...10} sk.update(i); } putBytesToJavaPath("req_float_mixed_n" + n + "_java.sk", sk.toByteArray()); } } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + checkReqPositiveValues(GroupLanguage.JAVA); + checkReqNegativeValues(GroupLanguage.JAVA); + checkReqMixedValues(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCpp() throws IOException { + public void checkCpp() { + checkReqPositiveValues(GroupLanguage.CPP); + checkReqNegativeValues(GroupLanguage.CPP); + checkReqMixedValues(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + checkReqPositiveValues(GroupLanguage.GO); + checkReqNegativeValues(GroupLanguage.GO); + checkReqMixedValues(GroupLanguage.GO); + } + + private static void checkReqPositiveValues(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "req_float_n" + n + "_cpp.sk"); + final String fileName = "req_float_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); final ReqSketch sk = ReqSketch.heapify(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); assertTrue(n > 10 ? sk.isEstimationMode() : !sk.isEstimationMode()); @@ -98,4 +123,56 @@ public void deserializeFromCpp() throws IOException { } } } + + private static void checkReqNegativeValues(final GroupLanguage lang) { + final int[] nArr = {1, 10}; + for (final int n: nArr) { + final String fileName = "req_float_negative_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReqSketch sk = ReqSketch.heapify(MemorySegment.ofArray(bytes)); + assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); + assertTrue(n > 10 ? sk.isEstimationMode() : !sk.isEstimationMode()); + assertEquals(sk.getN(), n); + if (n > 0) { + assertEquals(sk.getMinItem(), -n); + assertEquals(sk.getMaxItem(), -1); + final QuantilesFloatsSketchIterator it = sk.iterator(); + long weight = 0; + while(it.next()) { + assertTrue(it.getQuantile() >= sk.getMinItem()); + assertTrue(it.getQuantile() <= sk.getMaxItem()); + weight += it.getWeight(); + } + assertEquals(weight, n); + } + } + } + + private static void checkReqMixedValues(final GroupLanguage lang) { + final int[] nArr = {1, 10}; + for (final int n: nArr) { + final String fileName = "req_float_mixed_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReqSketch sk = ReqSketch.heapify(MemorySegment.ofArray(bytes)); + assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); + if (n == 1) { assertEquals(sk.getN(), 3); } + if (n == 10) { assertEquals(sk.getN(), 21); } + if (n > 0) { + assertEquals(sk.getMinItem(), -n); + assertEquals(sk.getMaxItem(), n); + final QuantilesFloatsSketchIterator it = sk.iterator(); + long weight = 0; + while(it.next()) { + assertTrue(it.getQuantile() >= sk.getMinItem()); + assertTrue(it.getQuantile() <= sk.getMaxItem()); + weight += it.getWeight(); + } + assertEquals(weight, 2 * n + 1); + } + } + } } diff --git a/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java b/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java index 564f3b53b..a7c9cc914 100644 --- a/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java @@ -19,16 +19,24 @@ package org.apache.datasketches.sampling; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import java.io.IOException; +import java.lang.foreign.MemorySegment; import java.util.ArrayList; import org.apache.datasketches.common.ArrayOfDoublesSerDe; import org.apache.datasketches.common.ArrayOfLongsSerDe; import org.apache.datasketches.common.ArrayOfStringsSerDe; import org.apache.datasketches.common.ResizeFactor; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -38,15 +46,14 @@ public class ReservoirCrossLanguageTest { @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsSketchEmpty() throws IOException { + public void generateReservoirLongsSketchEmpty() throws IOException { //1 final int k = 128; final ReservoirLongsSketch sk = ReservoirLongsSketch.newInstance(k); - putBytesToJavaPath("reservoir_longs_empty_k" + k + "_java.sk", sk.toByteArray()); } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsSketchExact() throws IOException { + public void generateReservoirLongsSketchExact() throws IOException { //2 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -55,12 +62,13 @@ public void generateReservoirLongsSketchExact() throws IOException { for (int i = 0; i < n; i++) { sk.update(i); } + putBytesToJavaPath("reservoir_longs_exact_n" + n + "_k" + k + "_java.sk", sk.toByteArray()); } } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsSketchSampling() throws IOException { + public void generateReservoirLongsSketchSampling() throws IOException { //3 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -82,7 +90,7 @@ public void generateReservoirLongsSketchSampling() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsUnionEmpty() throws IOException { + public void generateReservoirLongsUnionEmpty() throws IOException { //4 int maxK = 128; ReservoirLongsUnion union = ReservoirLongsUnion.newInstance(maxK); @@ -90,7 +98,7 @@ public void generateReservoirLongsUnionEmpty() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsUnionExact() throws IOException { + public void generateReservoirLongsUnionExact() throws IOException { //5 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -104,7 +112,7 @@ public void generateReservoirLongsUnionExact() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsUnionSampling() throws IOException { + public void generateReservoirLongsUnionSampling() throws IOException { //6 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -129,7 +137,7 @@ public void generateReservoirLongsUnionSampling() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchLongEmpty() throws IOException { + public void generateReservoirItemsSketchLongEmpty() throws IOException { //7 final int k = 128; final ReservoirItemsSketch sk = ReservoirItemsSketch.newInstance(k); @@ -137,7 +145,7 @@ public void generateReservoirItemsSketchLongEmpty() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchLongExact() throws IOException { + public void generateReservoirItemsSketchLongExact() throws IOException { //8 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -151,7 +159,7 @@ public void generateReservoirItemsSketchLongExact() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchLongSampling() throws IOException { + public void generateReservoirItemsSketchLongSampling() throws IOException { //9 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -173,7 +181,7 @@ public void generateReservoirItemsSketchLongSampling() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchDoubleEmpty() throws IOException { + public void generateReservoirItemsSketchDoubleEmpty() throws IOException { //10 final int k = 128; final ReservoirItemsSketch sk = ReservoirItemsSketch.newInstance(k); @@ -181,7 +189,7 @@ public void generateReservoirItemsSketchDoubleEmpty() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchDoubleExact() throws IOException { + public void generateReservoirItemsSketchDoubleExact() throws IOException { //11 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -195,7 +203,7 @@ public void generateReservoirItemsSketchDoubleExact() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchDoubleSampling() throws IOException { + public void generateReservoirItemsSketchDoubleSampling() throws IOException { //12 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -217,7 +225,7 @@ public void generateReservoirItemsSketchDoubleSampling() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchStringEmpty() throws IOException { + public void generateReservoirItemsSketchStringEmpty() throws IOException { //13 final int k = 128; final ReservoirItemsSketch sk = ReservoirItemsSketch.newInstance(k); @@ -225,7 +233,7 @@ public void generateReservoirItemsSketchStringEmpty() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchStringExact() throws IOException { + public void generateReservoirItemsSketchStringExact() throws IOException { //14 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -239,7 +247,7 @@ public void generateReservoirItemsSketchStringExact() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchStringSampling() throws IOException { + public void generateReservoirItemsSketchStringSampling() throws IOException { //15 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -255,13 +263,12 @@ public void generateReservoirItemsSketchStringSampling() throws IOException { ResizeFactor.X8, k ); - putBytesToJavaPath("reservoir_items_string_sampling_n" + n + "_k" + k + "_java.sk", sk.toByteArray(new ArrayOfStringsSerDe())); } } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionLongEmpty() throws IOException { + public void generateReservoirItemsUnionLongEmpty() throws IOException { //16 int maxK = 128; ReservoirItemsUnion union = ReservoirItemsUnion.newInstance(maxK); @@ -269,7 +276,7 @@ public void generateReservoirItemsUnionLongEmpty() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionLongExact() throws IOException { + public void generateReservoirItemsUnionLongExact() throws IOException { //17 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -284,7 +291,7 @@ public void generateReservoirItemsUnionLongExact() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionLongSampling() throws IOException { + public void generateReservoirItemsUnionLongSampling() throws IOException { //18 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -310,7 +317,7 @@ public void generateReservoirItemsUnionLongSampling() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionDoubleEmpty() throws IOException { + public void generateReservoirItemsUnionDoubleEmpty() throws IOException { //19 int maxK = 128; ReservoirItemsUnion union = ReservoirItemsUnion.newInstance(maxK); @@ -318,7 +325,7 @@ public void generateReservoirItemsUnionDoubleEmpty() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionDoubleExact() throws IOException { + public void generateReservoirItemsUnionDoubleExact() throws IOException { //20 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -333,7 +340,7 @@ public void generateReservoirItemsUnionDoubleExact() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionDoubleSampling() throws IOException { + public void generateReservoirItemsUnionDoubleSampling() throws IOException { //21 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -359,7 +366,7 @@ public void generateReservoirItemsUnionDoubleSampling() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionStringEmpty() throws IOException { + public void generateReservoirItemsUnionStringEmpty() throws IOException { //22 int maxK = 128; ReservoirItemsUnion union = ReservoirItemsUnion.newInstance(maxK); @@ -367,7 +374,7 @@ public void generateReservoirItemsUnionStringEmpty() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionStringExact() throws IOException { + public void generateReservoirItemsUnionStringExact() throws IOException { //23 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -382,7 +389,7 @@ public void generateReservoirItemsUnionStringExact() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionStringSampling() throws IOException { + public void generateReservoirItemsUnionStringSampling() throws IOException { //24 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -406,4 +413,464 @@ public void generateReservoirItemsUnionStringSampling() throws IOException { union.toByteArray(new ArrayOfStringsSerDe())); } } + /*****************************************************/ + /*****************************************************/ + + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + checkReservoirLongsSketchEmpty(GroupLanguage.JAVA); + checkReservoirLongsSketchExact(GroupLanguage.JAVA); + checkReservoirLongsSketchSampling(GroupLanguage.JAVA); + checkReservoirLongsUnionEmpty(GroupLanguage.JAVA); + checkReservoirLongsUnionExact(GroupLanguage.JAVA); + checkReservoirLongsUnionSampling(GroupLanguage.JAVA); + checkReservoirItemsSketchLongEmpty(GroupLanguage.JAVA); + checkReservoirItemsSketchLongExact(GroupLanguage.JAVA); + checkReservoirItemsSketchLongSampling(GroupLanguage.JAVA); + checkReservoirItemsSketchDoubleEmpty(GroupLanguage.JAVA); + checkReservoirItemsSketchDoubleExact(GroupLanguage.JAVA); + checkReservoirItemsSketchDoubleSampling(GroupLanguage.JAVA); + checkReservoirItemsSketchStringEmpty(GroupLanguage.JAVA); + checkReservoirItemsSketchStringExact(GroupLanguage.JAVA); + checkReservoirItemsSketchStringSampling(GroupLanguage.JAVA); + checkReservoirItemsUnionLongEmpty(GroupLanguage.JAVA); + checkReservoirItemsUnionLongExact(GroupLanguage.JAVA); + checkReservoirItemsUnionLongSampling(GroupLanguage.JAVA); + checkReservoirItemsUnionDoubleEmpty(GroupLanguage.JAVA); + checkReservoirItemsUnionDoubleExact(GroupLanguage.JAVA); + checkReservoirItemsUnionDoubleSampling(GroupLanguage.JAVA); + checkReservoirItemsUnionStringEmpty(GroupLanguage.JAVA); + checkReservoirItemsUnionStringExact(GroupLanguage.JAVA); + checkReservoirItemsUnionStringSampling(GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CPP_FILES}) + public void checkCpp() { + checkReservoirLongsSketchEmpty(GroupLanguage.CPP); + checkReservoirLongsSketchExact(GroupLanguage.CPP); + checkReservoirLongsSketchSampling(GroupLanguage.CPP); + checkReservoirLongsUnionEmpty(GroupLanguage.CPP); + checkReservoirLongsUnionExact(GroupLanguage.CPP); + checkReservoirLongsUnionSampling(GroupLanguage.CPP); + checkReservoirItemsSketchLongEmpty(GroupLanguage.CPP); + checkReservoirItemsSketchLongExact(GroupLanguage.CPP); + checkReservoirItemsSketchLongSampling(GroupLanguage.CPP); + checkReservoirItemsSketchDoubleEmpty(GroupLanguage.CPP); + checkReservoirItemsSketchDoubleExact(GroupLanguage.CPP); + checkReservoirItemsSketchDoubleSampling(GroupLanguage.CPP); + checkReservoirItemsSketchStringEmpty(GroupLanguage.CPP); + checkReservoirItemsSketchStringExact(GroupLanguage.CPP); + checkReservoirItemsSketchStringSampling(GroupLanguage.CPP); + checkReservoirItemsUnionLongEmpty(GroupLanguage.CPP); + checkReservoirItemsUnionLongExact(GroupLanguage.CPP); + checkReservoirItemsUnionLongSampling(GroupLanguage.CPP); + checkReservoirItemsUnionDoubleEmpty(GroupLanguage.CPP); + checkReservoirItemsUnionDoubleExact(GroupLanguage.CPP); + checkReservoirItemsUnionDoubleSampling(GroupLanguage.CPP); + checkReservoirItemsUnionStringEmpty(GroupLanguage.CPP); + checkReservoirItemsUnionStringExact(GroupLanguage.CPP); + checkReservoirItemsUnionStringSampling(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + checkReservoirLongsSketchEmpty(GroupLanguage.GO); + checkReservoirLongsSketchExact(GroupLanguage.GO); + checkReservoirLongsSketchSampling(GroupLanguage.GO); + checkReservoirLongsUnionEmpty(GroupLanguage.GO); + checkReservoirLongsUnionExact(GroupLanguage.GO); + checkReservoirLongsUnionSampling(GroupLanguage.GO); + checkReservoirItemsSketchLongEmpty(GroupLanguage.GO); + checkReservoirItemsSketchLongExact(GroupLanguage.GO); + checkReservoirItemsSketchLongSampling(GroupLanguage.GO); + checkReservoirItemsSketchDoubleEmpty(GroupLanguage.GO); + checkReservoirItemsSketchDoubleExact(GroupLanguage.GO); + checkReservoirItemsSketchDoubleSampling(GroupLanguage.GO); + checkReservoirItemsSketchStringEmpty(GroupLanguage.GO); + checkReservoirItemsSketchStringExact(GroupLanguage.GO); + checkReservoirItemsSketchStringSampling(GroupLanguage.GO); + checkReservoirItemsUnionLongEmpty(GroupLanguage.GO); + checkReservoirItemsUnionLongExact(GroupLanguage.GO); + checkReservoirItemsUnionLongSampling(GroupLanguage.GO); + checkReservoirItemsUnionDoubleEmpty(GroupLanguage.GO); + checkReservoirItemsUnionDoubleExact(GroupLanguage.GO); + checkReservoirItemsUnionDoubleSampling(GroupLanguage.GO); + checkReservoirItemsUnionStringEmpty(GroupLanguage.GO); + checkReservoirItemsUnionStringExact(GroupLanguage.GO); + checkReservoirItemsUnionStringSampling(GroupLanguage.GO); + } + + //ReservoirLongsSketch + + private static void checkReservoirLongsSketchEmpty(final GroupLanguage lang) { //1 + final int k = 128; + final String fileName = "reservoir_longs_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirLongsSketch sk = ReservoirLongsSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirLongsSketchExact(final GroupLanguage lang) { //1 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (final int n : nArr) { + final String fileName = "reservoir_longs_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsSketch sk = ReservoirLongsSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirLongsSketchSampling(final GroupLanguage lang) { //3 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_longs_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsSketch sk = ReservoirLongsSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirLongsUnion + + private static void checkReservoirLongsUnionEmpty(final GroupLanguage lang) { //4 + int maxK = 128; + final String fileName = "reservoir_longs_union_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirLongsUnion rlu = ReservoirLongsUnion.heapify(MemorySegment.ofArray(bytes)); + assertEquals(rlu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirLongsUnionExact(final GroupLanguage lang) { //5 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_longs_union_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsUnion rlu = ReservoirLongsUnion.heapify(MemorySegment.ofArray(bytes)); + assertEquals(rlu.getMaxK(), maxK); + final ReservoirLongsSketch sk = rlu.getResult(); + assertTrue(sk.getN() == n); + assertTrue(sk.getNumSamples() == n); + } + } + + private static void checkReservoirLongsUnionSampling(final GroupLanguage lang) { //6 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_longs_union_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsUnion rlu = ReservoirLongsUnion.heapify(MemorySegment.ofArray(bytes)); + assertEquals(rlu.getMaxK(), maxK); + final ReservoirLongsSketch sk = rlu.getResult(); + assertTrue(sk.getN() == n); + assertTrue(sk.getNumSamples() == maxK); + } + } + + //ReservoirItemsSketch + + private static void checkReservoirItemsSketchLongEmpty(final GroupLanguage lang) { //7 + final int k = 128; + final String fileName = "reservoir_items_long_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirItemsSketchLongExact(final GroupLanguage lang) { //8 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_long_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsSketchLongSampling(final GroupLanguage lang) { //9 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_items_long_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirItemsSketch + + private static void checkReservoirItemsSketchDoubleEmpty(final GroupLanguage lang) { //10 + final int k = 128; + final String fileName = "reservoir_items_double_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirItemsSketchDoubleExact(final GroupLanguage lang) { //11 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_double_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsSketchDoubleSampling(final GroupLanguage lang) { //12 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_items_double_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirItemsSketch + + private static void checkReservoirItemsSketchStringEmpty(final GroupLanguage lang) { //13 + final int k = 128; + final String fileName = "reservoir_items_string_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirItemsSketchStringExact(final GroupLanguage lang) { //14 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_string_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsSketchStringSampling(final GroupLanguage lang) { //15 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_items_string_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirItemsUnion + + private static void checkReservoirItemsUnionLongEmpty(final GroupLanguage lang) { //16 + int maxK = 128; + final String fileName = "reservoir_items_union_long_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(riu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirItemsUnionLongExact(final GroupLanguage lang) { //17 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_union_long_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsUnionLongSampling(final GroupLanguage lang) { //18 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_items_union_long_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), maxK); + } + } + + //ReservoirItemsUnion + + private static void checkReservoirItemsUnionDoubleEmpty(final GroupLanguage lang) { //19 + int maxK = 128; + final String fileName = "reservoir_items_union_double_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(riu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirItemsUnionDoubleExact(final GroupLanguage lang) { //20 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_union_double_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsUnionDoubleSampling(final GroupLanguage lang) { //21 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_items_union_double_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), maxK); + } + } + + //ReservoirItemsUnion + + private static void checkReservoirItemsUnionStringEmpty(final GroupLanguage lang) { //22 + int maxK = 128; + final String fileName = "reservoir_items_union_string_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(riu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirItemsUnionStringExact(final GroupLanguage lang) { //23 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_union_string_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsUnionStringSampling(final GroupLanguage lang) { //24 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_items_union_string_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), maxK); + } + } } diff --git a/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java b/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java index 169245d4b..9bbe125f2 100644 --- a/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.sampling; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -33,6 +34,7 @@ import org.apache.datasketches.common.ArrayOfDoublesSerDe; import org.apache.datasketches.common.ArrayOfLongsSerDe; import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -103,11 +105,37 @@ public void generateUnionDoubleSampling() throws IOException { putBytesToJavaPath("varopt_union_double_sampling_java.sk", union.toByteArray(new ArrayOfDoublesSerDe())); } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + deserializeSketchLongs(GroupLanguage.JAVA); + deserializeSketchStringsExact(GroupLanguage.JAVA); + deserializeSketdhLongsSampling(GroupLanguage.JAVA); + deserializeUnionDoubleSampling(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppSketchLongs() throws IOException { + public void checkCpp() { + deserializeSketchLongs(GroupLanguage.CPP); + deserializeSketchStringsExact(GroupLanguage.CPP); + deserializeSketdhLongsSampling(GroupLanguage.CPP); + deserializeUnionDoubleSampling(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeSketchLongs(GroupLanguage.GO); + deserializeSketchStringsExact(GroupLanguage.GO); + deserializeSketdhLongsSampling(GroupLanguage.GO); + deserializeUnionDoubleSampling(GroupLanguage.GO); + } + + private static void deserializeSketchLongs(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "varopt_sketch_long_n" + n + "_cpp.sk"); + final String fileName = "varopt_sketch_long_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final VarOptItemsSketch sk = VarOptItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); assertEquals(sk.getK(), 32); assertEquals(sk.getN(), n); @@ -115,10 +143,12 @@ public void deserializeFromCppSketchLongs() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) @SuppressWarnings("unused") - public void deserializeFromCppSketchStringsExact() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "varopt_sketch_string_exact_cpp.sk"); + private static void deserializeSketchStringsExact(final GroupLanguage lang) { + final String fileName = "varopt_sketch_string_exact" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final VarOptItemsSketch sk = VarOptItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertEquals(sk.getK(), 1024); assertEquals(sk.getN(), 200); @@ -131,10 +161,12 @@ public void deserializeFromCppSketchStringsExact() throws IOException { assertEquals(ss.getTotalSketchWeight(), weight, EPS); } - @Test(groups = {CHECK_CPP_FILES}) @SuppressWarnings("unused") - public void deserializeFromCppSketchLongsSampling() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "varopt_sketch_long_sampling_cpp.sk"); + private static void deserializeSketdhLongsSampling(final GroupLanguage lang) { + String fileName = "varopt_sketch_long_sampling" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final VarOptItemsSketch sk = VarOptItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); assertEquals(sk.getK(), 1024); assertEquals(sk.getN(), 2003); @@ -149,9 +181,11 @@ public void deserializeFromCppSketchLongsSampling() throws IOException { assertEquals(ss.getEstimate(), 2000.0, EPS); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppUnionDoubleSampling() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "varopt_union_double_sampling_cpp.sk"); + private static void deserializeUnionDoubleSampling(final GroupLanguage lang) { + final String fileName = "varopt_union_double_sampling" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final VarOptItemsUnion u = VarOptItemsUnion.heapify(MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); // must reduce k in the process diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java index 99d88b429..5936f0565 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java @@ -19,81 +19,23 @@ package org.apache.datasketches.tdigest; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; public class TDigestCrossLanguageTest { - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppDouble() throws IOException { - final boolean[] with_buffer = {false, true}; - for (final boolean buffered : with_buffer) { - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (final int n : nArr) { - final byte[] bytes; - if (buffered) { - bytes = getFileBytes(cppPath, "tdigest_double_buf_n" + n + "_cpp.sk"); - } else { - bytes = getFileBytes(cppPath, "tdigest_double_n" + n + "_cpp.sk"); - } - final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); - assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); - assertEquals(td.getTotalWeight(), n); - if (n > 0) { - assertEquals(td.getMinValue(), 1); - assertEquals(td.getMaxValue(), n); - assertEquals(td.getRank(0), 0); - assertEquals(td.getRank(n + 1), 1); - if (n == 1) { - assertEquals(td.getRank(n), 0.5); - } else { - assertEquals(td.getRank(n / 2), 0.5, 0.05); - } - } - } - } - } - - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppFloat() throws IOException { - final boolean[] with_buffer = {false, true}; - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (final boolean buffered : with_buffer) { - for (final int n : nArr) { - final byte[] bytes; - if (buffered) { - bytes = getFileBytes(cppPath, "tdigest_float_buf_n" + n + "_cpp.sk"); - } else { - bytes = getFileBytes(cppPath, "tdigest_float_n" + n + "_cpp.sk"); - } - final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), true); - assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); - assertEquals(td.getTotalWeight(), n); - if (n > 0) { - assertEquals(td.getMinValue(), 1); - assertEquals(td.getMaxValue(), n); - assertEquals(td.getRank(0), 0); - assertEquals(td.getRank(n + 1), 1); - if (n == 1) { - assertEquals(td.getRank(n), 0.5); - } else { - assertEquals(td.getRank(n / 2), 0.5, 0.05); - } - } - } - } - } - @Test(groups = {GENERATE_JAVA_FILES}) public void generateForCppDouble() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; @@ -106,4 +48,49 @@ public void generateForCppDouble() throws IOException { } } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + deserializeTDigest(GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CPP_FILES}) + public void checkCpp() { + deserializeTDigest(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeTDigest(GroupLanguage.GO); + } + + private static void deserializeTDigest(final GroupLanguage lang) { + final String[] dfArr = {"double_", "float_"}; + final String[] bufArr = {"buf_", ""}; + final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; + for (final String df: dfArr) { + for (final String buf: bufArr) { + for (final int n : nArr) { + final String fileName = "tdigest_" + df + buf + "n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), df == "float_"); + assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); + assertEquals(td.getTotalWeight(), n); + if (n > 0) { + assertEquals(td.getMinValue(), 1); + assertEquals(td.getMaxValue(), n); + assertEquals(td.getRank(0), 0); + assertEquals(td.getRank(n + 1), 1); + if (n == 1) { + assertEquals(td.getRank(n), 0.5); + } else { + assertEquals(td.getRank(n / 2), 0.5, 0.05); + } + } + } + } + } + } + } diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java index 18e4103cf..5b565074b 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java @@ -19,7 +19,6 @@ package org.apache.datasketches.tdigest; -import static org.apache.datasketches.common.TestUtil.resPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; @@ -30,7 +29,7 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.common.TestUtil; +import org.apache.datasketches.common.UtilityIO; import org.testng.annotations.Test; public class TDigestDoubleTest { @@ -238,7 +237,7 @@ public void deserializeNaNSingleValue() { @Test public void deserializeFromReferenceImplementationDouble() { - final byte[] bytes = TestUtil.getFileBytes(resPath, "tdigest_ref_k100_n10000_double.sk"); + final byte[] bytes = UtilityIO.getTestResourceBytes("tdigest_ref_k100_n10000_double.sk"); final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); final int n = 10000; assertEquals(td.getK(), 100); @@ -254,7 +253,7 @@ public void deserializeFromReferenceImplementationDouble() { @Test public void deserializeFromReferenceImplementationFloat() { - final byte[] bytes = TestUtil.getFileBytes(resPath, "tdigest_ref_k100_n10000_float.sk"); + final byte[] bytes = UtilityIO.getTestResourceBytes("tdigest_ref_k100_n10000_float.sk"); final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); final int n = 10000; assertEquals(td.getK(), 100); diff --git a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java index c9cf0361b..a67c77131 100644 --- a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.theta; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -31,6 +32,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -40,7 +42,7 @@ public class ThetaSketchCrossLanguageTest { @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTesting() throws IOException { + public void serializeSketches() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { final UpdatableThetaSketch sk = UpdatableThetaSketch.builder().build(); @@ -52,7 +54,7 @@ public void generateBinariesForCompatibilityTesting() throws IOException { } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingCompressed() throws IOException { + public void serializeCompressedSketches() throws IOException { final int[] nArr = {10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { final UpdatableThetaSketch sk = UpdatableThetaSketch.builder().build(); @@ -64,19 +66,51 @@ public void generateBinariesForCompatibilityTestingCompressed() throws IOExcepti } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IOException { + public void serializeNonEmptyNoEntries() throws IOException { final UpdatableThetaSketch sk = UpdatableThetaSketch.builder().setP(0.01f).build(); - sk.update(1); + sk.update(1); //ignored because of p = .01. assertFalse(sk.isEmpty()); assertEquals(sk.getRetainedEntries(), 0); putBytesToJavaPath("theta_non_empty_no_entries_java.sk", sk.compact().toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + deserializeSketchesUsingSegment(GroupLanguage.JAVA); + deserializeSketchesFromFile(GroupLanguage.JAVA); + deserializeCompressedUsingSegment(GroupLanguage.JAVA); + deserializeCompressedFromFile(GroupLanguage.JAVA); + deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.JAVA); + deserializeNonEmptyNoEntriesFromFile(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppSegment() throws IOException { + public void checkCpp() { + deserializeSketchesUsingSegment(GroupLanguage.CPP); + deserializeSketchesFromFile(GroupLanguage.CPP); + deserializeCompressedUsingSegment(GroupLanguage.CPP); + deserializeCompressedFromFile(GroupLanguage.CPP); + deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.CPP); + deserializeNonEmptyNoEntriesFromFile(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeSketchesUsingSegment(GroupLanguage.GO); + deserializeSketchesFromFile(GroupLanguage.GO); + deserializeCompressedUsingSegment(GroupLanguage.GO); + deserializeCompressedFromFile(GroupLanguage.GO); + deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.GO); + deserializeNonEmptyNoEntriesFromFile(GroupLanguage.GO); + } + + private static void deserializeSketchesUsingSegment(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_n" + n + "_cpp.sk"); + final String fileName = "theta_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -91,11 +125,13 @@ public void deserializeFromCppSegment() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppBytes() throws IOException { + private static void deserializeSketchesFromFile(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_n" + n + "_cpp.sk"); + final String fileName = "theta_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(bytes); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -110,11 +146,13 @@ public void deserializeFromCppBytes() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppCompressedSegment() throws IOException { + private static void deserializeCompressedUsingSegment(final GroupLanguage lang) { final int[] nArr = {10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_compressed_n" + n + "_cpp.sk"); + final String fileName = "theta_compressed_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -129,11 +167,13 @@ public void deserializeFromCppCompressedSegment() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppCompressedBytes() throws IOException { + private static void deserializeCompressedFromFile(final GroupLanguage lang) { final int[] nArr = {10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_compressed_n" + n + "_cpp.sk"); + final String fileName = "theta_compressed_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(bytes); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -148,17 +188,21 @@ public void deserializeFromCppCompressedBytes() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppNonEmptyNoEntriesSegment() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "theta_non_empty_no_entries_cpp.sk"); + private static void deserializeNonEmptyNoEntriesUsingSegment(final GroupLanguage lang) { + final String fileName = "theta_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(MemorySegment.ofArray(bytes)); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppNonEmptyNoEntriesBytes() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "theta_non_empty_no_entries_cpp.sk"); + private static void deserializeNonEmptyNoEntriesFromFile(final GroupLanguage lang) { + final String fileName = "theta_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(bytes); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); diff --git a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java index 6c182baa2..031f00fbc 100644 --- a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java @@ -19,13 +19,13 @@ package org.apache.datasketches.tuple; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_HISTORICAL_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; -import static org.apache.datasketches.common.TestUtil.resPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_HISTORICAL_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -33,7 +33,8 @@ import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.common.TestUtil; +import org.apache.datasketches.common.UtilityIO; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.tuple.adouble.DoubleSummary; import org.apache.datasketches.tuple.adouble.DoubleSummaryDeserializer; import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUnion; @@ -42,9 +43,57 @@ public class TupleCrossLanguageTest { + @Test(groups = {GENERATE_JAVA_FILES}) + public void generateForCppIntegerSummary() throws IOException { + final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; + for (int n: nArr) { + final UpdatableTupleSketch sk = + new UpdatableTupleSketchBuilder<>(new IntegerSummaryFactory()).build(); + for (int i = 0; i < n; i++) { + sk.update(i, i); + } + putBytesToJavaPath("tuple_int_n" + n + "_java.sk", sk.compact().toByteArray()); + } + } + + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + deserializeTupleIntegerSummary(GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CPP_FILES}) + public void checkCpp() { + deserializeTupleIntegerSummary(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeTupleIntegerSummary(GroupLanguage.GO); + } + + private static void deserializeTupleIntegerSummary(final GroupLanguage lang) { + final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; + for (int n: nArr) { + final String fileName = "tuple_int_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final TupleSketch sketch = + TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer()); + assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); + assertTrue(n > 1000 ? sketch.isEstimationMode() : !sketch.isEstimationMode()); + assertEquals(sketch.getEstimate(), n, n * 0.03); + final TupleSketchIterator it = sketch.iterator(); + while (it.next()) { + assertTrue(it.getHash() < sketch.getThetaLong()); + assertTrue(it.getSummary().getValue() < n); + } + } + } + @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) public void serialVersion1Compatibility() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "CompactSketchWithDoubleSummary4K_serialVersion1.sk"); + final byte[] byteArr = UtilityIO.getTestResourceBytes("CompactSketchWithDoubleSummary4K_serialVersion1.sk"); TupleSketch sketch = TupleSketch.heapifySketch(MemorySegment.ofArray(byteArr), new DoubleSummaryDeserializer()); Assert.assertTrue(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 8192, 8192 * 0.99); @@ -60,7 +109,7 @@ public void serialVersion1Compatibility() { @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) public void version2Compatibility() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "TupleWithTestIntegerSummary4kTrimmedSerVer2.sk"); + final byte[] byteArr = UtilityIO.getTestResourceBytes("TupleWithTestIntegerSummary4kTrimmedSerVer2.sk"); TupleSketch sketch1 = TupleSketch.heapifySketch(MemorySegment.ofArray(byteArr), new IntegerSummaryDeserializer()); // construct the same way @@ -81,46 +130,15 @@ public void version2Compatibility() { Assert.assertEquals(sketch1.isEstimationMode(), sketch2.isEstimationMode()); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppIntegerSummary() throws IOException { - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "tuple_int_n" + n + "_cpp.sk"); - final TupleSketch sketch = - TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer()); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertTrue(n > 1000 ? sketch.isEstimationMode() : !sketch.isEstimationMode()); - assertEquals(sketch.getEstimate(), n, n * 0.03); - final TupleSketchIterator it = sketch.iterator(); - while (it.next()) { - assertTrue(it.getHash() < sketch.getThetaLong()); - assertTrue(it.getSummary().getValue() < n); - } - } - } - - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateForCppIntegerSummary() throws IOException { - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { - final UpdatableTupleSketch sk = - new UpdatableTupleSketchBuilder<>(new IntegerSummaryFactory()).build(); - for (int i = 0; i < n; i++) { - sk.update(i, i); - } - putBytesToJavaPath("tuple_int_n" + n + "_java.sk", sk.compact().toByteArray()); - } - } - @Test(expectedExceptions = SketchesArgumentException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) public void noSupportHeapifyV0_9_1() throws Exception { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "ArrayOfDoublesUnion_v0.9.1.sk"); + final byte[] byteArr = UtilityIO.getTestResourceBytes("ArrayOfDoublesUnion_v0.9.1.sk"); ArrayOfDoublesUnion.heapify(MemorySegment.ofArray(byteArr)); } @Test(expectedExceptions = SketchesArgumentException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) public void noSupportWrapV0_9_1() throws Exception { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "ArrayOfDoublesUnion_v0.9.1.sk"); + final byte[] byteArr = UtilityIO.getTestResourceBytes("ArrayOfDoublesUnion_v0.9.1.sk"); ArrayOfDoublesUnion.wrap(MemorySegment.ofArray(byteArr)); } diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java index 121b8f52d..6d71da39b 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.tuple.arrayofdoubles; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -31,6 +32,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -40,7 +42,7 @@ public class AodSketchCrossLanguageTest { @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingOneValue() throws IOException { + public void serializeOneValue() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n: nArr) { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().build(); @@ -52,7 +54,7 @@ public void generateBinariesForCompatibilityTestingOneValue() throws IOException } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingThreeValues() throws IOException { + public void serializeThreeValues() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n: nArr) { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(3).build(); @@ -64,7 +66,7 @@ public void generateBinariesForCompatibilityTestingThreeValues() throws IOExcept } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IOException { + public void serializeNonEmptyNoEntries() throws IOException { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().setSamplingProbability(0.01f).build(); sk.update(1, new double[] {1}); @@ -73,11 +75,34 @@ public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IO putBytesToJavaPath("aod_1_non_empty_no_entries_java.sk", sk.compact().toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + deserializeOneValue(GroupLanguage.JAVA); + deserializeThreeValues(GroupLanguage.JAVA); + deserializeOneValueNonEmptyNoEntries(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneValue() throws IOException { + public void checkCpp() { + deserializeOneValue(GroupLanguage.CPP); + deserializeThreeValues(GroupLanguage.CPP); + deserializeOneValueNonEmptyNoEntries(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeOneValue(GroupLanguage.GO); + deserializeThreeValues(GroupLanguage.GO); + deserializeOneValueNonEmptyNoEntries(GroupLanguage.GO); + } + + private static void deserializeOneValue(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "aod_1_n" + n + "_cpp.sk"); + final String fileName = "aod_1_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -89,11 +114,13 @@ public void deserializeFromCppOneValue() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppThreeValues() throws IOException { + private static void deserializeThreeValues(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "aod_3_n" + n + "_cpp.sk"); + final String fileName = "aod_3_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -107,9 +134,11 @@ public void deserializeFromCppThreeValues() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneValueNonEmptyNoEntries() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "aod_1_non_empty_no_entries_cpp.sk"); + private static void deserializeOneValueNonEmptyNoEntries(final GroupLanguage lang) { + final String fileName = "aod_1_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.err.println(fileName); final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); diff --git a/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java index 8a1bf4187..a5466dc67 100644 --- a/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java @@ -19,23 +19,25 @@ package org.apache.datasketches.tuple.strings; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.file.Files; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.datasketches.common.ResizeFactor; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.tuple.TupleSketch; import org.apache.datasketches.tuple.TupleSketchIterator; import org.testng.annotations.Test; @@ -47,7 +49,7 @@ public class AosSketchCrossLanguageTest { @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingOneString() throws IOException { + public void serializeOneString() throws IOException { int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); @@ -59,7 +61,7 @@ public void generateBinariesForCompatibilityTestingOneString() throws IOExceptio } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingThreeStrings() throws IOException { + public void serializeThreeStrings() throws IOException { int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); @@ -71,7 +73,7 @@ public void generateBinariesForCompatibilityTestingThreeStrings() throws IOExcep } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IOException { + public void serializeOneStringNonEmptyNoEntries() throws IOException { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(12, ResizeFactor.X8, 0.01f); sk.update(new String[] {"key1"}, new String[] {"value1"}); @@ -81,7 +83,7 @@ public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IO } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingMultiKeyStrings() throws IOException { + public void serializeMultiKeyStrings() throws IOException { int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); @@ -93,11 +95,15 @@ public void generateBinariesForCompatibilityTestingMultiKeyStrings() throws IOEx } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingUnicodeStrings() throws IOException { + public void serializeUnicodeStrings() throws IOException { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); sk.update(new String[]{"키", "열쇠"}, new String[]{"밸류", "값"}); - sk.update(new String[]{"🔑", "🗝️"}, new String[]{"📦", "🎁"}); + //These are emojis that are outside the Basic Multilingual Plane and explicitly coded here + // as 16-bit surrogate pairs to fix a bug in the TestNG Eclipse Plugin (7.11.0). + // These 4 emojis are the Unicode Code Points (in order): + //"Key", U+1F511; "Old Key", U+1F5DD,U+FE0F; "Package", U+1F4E6; "Gift", U+1F381. + sk.update(new String[]{"\uD83D\uDD11", "\uD83D\uDDDD\uFE0F"}, new String[]{"\uD83D\uDCE6", "\uD83C\uDF81"}); sk.update(new String[]{"ключ1", "ключ2"}, new String[]{"ценить1", "ценить2"}); assertFalse(sk.isEmpty()); @@ -107,7 +113,7 @@ public void generateBinariesForCompatibilityTestingUnicodeStrings() throws IOExc } @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingEmptyStrings() throws IOException { + public void serializeEmptyStrings() throws IOException { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); sk.update(new String[]{""}, new String[]{"empty_key_value"}); @@ -120,12 +126,45 @@ public void generateBinariesForCompatibilityTestingEmptyStrings() throws IOExcep putBytesToJavaPath("aos_empty_strings_java.sk", sk.compact().toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}) + public void checkJava() { + deserializeOneString(GroupLanguage.JAVA); + deserializeFromThreeStrings(GroupLanguage.JAVA); + deserializeOneStringNonEmptyNoEntries(GroupLanguage.JAVA); + deserializeMultiKeyStrings(GroupLanguage.JAVA); + deserializeUnicodeStrings(GroupLanguage.JAVA); + deserializeEmptyStrings(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneString() throws IOException { + public void checkCpp() { + deserializeOneString(GroupLanguage.CPP); + deserializeFromThreeStrings(GroupLanguage.CPP); + deserializeOneStringNonEmptyNoEntries(GroupLanguage.CPP); + deserializeMultiKeyStrings(GroupLanguage.CPP); + deserializeUnicodeStrings(GroupLanguage.CPP); + deserializeEmptyStrings(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeOneString(GroupLanguage.GO); + deserializeFromThreeStrings(GroupLanguage.GO); + deserializeOneStringNonEmptyNoEntries(GroupLanguage.GO); + deserializeMultiKeyStrings(GroupLanguage.GO); + deserializeUnicodeStrings(GroupLanguage.GO); + deserializeEmptyStrings(GroupLanguage.GO); + } + + private static void deserializeOneString(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_1_n" + n + "_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + final String fileName = "aos_1_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertTrue(n > 1000? sketch.isEstimationMode() : !sketch.isEstimationMode()); @@ -139,12 +178,15 @@ public void deserializeFromCppOneString() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppThreeStrings() throws IOException { + private static void deserializeFromThreeStrings(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_3_n" + n + "_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + final String fileName = "aos_3_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertTrue(n > 1000? sketch.isEstimationMode() : !sketch.isEstimationMode()); @@ -158,21 +200,26 @@ public void deserializeFromCppThreeStrings() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneStringNonEmptyNoEntries() throws IOException { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_1_non_empty_no_entries_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); - + private static void deserializeOneStringNonEmptyNoEntries(final GroupLanguage lang) { + final String fileName = "aos_1_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppMultiKeyStrings() throws IOException { + private static void deserializeMultiKeyStrings(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_multikey_n" + n + "_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + final String fileName = "aos_multikey_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertTrue(n > 1000? sketch.isEstimationMode() : !sketch.isEstimationMode()); @@ -186,10 +233,13 @@ public void deserializeFromCppMultiKeyStrings() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppUnicodeStrings() throws IOException { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_unicode_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + private static void deserializeUnicodeStrings(final GroupLanguage lang) { + final String fileName = "aos_unicode" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertFalse(sketch.isEmpty()); assertFalse(sketch.isEstimationMode()); assertEquals(sketch.getEstimate(), 3.0); @@ -200,10 +250,13 @@ public void deserializeFromCppUnicodeStrings() throws IOException { assertTrue(summaries.contains(Arrays.asList("ценить1", "ценить2"))); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppEmptyStrings() throws IOException { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_empty_strings_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + private static void deserializeEmptyStrings(final GroupLanguage lang) { + final String fileName = "aos_empty_strings" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertFalse(sketch.isEmpty()); assertFalse(sketch.isEstimationMode()); assertEquals(sketch.getEstimate(), 3.0); diff --git a/src/test/resources/testng.xml b/src/test/resources/testng.xml index a0e30835e..e1d59d21b 100644 --- a/src/test/resources/testng.xml +++ b/src/test/resources/testng.xml @@ -1,5 +1,5 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +