JMH improvements - faster build and async profiler

- Don't create uber jar for microbenchmarks
- Add async profiler to jmh tests
- Benchmark classes names validation
- Add jmh.args property to make it possible passing extra args to JMH
- Add missing test/anttasks to idea configuration

Patch by Jacek Lewandowski; reviewed by Branimir Lambov, Maxim Muzafarov, Stefan Miklosovic for CASSANDRA-18871
This commit is contained in:
Jacek Lewandowski 2023-09-20 11:44:41 +02:00
parent c6385ac3dd
commit 3658ba58c7
12 changed files with 340 additions and 106 deletions

136
.build/build-bench.xml Normal file
View File

@ -0,0 +1,136 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project basedir="." name="apache-cassandra-bench"
xmlns:if="ant:if" xmlns:unless="ant:unless">
<property name="async-profiler.version" value="2.9"/>
<condition property="async-profiler.suffix" value="linux-arm64">
<and>
<os arch="aarch64"/>
<isset property="isLinux"/>
</and>
</condition>
<condition property="async-profiler.suffix" value="linux-x64">
<and>
<os arch="amd64"/>
<isset property="isLinux"/>
</and>
</condition>
<condition property="async-profiler.suffix" value="macos">
<isset property="isMac"/>
</condition>
<property name="async-profiler.name" value="async-profiler-${async-profiler.version}-${async-profiler.suffix}"/>
<property name="async-profiler.base" value="${build.dir}/async-profiler"/>
<property name="test.profiler.opts.default" value="event=cpu;threads=true;output=flamegraph;simple=true;ann=true"/>
<property name="test.profiler.output" value="${build.test.dir}/profiler"/>
<target name="-microbench">
<jmh/>
</target>
<target name="-microbench-with-profiler" depends="-fetch-async-profiler">
<condition property="test.profiler.opts" value="${test.profiler.opts.default}" else="${profiler.opts}">
<or>
<not>
<isset property="profiler.opts"/>
</not>
<equals arg1="${profiler.opts}" arg2=""/>
</or>
</condition>
<property name="async-profiler.lib.path" value="${async-profiler.base}/build/libasyncProfiler.so"/>
<mkdir dir="${test.profiler.output}"/>
<jmh>
<extra-args>
<arg value="-prof"/>
<arg value="async:libPath=${async-profiler.lib.path};dir=${test.profiler.output};${test.profiler.opts}"/>
</extra-args>
</jmh>
</target>
<macrodef name="jmh">
<element name="extra-args" optional="true"/>
<sequential>
<java classname="org.openjdk.jmh.Main" fork="true" failonerror="true" >
<classpath>
<path refid="cassandra.classpath.test"/>
<pathelement location="${test.classes}"/>
<pathelement location="${test.conf}"/>
<pathelement location="${async-profiler.base}/lib/async-profiler.jar"/>
<pathelement location="${async-profiler.base}/lib/converter.jar"/>
</classpath>
<jvmarg line="${test-jvmargs}"/>
<jvmarg line="${java11-jvmargs}"/>
<arg value="-foe"/>
<arg value="true"/>
<arg value="-rf"/>
<arg value="json"/>
<arg value="-rff"/>
<arg value="${build.test.dir}/jmh-result.json"/>
<arg value="-v"/>
<arg value="EXTRA"/>
<arg line="${jmh.args}"/>
<extra-args/>
<!-- TODO https://issues.apache.org/jira/browse/CASSANDRA-18873 -->
<arg value="-e" if:blank="${benchmark.name}"/>
<arg value="ZeroCopyStreamingBench|MutationBench|FastThreadLocalBench" if:blank="${benchmark.name}"/>
<arg value=".*microbench.*${benchmark.name}"/>
</java>
</sequential>
</macrodef>
<target name="-fetch-async-profiler">
<mkdir dir="${tmp.dir}"/>
<mkdir dir="${async-profiler.base}"/>
<antcall target="-fetch-async-profiler-mac" if:true="${isMac}" inheritrefs="true"/>
<antcall target="-fetch-async-profiler-linux" if:true="${isLinux}" inheritrefs="true"/>
<move todir="${async-profiler.base}">
<fileset dir="${async-profiler.base}/${async-profiler.name}">
<include name="**"/>
</fileset>
</move>
<delete dir="${async-profiler.base}/${async-profiler.name}" includeemptydirs="true"/>
</target>
<target name="-fetch-async-profiler-linux">
<get src="https://github.com/async-profiler/async-profiler/releases/download/v${async-profiler.version}/${async-profiler.name}.tar.gz"
dest="${tmp.dir}/${async-profiler.name}.tar.gz" retries="3" httpusecaches="true" skipexisting="true"/>
<gunzip src="${tmp.dir}/${async-profiler.name}.tar.gz" dest="${tmp.dir}/"/>
<untar src="${tmp.dir}/${async-profiler.name}.tar" dest="${async-profiler.base}/"/>
<delete file="${tmp.dir}/${async-profiler.name}.tar"/>
</target>
<target name="-fetch-async-profiler-mac">
<get src="https://github.com/async-profiler/async-profiler/releases/download/v${async-profiler.version}/${async-profiler.name}.zip"
dest="${tmp.dir}/${async-profiler.name}.zip" retries="3" httpusecaches="true" skipexisting="true"/>
<unzip src="${tmp.dir}/${async-profiler.name}.zip" dest="${async-profiler.base}/"/>
</target>
</project>

View File

@ -1,4 +1,5 @@
4.0.12
* JMH improvements - faster build and async profiler (CASSANDRA-18871)
* Enable 3rd party JDK installations for Debian package (CASSANDRA-18844)
* Fix NTS log message when an unrecognized strategy option is passed (CASSANDRA-18679)
* Fix BulkLoader ignoring cipher suites options (CASSANDRA-18582)

121
build.xml
View File

@ -63,6 +63,7 @@
<property name="test.classlistfile" value="testlist.txt"/>
<property name="test.classlistprefix" value="unit"/>
<property name="benchmark.name" value=""/>
<property name="jmh.args" value=""/>
<property name="test.anttasks.src" value="${test.dir}/anttasks"/>
<property name="test.methods" value=""/>
<property name="test.unit.src" value="${test.dir}/unit"/>
@ -130,6 +131,18 @@
<property name="jacoco.finalexecfile" value="${jacoco.export.dir}/jacoco.exec" />
<property name="jacoco.version" value="0.8.6"/>
<condition property="isMac" value="true">
<os family="mac"/>
</condition>
<condition property="isLinux" value="true">
<and>
<os family="unix"/>
<not>
<os family="mac"/>
</not>
</and>
</condition>
<property name="byteman.version" value="4.0.6"/>
<property name="jamm.version" value="0.3.2"/>
<property name="ecj.version" value="4.6.1"/>
@ -586,8 +599,8 @@
<dependency groupId="net.bytebuddy" artifactId="byte-buddy" version="${bytebuddy.version}" />
<dependency groupId="net.bytebuddy" artifactId="byte-buddy-agent" version="${bytebuddy.version}" />
<dependency groupId="org.openjdk.jmh" artifactId="jmh-core" version="1.21" scope="test"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess" version="1.21" scope="test"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-core" version="1.37" scope="test"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess" version="1.37" scope="test"/>
<dependency groupId="org.apache.ant" artifactId="ant-junit" version="1.10.12" scope="test"/>
@ -1248,28 +1261,6 @@
</checksum>
</target>
<target name="build-jmh" depends="build-test, jar" description="Create JMH uber jar">
<jar jarfile="${build.test.dir}/deps.jar">
<zipgroupfileset dir="${test.lib}/jars">
<include name="*jmh*.jar"/>
<include name="jopt*.jar"/>
<include name="commons*.jar"/>
<include name="junit*.jar"/>
<include name="hamcrest*.jar"/>
</zipgroupfileset>
<zipgroupfileset dir="${build.lib}" includes="*.jar"/>
</jar>
<jar jarfile="${build.test.dir}/benchmarks.jar">
<manifest>
<attribute name="Main-Class" value="org.openjdk.jmh.Main"/>
</manifest>
<zipfileset src="${build.test.dir}/deps.jar" excludes="META-INF/*.SF" />
<fileset dir="${build.classes.main}"/>
<fileset dir="${test.classes}"/>
<fileset dir="${test.conf}" />
</jar>
</target>
<!-- Wrapper of build-test without dependencies, so both that target and its dependencies are skipped if the property
no-build-test is true. This is meant to be used to run tests without actually building them, provided that they have
been built before. All test targets depend on this, so one can run them using the no-build-test property.
@ -1316,7 +1307,7 @@
<src path="${test.distributed.src}"/>
</javac>
<checktestnameshelper/>
<antcall target="_check-test-names" inheritRefs="true"/>
<!-- Non-java resources needed by the test suite -->
<copy todir="${test.classes}">
@ -1324,17 +1315,41 @@
</copy>
</target>
<macrodef name="checktestnameshelper">
<sequential>
<taskdef name="test-name-check_" classname="org.apache.cassandra.anttasks.TestNameCheckTask" classpath="${test.classes}">
<classpath>
<path refid="cassandra.classpath.test"/>
<path location="${fqltool.build.classes}"/>
</classpath>
</taskdef>
<test-name-check_/>
</sequential>
</macrodef>
<macrodef name="check-test-names">
<attribute name="annotationName"/>
<attribute name="regex"/>
<attribute name="scanClassPath" default="${test.classes}:${fqltool.test.classes}:${stress.test.classes}"/>
<attribute name="packageName" default="org.apache.cassandra"/>
<attribute name="expand" default="true"/>
<attribute name="normalize" default="true"/>
<attribute name="verbose" default="false"/>
<sequential>
<java taskname="check-test-names" fork="true" failonerror="yes" classname="org.apache.cassandra.anttasks.TestNameCheckTask">
<classpath>
<path refid="cassandra.classpath.test"/>
<pathelement location="${fqltool.build.classes}"/>
<pathelement location="${stress.build.classes}"/>
<pathelement location="${test.classes}"/>
<pathelement location="${fqltool.test.classes}"/>
<pathelement location="${stress.test.classes}"/>
</classpath>
<jvmarg line="${java11-jvmargs}"/>
<jvmarg value="-DscanClassPath=@{scanClassPath}"/>
<jvmarg value="-DpackageName=@{packageName}"/>
<jvmarg value="-DannotationName=@{annotationName}"/>
<jvmarg value="-Dexpand=@{expand}"/>
<jvmarg value="-Dnormalize=@{normalize}"/>
<jvmarg value="-Dverbose=@{verbose}"/>
<jvmarg value="-Dregex=@{regex}"/>
</java>
</sequential>
</macrodef>
<target name="_check-test-names">
<check-test-names annotationName="org.junit.Test" regex=".*Test$"/>
<check-test-names annotationName="org.openjdk.jmh.annotations.Benchmark" regex=".*(Bench|_jmhType.*)$"/>
</target>
<!-- Run tests separately and report errors after and generate a junit report -->
<macrodef name="testhelper">
@ -1948,35 +1963,12 @@
</testmacro>
</target>
<!-- run microbenchmarks suite -->
<target name="microbench" depends="build-jmh">
<java classname="org.openjdk.jmh.Main"
fork="true"
failonerror="true">
<classpath>
<path refid="cassandra.classpath.test" />
<pathelement location="${test.classes}"/>
<pathelement location="${test.conf}"/>
<fileset dir="${test.lib}">
<include name="**/*.jar" />
</fileset>
</classpath>
<arg value="-foe"/>
<arg value="true"/>
<arg value="-rf"/>
<arg value="json"/>
<arg value="-rff"/>
<arg value="${build.test.dir}/jmh-result.json"/>
<arg value="-v"/>
<arg value="EXTRA"/>
<jvmarg line="${test-jvmargs}"/>
<jvmarg line="${java11-jvmargs}"/>
<target name="microbench" depends="jar">
<antcall target="-microbench" inheritrefs="true"/>
</target>
<!-- Broken: ZeroCopyStreamingBench,MutationBench,FastThreadLocalBench (FIXME) -->
<arg value="-e"/><arg value="ZeroCopyStreamingBench|MutationBench|FastThreadLocalBench"/>
<arg value=".*microbench.*${benchmark.name}"/>
</java>
<target name="microbench-with-profiler" depends="jar">
<antcall target="-microbench-with-profiler" inheritrefs="true"/>
</target>
<!-- run arbitrary mains in tests, for example to run the long running memory tests with lots of memory pressure
@ -2225,4 +2217,5 @@
<import file="${basedir}/.build/build-rat.xml"/>
<import file="${basedir}/.build/build-owasp.xml"/>
<import file="${basedir}/.build/build-cqlsh.xml"/>
<import file="${basedir}/.build/build-bench.xml"/>
</project>

View File

@ -36,6 +36,7 @@
<sourceFolder url="file://$MODULE_DIR$/test/microbench" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/burn" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/distributed" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/anttasks" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/resources" type="java-test-resource" />
<sourceFolder url="file://$MODULE_DIR$/test/conf" type="java-test-resource" />
<excludeFolder url="file://$MODULE_DIR$/.idea" />

View File

@ -17,69 +17,168 @@
*/
package org.apache.cassandra.anttasks;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
import org.junit.Test;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Task;
import org.reflections.Reflections;
import org.reflections.scanners.Scanners;
import org.reflections.util.ConfigurationBuilder;
import java.io.File;
import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static java.util.stream.Collectors.toList;
public class TestNameCheckTask extends Task
public class TestNameCheckTask
{
private static final Reflections reflections = new Reflections(new ConfigurationBuilder()
.forPackage("org.apache.cassandra")
.setScanners(Scanners.MethodsAnnotated, Scanners.SubTypes)
.setExpandSuperTypes(true)
.setParallel(true));
private String scanClassPath = "build/test/classes";
private String packageName = "org.apache.cassandra";
private String annotationName = Test.class.getName();
private boolean expand = true;
private boolean normalize = true;
private boolean verbose = false;
private String regex = ".*Test$";
public TestNameCheckTask()
{
}
@Override
public void execute() throws BuildException
public void setScanClassPath(String scanClassPath)
{
Set<Method> methodsAnnotatedWith = reflections.getMethodsAnnotatedWith(Test.class);
List<String> testFiles = methodsAnnotatedWith.stream().map(Method::getDeclaringClass).distinct()
.flatMap(TestNameCheckTask::expand)
.map(TestNameCheckTask::normalize)
.map(Class::getCanonicalName)
.filter(s -> !s.endsWith("Test"))
.distinct().sorted()
.collect(toList());
if (!testFiles.isEmpty())
throw new BuildException("Detected tests that have a bad naming convention. All tests have to end on 'Test': \n" + String.join("\n", testFiles));
this.scanClassPath = scanClassPath;
}
private static Class<?> normalize(Class<?> klass)
public void setPackageName(String packageName)
{
for (; klass.getEnclosingClass() != null; klass = klass.getEnclosingClass())
this.packageName = packageName;
}
public void setAnnotationName(String annotationName)
{
this.annotationName = annotationName;
}
public void setExpand(boolean expand)
{
this.expand = expand;
}
public void setNormalize(boolean normalize)
{
this.normalize = normalize;
}
public void setVerbose(boolean verbose)
{
this.verbose = verbose;
}
public void setRegex(String regex)
{
this.regex = regex;
}
public void execute()
{
List<URL> scanClassPathUrls = Arrays.stream(scanClassPath.split(File.pathSeparator)).map(Paths::get).map(path -> {
try
{
return path.toUri().toURL();
}
catch (MalformedURLException e)
{
throw new RuntimeException(e);
}
}).collect(toList());
Reflections reflections = new Reflections(new ConfigurationBuilder()
.forPackage(packageName)
.setScanners(Scanners.MethodsAnnotated, Scanners.SubTypes)
.setUrls(scanClassPathUrls)
.setExpandSuperTypes(true)
.setParallel(true));
Class<? extends Annotation> annotationClass;
try
{
annotationClass = (Class<? extends Annotation>) Class.forName(annotationName);
}
catch (ClassNotFoundException e)
{
throw new RuntimeException(e);
}
Set<Method> methodsAnnotatedWith = reflections.getMethodsAnnotatedWith(annotationClass);
Stream<? extends Class<?>> stream = methodsAnnotatedWith.stream().map(Method::getDeclaringClass).distinct();
if (expand)
stream = stream.flatMap(c -> expand(c, reflections));
if (normalize)
stream = stream.map(this::normalize);
Pattern pattern = Pattern.compile(regex);
Predicate<String> patternPredicate = s -> !pattern.matcher(s).matches();
List<String> classes = stream.map(Class::getCanonicalName)
.distinct()
.sorted()
.peek(verbose ? System.out::println : ignored -> {})
.filter(patternPredicate)
.collect(toList());
if (!classes.isEmpty())
throw new RuntimeException(String.format("Detected classes that have a bad naming convention. All classes from the following locations %s which have methods annotated with %s should have names that match %s: \n%s", scanClassPath, annotationName, regex, String.join("\n", classes)));
}
/**
* Get top outer class if it is an inner class
*/
private Class<?> normalize(Class<?> klass)
{
while (klass.getEnclosingClass() != null)
klass = klass.getEnclosingClass();
return klass;
}
private static Stream<Class<?>> expand(Class<?> klass)
/**
* Expand a class to all its subtypes. We need this because it possible that there is a top level class with
* annotated test methods and there are subclasses which modifies some configuration but do not introduce
* any additional test methods. In such case, those subclasses would not be included in the result of
* {@link Reflections#getMethodsAnnotatedWith(Class)}.
*/
private Stream<? extends Class<?>> expand(Class<?> klass, Reflections reflections)
{
Set<? extends Class<?>> subTypes = reflections.getSubTypesOf(klass);
if (subTypes == null || subTypes.isEmpty())
return Stream.of(klass);
Stream<Class<?>> subs = (Stream<Class<?>>) subTypes.stream();
Stream<? extends Class<?>> subs = subTypes.stream();
// assume we include if not abstract
if (!Modifier.isAbstract(klass.getModifiers()))
subs = Stream.concat(Stream.of(klass), subs);
return subs;
}
public static void main(String[] args)
{
TestNameCheckTask check = new TestNameCheckTask();
// checkstyle: suppress below 'blockSystemPropertyUsage'
Optional.ofNullable(System.getProperty("scanClassPath")).ifPresent(check::setScanClassPath);
Optional.ofNullable(System.getProperty("packageName")).ifPresent(check::setPackageName);
Optional.ofNullable(System.getProperty("annotationName")).ifPresent(check::setAnnotationName);
Optional.ofNullable(System.getProperty("regex")).ifPresent(check::setRegex);
Optional.ofNullable(System.getProperty("expand")).map(Boolean::parseBoolean).ifPresent(check::setExpand);
Optional.ofNullable(System.getProperty("normalize")).map(Boolean::parseBoolean).ifPresent(check::setNormalize);
Optional.ofNullable(System.getProperty("verbose")).map(Boolean::parseBoolean).ifPresent(check::setVerbose);
check.execute();
}
}

View File

@ -16,7 +16,7 @@
# limitations under the License.
jvmoptions_variant="-server"
export CASSANDRA_HOME=`dirname "$0"`/../../
export CASSANDRA_HOME="$(cd $(dirname "$0")/../../; pwd)"
. $CASSANDRA_HOME/bin/cassandra.in.sh
# Use JAVA_HOME if set, otherwise look for java in PATH
@ -127,11 +127,15 @@ cassandra_parms="$cassandra_parms -XX:+PreserveFramePointer"
# Create log directory, some tests require that
mkdir -p $CASSANDRA_HOME/logs
if [ ! -f $CASSANDRA_HOME/build/test/benchmarks.jar ] ; then
echo "$CASSANDRA_HOME/build/test/benchmarks.jar does not exist - execute 'ant build-jmh'"
if [ ! -f $CASSANDRA_HOME/build/apache-cassandra-*.jar ] ; then
echo "$CASSANDRA_HOME/build/apache-cassandra-*.jar does not exist - execute 'ant jar' first"
exit 1
fi
exec $NUMACTL "$JAVA" -cp "$CLASSPATH:$CASSANDRA_HOME/build/test/benchmarks.jar:$CASSANDRA_HOME/build/test/deps.jar" org.openjdk.jmh.Main -jvmArgs="$cassandra_parms $JVM_OPTS" $@
CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/test/conf/"
CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/build/test/classes/"
CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/build/test/lib/jars/*"
exec $NUMACTL "$JAVA" -cp "$CLASSPATH" org.openjdk.jmh.Main -jvmArgs="$cassandra_parms $JVM_OPTS" "$@"
# vi:ai sw=4 ts=4 tw=0 et

View File

@ -34,7 +34,7 @@ import org.openjdk.jmh.annotations.*;
@Fork(value = 1)
@Threads(1)
@State(Scope.Benchmark)
public class ReadWriteTest extends CQLTester
public class ReadWriteBench extends CQLTester
{
static String keyspace;
String table;

View File

@ -35,7 +35,7 @@ import java.util.concurrent.TimeUnit;
@Fork(value = 1, jvmArgsAppend = "-Xmx512M")
@Threads(1)
@State(Scope.Benchmark)
public class Sample
public class SampleBench
{
@Param({"65536"})
private int pageSize;

View File

@ -87,7 +87,7 @@ import org.openjdk.jmh.annotations.Warmup;
@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
@Fork(value = 1)
@Threads(1)
public class ZeroCopyStreamingBenchmark
public class ZeroCopyStreamingBench
{
static final int STREAM_SIZE = 50 * 1024 * 1024;

View File

@ -42,7 +42,7 @@ import org.openjdk.jmh.annotations.*;
@Fork(value = 1)
@Threads(1)
@State(Scope.Benchmark)
public abstract class ReadTest extends CQLTester
public abstract class ReadBenchBase extends CQLTester
{
static String keyspace;
String table;

View File

@ -21,7 +21,7 @@ package org.apache.cassandra.test.microbench.instance;
import org.openjdk.jmh.annotations.Benchmark;
public class ReadTestSmallPartitions extends ReadTest
public class ReadSmallPartitionsBench extends ReadBenchBase
{
String readStatement()
{

View File

@ -22,7 +22,7 @@ package org.apache.cassandra.test.microbench.instance;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Param;
public class ReadTestWidePartitions extends ReadTest
public class ReadWidePartitionsBench extends ReadBenchBase
{
@Param({"1000", "4"}) // wide and very wide partitions
int partitions = 4;