diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceSelfComparisonTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceSelfComparisonTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..9e6f8c9ea754079be188132f97313a91740a8dba
--- /dev/null
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceSelfComparisonTest.java
@@ -0,0 +1,23 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.system.Clock;
+import org.junit.Ignore;
+
+import java.sql.Connection;
+
+@Ignore
+public class H2DatabasePerformanceSelfComparisonTest
+ extends JdbcDatabasePerformanceComparisonTest {
+
+ @Override
+ Database<Connection> createDatabase(boolean conditionA,
+ DatabaseConfig databaseConfig, Clock clock) {
+ return new H2Database(databaseConfig, clock);
+ }
+
+ @Override
+ protected String getTestName() {
+ return H2DatabasePerformanceSelfComparisonTest.class.getSimpleName();
+ }
+}
diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceTest.java
index 86d9b11d50675065b079991983e34fecbfa5194e..467bc92317fd6d54a603e6fdec009dc7acc0f718 100644
--- a/bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceTest.java
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceTest.java
@@ -5,7 +5,8 @@ import org.briarproject.bramble.api.system.Clock;
import org.junit.Ignore;
@Ignore
-public class H2DatabasePerformanceTest extends JdbcDatabasePerformanceTest {
+public class H2DatabasePerformanceTest
+ extends JdbcSingleDatabasePerformanceTest {
@Override
protected String getTestName() {
diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/H2HyperSqlDatabasePerformanceComparisonTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/H2HyperSqlDatabasePerformanceComparisonTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..5e112bb240027e28fdcafc46f9df5909cc1560bf
--- /dev/null
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/H2HyperSqlDatabasePerformanceComparisonTest.java
@@ -0,0 +1,25 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.system.Clock;
+import org.junit.Ignore;
+
+import java.sql.Connection;
+
+@Ignore
+public class H2HyperSqlDatabasePerformanceComparisonTest
+ extends JdbcDatabasePerformanceComparisonTest {
+
+ @Override
+ Database<Connection> createDatabase(boolean conditionA,
+ DatabaseConfig databaseConfig, Clock clock) {
+ if (conditionA) return new H2Database(databaseConfig, clock);
+ else return new HyperSqlDatabase(databaseConfig, clock);
+ }
+
+ @Override
+ protected String getTestName() {
+ return H2HyperSqlDatabasePerformanceComparisonTest.class
+ .getSimpleName();
+ }
+}
diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/HyperSqlDatabasePerformanceTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/HyperSqlDatabasePerformanceTest.java
index a1b48333c30b52a7968d3553a339b68d3cedb188..3e615e9b9e374231d776833ea432e7862078f2a9 100644
--- a/bramble-core/src/test/java/org/briarproject/bramble/db/HyperSqlDatabasePerformanceTest.java
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/HyperSqlDatabasePerformanceTest.java
@@ -6,7 +6,7 @@ import org.junit.Ignore;
@Ignore
public class HyperSqlDatabasePerformanceTest
- extends JdbcDatabasePerformanceTest {
+ extends JdbcSingleDatabasePerformanceTest {
@Override
protected String getTestName() {
diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceComparisonTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceComparisonTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..ea8710bafbe9789c79e66e984ca04c393f343ede
--- /dev/null
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceComparisonTest.java
@@ -0,0 +1,89 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.db.DbException;
+import org.briarproject.bramble.api.system.Clock;
+import org.briarproject.bramble.system.SystemClock;
+import org.briarproject.bramble.test.TestDatabaseConfig;
+import org.briarproject.bramble.test.UTest;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
+import static org.briarproject.bramble.test.TestUtils.getMean;
+import static org.briarproject.bramble.test.TestUtils.getMedian;
+import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
+import static org.briarproject.bramble.test.UTest.Z_CRITICAL_0_01;
+
+public abstract class JdbcDatabasePerformanceComparisonTest
+ extends JdbcDatabasePerformanceTest {
+
+ /**
+ * How many blocks of each condition to compare.
+ */
+ private static final int COMPARISON_BLOCKS = 10;
+
+ abstract Database<Connection> createDatabase(boolean conditionA,
+ DatabaseConfig databaseConfig, Clock clock);
+
+ @Override
+ protected void benchmark(String name,
+ BenchmarkTask<Database<Connection>> task) throws Exception {
+ List<Double> aDurations = new ArrayList<>();
+ List<Double> bDurations = new ArrayList<>();
+ boolean aFirst = true;
+ for (int i = 0; i < COMPARISON_BLOCKS; i++) {
+ // Alternate between running the A and B benchmarks first
+ if (aFirst) {
+ aDurations.addAll(benchmark(true, task).durations);
+ bDurations.addAll(benchmark(false, task).durations);
+ } else {
+ bDurations.addAll(benchmark(false, task).durations);
+ aDurations.addAll(benchmark(true, task).durations);
+ }
+ aFirst = !aFirst;
+ }
+ // Compare the results using a small P value, which increases our
+ // chance of getting an inconclusive result, making this a conservative
+ // test for performance differences
+ UTest.Result comparison = UTest.test(aDurations, bDurations,
+ Z_CRITICAL_0_01);
+ writeResult(name, aDurations, bDurations, comparison);
+ }
+
+ private SteadyStateResult benchmark(boolean conditionA,
+ BenchmarkTask<Database<Connection>> task) throws Exception {
+ deleteTestDirectory(testDir);
+ Database<Connection> db = openDatabase(conditionA);
+ populateDatabase(db);
+ db.close();
+ db = openDatabase(conditionA);
+ // Measure blocks of iterations until we reach a steady state
+ SteadyStateResult result = measureSteadyState(db, task);
+ db.close();
+ return result;
+ }
+
+ private Database<Connection> openDatabase(boolean conditionA)
+ throws DbException {
+ Database<Connection> db = createDatabase(conditionA,
+ new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
+ db.open();
+ return db;
+ }
+
+ private void writeResult(String name, List<Double> aDurations,
+ List<Double> bDurations, UTest.Result comparison)
+ throws IOException {
+ String result = String.format("%s\t%,d\t%,d\t%,d\t%,d\t%,d\t%,d\t%s",
+ name, (long) getMean(aDurations), (long) getMedian(aDurations),
+ (long) getStandardDeviation(aDurations),
+ (long) getMean(bDurations), (long) getMedian(bDurations),
+ (long) getStandardDeviation(bDurations),
+ comparison.name());
+ writeResult(result);
+ }
+}
diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceTest.java
index b7580853cc0b764e1ecfabd7797c0287dee8a355..ffb6ed965de6357ef9ef705ea6dde67b2ee48242 100644
--- a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceTest.java
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceTest.java
@@ -2,7 +2,6 @@ package org.briarproject.bramble.db;
import org.briarproject.bramble.api.contact.Contact;
import org.briarproject.bramble.api.contact.ContactId;
-import org.briarproject.bramble.api.db.DatabaseConfig;
import org.briarproject.bramble.api.db.DbException;
import org.briarproject.bramble.api.db.Metadata;
import org.briarproject.bramble.api.identity.AuthorId;
@@ -13,10 +12,7 @@ import org.briarproject.bramble.api.sync.GroupId;
import org.briarproject.bramble.api.sync.Message;
import org.briarproject.bramble.api.sync.MessageId;
import org.briarproject.bramble.api.sync.ValidationManager.State;
-import org.briarproject.bramble.api.system.Clock;
-import org.briarproject.bramble.system.SystemClock;
import org.briarproject.bramble.test.BrambleTestCase;
-import org.briarproject.bramble.test.TestDatabaseConfig;
import org.briarproject.bramble.test.UTest;
import org.junit.After;
import org.junit.Before;
@@ -24,6 +20,7 @@ import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.io.PrintWriter;
import java.sql.Connection;
import java.util.ArrayList;
@@ -39,12 +36,9 @@ import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
import static org.briarproject.bramble.test.TestUtils.getAuthor;
import static org.briarproject.bramble.test.TestUtils.getGroup;
import static org.briarproject.bramble.test.TestUtils.getLocalAuthor;
-import static org.briarproject.bramble.test.TestUtils.getMean;
-import static org.briarproject.bramble.test.TestUtils.getMedian;
import static org.briarproject.bramble.test.TestUtils.getMessage;
import static org.briarproject.bramble.test.TestUtils.getRandomBytes;
import static org.briarproject.bramble.test.TestUtils.getRandomId;
-import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
import static org.briarproject.bramble.test.TestUtils.getTestDirectory;
import static org.briarproject.bramble.test.UTest.Result.INCONCLUSIVE;
import static org.briarproject.bramble.test.UTest.Z_CRITICAL_0_1;
@@ -54,7 +48,7 @@ import static org.junit.Assert.assertTrue;
public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
private static final int ONE_MEGABYTE = 1024 * 1024;
- private static final int MAX_SIZE = 100 * ONE_MEGABYTE;
+ static final int MAX_SIZE = 100 * ONE_MEGABYTE;
/**
* How many contacts to simulate.
@@ -90,12 +84,19 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
private static final int OFFERED_MESSAGES_PER_CONTACT = 100;
/**
- * How many times to run each benchmark while measuring.
+ * How many benchmark iterations to run in each block.
*/
- private static final int MEASUREMENT_ITERATIONS = 100;
+ private static final int ITERATIONS_PER_BLOCK = 10;
- private final File testDir = getTestDirectory();
- private final Random random = new Random();
+ /**
+ * How many blocks must be similar before we conclude a steady state has
+ * been reached.
+ */
+ private static final int STEADY_STATE_BLOCKS = 5;
+
+ protected final File testDir = getTestDirectory();
+ private final File resultsFile = new File(getTestName() + ".tsv");
+ protected final Random random = new Random();
private LocalAuthor localAuthor;
private List<ClientId> clientIds;
@@ -108,8 +109,8 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
protected abstract String getTestName();
- protected abstract JdbcDatabase createDatabase(DatabaseConfig config,
- Clock clock);
+ protected abstract void benchmark(String name,
+ BenchmarkTask<Database<Connection>> task) throws Exception;
@Before
public void setUp() {
@@ -522,48 +523,7 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
return list.get(random.nextInt(list.size()));
}
- private void benchmark(String name,
- BenchmarkTask<Database<Connection>> task) throws Exception {
- deleteTestDirectory(testDir);
- Database<Connection> db = openDatabase();
- populateDatabase(db);
- db.close();
- db = openDatabase();
- // Measure the first iteration
- long start = System.nanoTime();
- task.run(db);
- long firstDuration = System.nanoTime() - start;
- // Measure blocks of iterations until we reach a steady state
- List<Double> oldDurations = measureBlock(db, task);
- List<Double> durations = measureBlock(db, task);
- int blocks = 2;
- while (UTest.test(oldDurations, durations, Z_CRITICAL_0_1)
- != INCONCLUSIVE) {
- oldDurations = durations;
- durations = measureBlock(db, task);
- blocks++;
- }
- db.close();
- String result = String.format("%s\t%d\t%,d\t%,d\t%,d\t%,d", name,
- blocks, firstDuration, (long) getMean(durations),
- (long) getMedian(durations),
- (long) getStandardDeviation(durations));
- System.out.println(result);
- File resultsFile = new File(getTestName() + ".tsv");
- PrintWriter out =
- new PrintWriter(new FileOutputStream(resultsFile, true), true);
- out.println(new Date() + "\t" + result);
- out.close();
- }
-
- private Database<Connection> openDatabase() throws DbException {
- Database<Connection> db = createDatabase(
- new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
- db.open();
- return db;
- }
-
- private void populateDatabase(Database<Connection> db) throws DbException {
+ void populateDatabase(Database<Connection> db) throws DbException {
localAuthor = getLocalAuthor();
clientIds = new ArrayList<>();
contacts = new ArrayList<>();
@@ -640,17 +600,6 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
db.commitTransaction(txn);
}
- private List<Double> measureBlock(Database<Connection> db,
- BenchmarkTask<Database<Connection>> task) throws Exception {
- List<Double> durations = new ArrayList<>(MEASUREMENT_ITERATIONS);
- for (int i = 0; i < MEASUREMENT_ITERATIONS; i++) {
- long start = System.nanoTime();
- task.run(db);
- durations.add((double) (System.nanoTime() - start));
- }
- return durations;
- }
-
private ClientId getClientId() {
return new ClientId(getRandomString(CLIENT_ID_LENGTH));
}
@@ -664,4 +613,56 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
}
return meta;
}
+
+ long measureOne(Database<Connection> db,
+ BenchmarkTask<Database<Connection>> task) throws Exception {
+ long start = System.nanoTime();
+ task.run(db);
+ return System.nanoTime() - start;
+ }
+
+ private List<Double> measureBlock(Database<Connection> db,
+ BenchmarkTask<Database<Connection>> task) throws Exception {
+ List<Double> durations = new ArrayList<>(ITERATIONS_PER_BLOCK);
+ for (int i = 0; i < ITERATIONS_PER_BLOCK; i++)
+ durations.add((double) measureOne(db, task));
+ return durations;
+ }
+
+ SteadyStateResult measureSteadyState(Database<Connection> db,
+ BenchmarkTask<Database<Connection>> task) throws Exception {
+ List<Double> durations = measureBlock(db, task);
+ int blocks = 1, steadyBlocks = 1;
+ while (steadyBlocks < STEADY_STATE_BLOCKS) {
+ List<Double> prev = durations;
+ durations = measureBlock(db, task);
+ // Compare to the previous block with a large P value, which
+ // decreases our chance of getting an inconclusive result, making
+ // this a conservative test for steady state
+ if (UTest.test(prev, durations, Z_CRITICAL_0_1) == INCONCLUSIVE)
+ steadyBlocks++;
+ else steadyBlocks = 1;
+ blocks++;
+ }
+ return new SteadyStateResult(blocks, durations);
+ }
+
+ void writeResult(String result) throws IOException {
+ System.out.println(result);
+ PrintWriter out =
+ new PrintWriter(new FileOutputStream(resultsFile, true), true);
+ out.println(new Date() + "\t" + result);
+ out.close();
+ }
+
+ static class SteadyStateResult {
+
+ final int blocks;
+ final List<Double> durations;
+
+ SteadyStateResult(int blocks, List<Double> durations) {
+ this.blocks = blocks;
+ this.durations = durations;
+ }
+ }
}
diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcSingleDatabasePerformanceTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcSingleDatabasePerformanceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..2356d9346a884e19e42766185b40f9f6dc62a11a
--- /dev/null
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcSingleDatabasePerformanceTest.java
@@ -0,0 +1,55 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.db.DbException;
+import org.briarproject.bramble.api.system.Clock;
+import org.briarproject.bramble.system.SystemClock;
+import org.briarproject.bramble.test.TestDatabaseConfig;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.util.List;
+
+import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
+import static org.briarproject.bramble.test.TestUtils.getMean;
+import static org.briarproject.bramble.test.TestUtils.getMedian;
+import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
+
+public abstract class JdbcSingleDatabasePerformanceTest
+ extends JdbcDatabasePerformanceTest {
+
+ abstract Database<Connection> createDatabase(DatabaseConfig databaseConfig,
+ Clock clock);
+
+ @Override
+ protected void benchmark(String name,
+ BenchmarkTask<Database<Connection>> task) throws Exception {
+ deleteTestDirectory(testDir);
+ Database<Connection> db = openDatabase();
+ populateDatabase(db);
+ db.close();
+ db = openDatabase();
+ // Measure the first iteration
+ long firstDuration = measureOne(db, task);
+ // Measure blocks of iterations until we reach a steady state
+ SteadyStateResult result = measureSteadyState(db, task);
+ db.close();
+ writeResult(name, result.blocks, firstDuration, result.durations);
+ }
+
+ private Database<Connection> openDatabase() throws DbException {
+ Database<Connection> db = createDatabase(
+ new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
+ db.open();
+ return db;
+ }
+
+ private void writeResult(String name, int blocks, long firstDuration,
+ List<Double> durations) throws IOException {
+ String result = String.format("%s\t%d\t%,d\t%,d\t%,d\t%,d", name,
+ blocks, firstDuration, (long) getMean(durations),
+ (long) getMedian(durations),
+ (long) getStandardDeviation(durations));
+ writeResult(result);
+ }
+}