Skip to content
Snippets Groups Projects
Verified Commit d0c1be0c authored by akwizgran's avatar akwizgran
Browse files

Add tests to compare benchmarks.

parent 0b9894a0
No related branches found
No related tags found
No related merge requests found
package org.briarproject.bramble.db;
import org.briarproject.bramble.api.db.DatabaseConfig;
import org.briarproject.bramble.api.system.Clock;
import org.junit.Ignore;
import java.sql.Connection;
@Ignore
public class H2DatabasePerformanceSelfComparisonTest
extends JdbcDatabasePerformanceComparisonTest {
@Override
Database<Connection> createDatabase(boolean conditionA,
DatabaseConfig databaseConfig, Clock clock) {
return new H2Database(databaseConfig, clock);
}
@Override
protected String getTestName() {
return H2DatabasePerformanceSelfComparisonTest.class.getSimpleName();
}
}
......@@ -5,7 +5,8 @@ import org.briarproject.bramble.api.system.Clock;
import org.junit.Ignore;
@Ignore
public class H2DatabasePerformanceTest extends JdbcDatabasePerformanceTest {
public class H2DatabasePerformanceTest
extends JdbcSingleDatabasePerformanceTest {
@Override
protected String getTestName() {
......
package org.briarproject.bramble.db;
import org.briarproject.bramble.api.db.DatabaseConfig;
import org.briarproject.bramble.api.system.Clock;
import org.junit.Ignore;
import java.sql.Connection;
@Ignore
public class H2HyperSqlDatabasePerformanceComparisonTest
extends JdbcDatabasePerformanceComparisonTest {
@Override
Database<Connection> createDatabase(boolean conditionA,
DatabaseConfig databaseConfig, Clock clock) {
if (conditionA) return new H2Database(databaseConfig, clock);
else return new HyperSqlDatabase(databaseConfig, clock);
}
@Override
protected String getTestName() {
return H2HyperSqlDatabasePerformanceComparisonTest.class
.getSimpleName();
}
}
......@@ -6,7 +6,7 @@ import org.junit.Ignore;
@Ignore
public class HyperSqlDatabasePerformanceTest
extends JdbcDatabasePerformanceTest {
extends JdbcSingleDatabasePerformanceTest {
@Override
protected String getTestName() {
......
package org.briarproject.bramble.db;
import org.briarproject.bramble.api.db.DatabaseConfig;
import org.briarproject.bramble.api.db.DbException;
import org.briarproject.bramble.api.system.Clock;
import org.briarproject.bramble.system.SystemClock;
import org.briarproject.bramble.test.TestDatabaseConfig;
import org.briarproject.bramble.test.UTest;
import java.io.IOException;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
import static org.briarproject.bramble.test.TestUtils.getMean;
import static org.briarproject.bramble.test.TestUtils.getMedian;
import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
import static org.briarproject.bramble.test.UTest.Z_CRITICAL_0_01;
public abstract class JdbcDatabasePerformanceComparisonTest
extends JdbcDatabasePerformanceTest {
/**
* How many blocks of each condition to compare.
*/
private static final int COMPARISON_BLOCKS = 10;
abstract Database<Connection> createDatabase(boolean conditionA,
DatabaseConfig databaseConfig, Clock clock);
@Override
protected void benchmark(String name,
BenchmarkTask<Database<Connection>> task) throws Exception {
List<Double> aDurations = new ArrayList<>();
List<Double> bDurations = new ArrayList<>();
boolean aFirst = true;
for (int i = 0; i < COMPARISON_BLOCKS; i++) {
// Alternate between running the A and B benchmarks first
if (aFirst) {
aDurations.addAll(benchmark(true, task).durations);
bDurations.addAll(benchmark(false, task).durations);
} else {
bDurations.addAll(benchmark(false, task).durations);
aDurations.addAll(benchmark(true, task).durations);
}
aFirst = !aFirst;
}
// Compare the results using a small P value, which increases our
// chance of getting an inconclusive result, making this a conservative
// test for performance differences
UTest.Result comparison = UTest.test(aDurations, bDurations,
Z_CRITICAL_0_01);
writeResult(name, aDurations, bDurations, comparison);
}
private SteadyStateResult benchmark(boolean conditionA,
BenchmarkTask<Database<Connection>> task) throws Exception {
deleteTestDirectory(testDir);
Database<Connection> db = openDatabase(conditionA);
populateDatabase(db);
db.close();
db = openDatabase(conditionA);
// Measure blocks of iterations until we reach a steady state
SteadyStateResult result = measureSteadyState(db, task);
db.close();
return result;
}
private Database<Connection> openDatabase(boolean conditionA)
throws DbException {
Database<Connection> db = createDatabase(conditionA,
new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
db.open();
return db;
}
private void writeResult(String name, List<Double> aDurations,
List<Double> bDurations, UTest.Result comparison)
throws IOException {
String result = String.format("%s\t%,d\t%,d\t%,d\t%,d\t%,d\t%,d\t%s",
name, (long) getMean(aDurations), (long) getMedian(aDurations),
(long) getStandardDeviation(aDurations),
(long) getMean(bDurations), (long) getMedian(bDurations),
(long) getStandardDeviation(bDurations),
comparison.name());
writeResult(result);
}
}
......@@ -2,7 +2,6 @@ package org.briarproject.bramble.db;
import org.briarproject.bramble.api.contact.Contact;
import org.briarproject.bramble.api.contact.ContactId;
import org.briarproject.bramble.api.db.DatabaseConfig;
import org.briarproject.bramble.api.db.DbException;
import org.briarproject.bramble.api.db.Metadata;
import org.briarproject.bramble.api.identity.AuthorId;
......@@ -13,10 +12,7 @@ import org.briarproject.bramble.api.sync.GroupId;
import org.briarproject.bramble.api.sync.Message;
import org.briarproject.bramble.api.sync.MessageId;
import org.briarproject.bramble.api.sync.ValidationManager.State;
import org.briarproject.bramble.api.system.Clock;
import org.briarproject.bramble.system.SystemClock;
import org.briarproject.bramble.test.BrambleTestCase;
import org.briarproject.bramble.test.TestDatabaseConfig;
import org.briarproject.bramble.test.UTest;
import org.junit.After;
import org.junit.Before;
......@@ -24,6 +20,7 @@ import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.sql.Connection;
import java.util.ArrayList;
......@@ -39,12 +36,9 @@ import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
import static org.briarproject.bramble.test.TestUtils.getAuthor;
import static org.briarproject.bramble.test.TestUtils.getGroup;
import static org.briarproject.bramble.test.TestUtils.getLocalAuthor;
import static org.briarproject.bramble.test.TestUtils.getMean;
import static org.briarproject.bramble.test.TestUtils.getMedian;
import static org.briarproject.bramble.test.TestUtils.getMessage;
import static org.briarproject.bramble.test.TestUtils.getRandomBytes;
import static org.briarproject.bramble.test.TestUtils.getRandomId;
import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
import static org.briarproject.bramble.test.TestUtils.getTestDirectory;
import static org.briarproject.bramble.test.UTest.Result.INCONCLUSIVE;
import static org.briarproject.bramble.test.UTest.Z_CRITICAL_0_1;
......@@ -54,7 +48,7 @@ import static org.junit.Assert.assertTrue;
public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
private static final int ONE_MEGABYTE = 1024 * 1024;
private static final int MAX_SIZE = 100 * ONE_MEGABYTE;
static final int MAX_SIZE = 100 * ONE_MEGABYTE;
/**
* How many contacts to simulate.
......@@ -90,12 +84,19 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
private static final int OFFERED_MESSAGES_PER_CONTACT = 100;
/**
* How many times to run each benchmark while measuring.
* How many benchmark iterations to run in each block.
*/
private static final int MEASUREMENT_ITERATIONS = 100;
private static final int ITERATIONS_PER_BLOCK = 10;
private final File testDir = getTestDirectory();
private final Random random = new Random();
/**
* How many blocks must be similar before we conclude a steady state has
* been reached.
*/
private static final int STEADY_STATE_BLOCKS = 5;
protected final File testDir = getTestDirectory();
private final File resultsFile = new File(getTestName() + ".tsv");
protected final Random random = new Random();
private LocalAuthor localAuthor;
private List<ClientId> clientIds;
......@@ -108,8 +109,8 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
protected abstract String getTestName();
protected abstract JdbcDatabase createDatabase(DatabaseConfig config,
Clock clock);
protected abstract void benchmark(String name,
BenchmarkTask<Database<Connection>> task) throws Exception;
@Before
public void setUp() {
......@@ -522,48 +523,7 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
return list.get(random.nextInt(list.size()));
}
private void benchmark(String name,
BenchmarkTask<Database<Connection>> task) throws Exception {
deleteTestDirectory(testDir);
Database<Connection> db = openDatabase();
populateDatabase(db);
db.close();
db = openDatabase();
// Measure the first iteration
long start = System.nanoTime();
task.run(db);
long firstDuration = System.nanoTime() - start;
// Measure blocks of iterations until we reach a steady state
List<Double> oldDurations = measureBlock(db, task);
List<Double> durations = measureBlock(db, task);
int blocks = 2;
while (UTest.test(oldDurations, durations, Z_CRITICAL_0_1)
!= INCONCLUSIVE) {
oldDurations = durations;
durations = measureBlock(db, task);
blocks++;
}
db.close();
String result = String.format("%s\t%d\t%,d\t%,d\t%,d\t%,d", name,
blocks, firstDuration, (long) getMean(durations),
(long) getMedian(durations),
(long) getStandardDeviation(durations));
System.out.println(result);
File resultsFile = new File(getTestName() + ".tsv");
PrintWriter out =
new PrintWriter(new FileOutputStream(resultsFile, true), true);
out.println(new Date() + "\t" + result);
out.close();
}
private Database<Connection> openDatabase() throws DbException {
Database<Connection> db = createDatabase(
new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
db.open();
return db;
}
private void populateDatabase(Database<Connection> db) throws DbException {
void populateDatabase(Database<Connection> db) throws DbException {
localAuthor = getLocalAuthor();
clientIds = new ArrayList<>();
contacts = new ArrayList<>();
......@@ -640,17 +600,6 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
db.commitTransaction(txn);
}
private List<Double> measureBlock(Database<Connection> db,
BenchmarkTask<Database<Connection>> task) throws Exception {
List<Double> durations = new ArrayList<>(MEASUREMENT_ITERATIONS);
for (int i = 0; i < MEASUREMENT_ITERATIONS; i++) {
long start = System.nanoTime();
task.run(db);
durations.add((double) (System.nanoTime() - start));
}
return durations;
}
private ClientId getClientId() {
return new ClientId(getRandomString(CLIENT_ID_LENGTH));
}
......@@ -664,4 +613,56 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
}
return meta;
}
long measureOne(Database<Connection> db,
BenchmarkTask<Database<Connection>> task) throws Exception {
long start = System.nanoTime();
task.run(db);
return System.nanoTime() - start;
}
private List<Double> measureBlock(Database<Connection> db,
BenchmarkTask<Database<Connection>> task) throws Exception {
List<Double> durations = new ArrayList<>(ITERATIONS_PER_BLOCK);
for (int i = 0; i < ITERATIONS_PER_BLOCK; i++)
durations.add((double) measureOne(db, task));
return durations;
}
SteadyStateResult measureSteadyState(Database<Connection> db,
BenchmarkTask<Database<Connection>> task) throws Exception {
List<Double> durations = measureBlock(db, task);
int blocks = 1, steadyBlocks = 1;
while (steadyBlocks < STEADY_STATE_BLOCKS) {
List<Double> prev = durations;
durations = measureBlock(db, task);
// Compare to the previous block with a large P value, which
// decreases our chance of getting an inconclusive result, making
// this a conservative test for steady state
if (UTest.test(prev, durations, Z_CRITICAL_0_1) == INCONCLUSIVE)
steadyBlocks++;
else steadyBlocks = 1;
blocks++;
}
return new SteadyStateResult(blocks, durations);
}
void writeResult(String result) throws IOException {
System.out.println(result);
PrintWriter out =
new PrintWriter(new FileOutputStream(resultsFile, true), true);
out.println(new Date() + "\t" + result);
out.close();
}
static class SteadyStateResult {
final int blocks;
final List<Double> durations;
SteadyStateResult(int blocks, List<Double> durations) {
this.blocks = blocks;
this.durations = durations;
}
}
}
package org.briarproject.bramble.db;
import org.briarproject.bramble.api.db.DatabaseConfig;
import org.briarproject.bramble.api.db.DbException;
import org.briarproject.bramble.api.system.Clock;
import org.briarproject.bramble.system.SystemClock;
import org.briarproject.bramble.test.TestDatabaseConfig;
import java.io.IOException;
import java.sql.Connection;
import java.util.List;
import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
import static org.briarproject.bramble.test.TestUtils.getMean;
import static org.briarproject.bramble.test.TestUtils.getMedian;
import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
public abstract class JdbcSingleDatabasePerformanceTest
extends JdbcDatabasePerformanceTest {
abstract Database<Connection> createDatabase(DatabaseConfig databaseConfig,
Clock clock);
@Override
protected void benchmark(String name,
BenchmarkTask<Database<Connection>> task) throws Exception {
deleteTestDirectory(testDir);
Database<Connection> db = openDatabase();
populateDatabase(db);
db.close();
db = openDatabase();
// Measure the first iteration
long firstDuration = measureOne(db, task);
// Measure blocks of iterations until we reach a steady state
SteadyStateResult result = measureSteadyState(db, task);
db.close();
writeResult(name, result.blocks, firstDuration, result.durations);
}
private Database<Connection> openDatabase() throws DbException {
Database<Connection> db = createDatabase(
new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
db.open();
return db;
}
private void writeResult(String name, int blocks, long firstDuration,
List<Double> durations) throws IOException {
String result = String.format("%s\t%d\t%,d\t%,d\t%,d\t%,d", name,
blocks, firstDuration, (long) getMean(durations),
(long) getMedian(durations),
(long) getStandardDeviation(durations));
writeResult(result);
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment