From 47c91a96ae80ff5a118a7aec89b72bf10781988d Mon Sep 17 00:00:00 2001
From: akwizgran <michael@briarproject.org>
Date: Wed, 10 Oct 2018 13:40:36 +0100
Subject: [PATCH] Compact the database at startup.
---
.../briarproject/bramble/api/StringMap.java | 14 ++++
.../bramble/api/db/MigrationListener.java | 6 +-
.../api/lifecycle/LifecycleManager.java | 3 +-
.../bramble/test/SettableClock.java | 24 +++++++
.../bramble/db/DatabaseConstants.java | 14 ++++
.../briarproject/bramble/db/H2Database.java | 19 ++++++
.../bramble/db/HyperSqlDatabase.java | 26 ++++++-
.../briarproject/bramble/db/JdbcDatabase.java | 67 ++++++++++++++++---
.../lifecycle/LifecycleManagerImpl.java | 9 ++-
.../bramble/db/JdbcDatabaseTest.java | 17 ++---
.../android/login/OpenDatabaseActivity.java | 13 +++-
briar-android/src/main/res/values/strings.xml | 1 +
12 files changed, 191 insertions(+), 22 deletions(-)
create mode 100644 bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java
diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java b/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java
index aeb9823a17..bda2a94d24 100644
--- a/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java
+++ b/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java
@@ -38,4 +38,18 @@ public abstract class StringMap extends Hashtable<String, String> {
public void putInt(String key, int value) {
put(key, String.valueOf(value));
}
+
+ public long getLong(String key, long defaultValue) {
+ String s = get(key);
+ if (s == null) return defaultValue;
+ try {
+ return Long.valueOf(s);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ public void putLong(String key, long value) {
+ put(key, String.valueOf(value));
+ }
}
diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java b/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java
index 79e292a75f..92c9bf1638 100644
--- a/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java
+++ b/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java
@@ -6,6 +6,10 @@ public interface MigrationListener {
* This is called when a migration is started while opening the database.
* It will be called once for each migration being applied.
*/
- void onMigrationRun();
+ void onDatabaseMigration();
+ /**
+ * This is called when compaction is started while opening the database.
+ */
+ void onDatabaseCompaction();
}
diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java b/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java
index c44cf8879e..2ef340a872 100644
--- a/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java
+++ b/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java
@@ -34,7 +34,8 @@ public interface LifecycleManager {
*/
enum LifecycleState {
- STARTING, MIGRATING_DATABASE, STARTING_SERVICES, RUNNING, STOPPING;
+ STARTING, MIGRATING_DATABASE, COMPACTING_DATABASE, STARTING_SERVICES,
+ RUNNING, STOPPING;
public boolean isAfter(LifecycleState state) {
return ordinal() > state.ordinal();
diff --git a/bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java b/bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java
new file mode 100644
index 0000000000..26f885de80
--- /dev/null
+++ b/bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java
@@ -0,0 +1,24 @@
+package org.briarproject.bramble.test;
+
+import org.briarproject.bramble.api.system.Clock;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class SettableClock implements Clock {
+
+ private final AtomicLong time;
+
+ public SettableClock(AtomicLong time) {
+ this.time = time;
+ }
+
+ @Override
+ public long currentTimeMillis() {
+ return time.get();
+ }
+
+ @Override
+ public void sleep(long milliseconds) throws InterruptedException {
+ Thread.sleep(milliseconds);
+ }
+}
diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java b/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java
index 707234b1ff..f27d45cedd 100644
--- a/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java
+++ b/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java
@@ -2,6 +2,8 @@ package org.briarproject.bramble.db;
import org.briarproject.bramble.api.settings.Settings;
+import static java.util.concurrent.TimeUnit.DAYS;
+
interface DatabaseConstants {
/**
@@ -23,4 +25,16 @@ interface DatabaseConstants {
*/
String SCHEMA_VERSION_KEY = "schemaVersion";
+ /**
+ * The {@link Settings} key under which the time of the last database
+ * compaction is stored.
+ */
+ String LAST_COMPACTED_KEY = "lastCompacted";
+
+ /**
+ * The maximum time between database compactions in milliseconds. When the
+ * database is opened it will be compacted if more than this amount of time
+ * has passed since the last compaction.
+ */
+ long MAX_COMPACTION_INTERVAL_MS = DAYS.toMillis(30);
}
diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java b/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java
index 028312d81d..85bebb67c5 100644
--- a/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java
+++ b/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java
@@ -13,6 +13,7 @@ import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.Properties;
import javax.annotation.Nullable;
@@ -106,4 +107,22 @@ class H2Database extends JdbcDatabase {
String getUrl() {
return url;
}
+
+ @Override
+ protected void compactAndClose() throws DbException {
+ Connection c = null;
+ Statement s = null;
+ try {
+ c = createConnection();
+ closeAllConnections();
+ s = c.createStatement();
+ s.execute("SHUTDOWN COMPACT");
+ s.close();
+ c.close();
+ } catch (SQLException e) {
+ tryToClose(s);
+ tryToClose(c);
+ throw new DbException(e);
+ }
+ }
}
diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java b/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java
index f5419f7cb4..e8ba7b33dc 100644
--- a/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java
+++ b/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java
@@ -61,14 +61,18 @@ class HyperSqlDatabase extends JdbcDatabase {
@Override
public void close() throws DbException {
+ Connection c = null;
+ Statement s = null;
try {
super.closeAllConnections();
- Connection c = createConnection();
- Statement s = c.createStatement();
+ c = createConnection();
+ s = c.createStatement();
s.executeQuery("SHUTDOWN");
s.close();
c.close();
} catch (SQLException e) {
+ tryToClose(s);
+ tryToClose(c);
throw new DbException(e);
}
}
@@ -104,4 +108,22 @@ class HyperSqlDatabase extends JdbcDatabase {
String hex = StringUtils.toHexString(key.getBytes());
return DriverManager.getConnection(url + ";crypt_key=" + hex);
}
+
+ @Override
+ protected void compactAndClose() throws DbException {
+ Connection c = null;
+ Statement s = null;
+ try {
+ super.closeAllConnections();
+ c = createConnection();
+ s = c.createStatement();
+ s.executeQuery("SHUTDOWN COMPACT");
+ s.close();
+ c.close();
+ } catch (SQLException e) {
+ tryToClose(s);
+ tryToClose(c);
+ throw new DbException(e);
+ }
+ }
}
diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java b/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java
index 2396ee9077..307da37d7a 100644
--- a/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java
+++ b/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java
@@ -67,9 +67,13 @@ import static org.briarproject.bramble.api.sync.ValidationManager.State.DELIVERE
import static org.briarproject.bramble.api.sync.ValidationManager.State.PENDING;
import static org.briarproject.bramble.api.sync.ValidationManager.State.UNKNOWN;
import static org.briarproject.bramble.db.DatabaseConstants.DB_SETTINGS_NAMESPACE;
+import static org.briarproject.bramble.db.DatabaseConstants.LAST_COMPACTED_KEY;
+import static org.briarproject.bramble.db.DatabaseConstants.MAX_COMPACTION_INTERVAL_MS;
import static org.briarproject.bramble.db.DatabaseConstants.SCHEMA_VERSION_KEY;
import static org.briarproject.bramble.db.ExponentialBackoff.calculateExpiry;
+import static org.briarproject.bramble.util.LogUtils.logDuration;
import static org.briarproject.bramble.util.LogUtils.logException;
+import static org.briarproject.bramble.util.LogUtils.now;
/**
* A generic database implementation that can be used with any JDBC-compatible
@@ -317,9 +321,10 @@ abstract class JdbcDatabase implements Database<Connection> {
private int openConnections = 0; // Locking: connectionsLock
private boolean closed = false; // Locking: connectionsLock
- @Nullable
protected abstract Connection createConnection() throws SQLException;
+ protected abstract void compactAndClose() throws DbException;
+
private final Lock connectionsLock = new ReentrantLock();
private final Condition connectionsChanged = connectionsLock.newCondition();
@@ -344,13 +349,21 @@ abstract class JdbcDatabase implements Database<Connection> {
throw new DbException(e);
}
// Open the database and create the tables and indexes if necessary
+ boolean compact;
Connection txn = startTransaction();
try {
if (reopen) {
- checkSchemaVersion(txn, listener);
+ Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE);
+ checkSchemaVersion(txn, s, listener);
+ long lastCompacted = s.getLong(LAST_COMPACTED_KEY, 0);
+ long elapsed = clock.currentTimeMillis() - lastCompacted;
+ if (LOG.isLoggable(INFO))
+ LOG.info(elapsed + " ms since last compaction");
+ compact = elapsed > MAX_COMPACTION_INTERVAL_MS;
} else {
createTables(txn);
- storeSchemaVersion(txn, CODE_SCHEMA_VERSION);
+ initialiseSettings(txn);
+ compact = false;
}
createIndexes(txn);
commitTransaction(txn);
@@ -358,6 +371,25 @@ abstract class JdbcDatabase implements Database<Connection> {
abortTransaction(txn);
throw e;
}
+ // Compact the database if necessary
+ if (compact) {
+ if (listener != null) listener.onDatabaseCompaction();
+ long start = now();
+ compactAndClose();
+ logDuration(LOG, "Compacting database", start);
+ // Allow the next transaction to reopen the DB
+ synchronized (connectionsLock) {
+ closed = false;
+ }
+ txn = startTransaction();
+ try {
+ storeLastCompacted(txn);
+ commitTransaction(txn);
+ } catch (DbException e) {
+ abortTransaction(txn);
+ throw e;
+ }
+ }
}
/**
@@ -370,9 +402,8 @@ abstract class JdbcDatabase implements Database<Connection> {
* @throws DataTooOldException if the data uses an older schema than the
* current code and cannot be migrated
*/
- private void checkSchemaVersion(Connection txn,
+ private void checkSchemaVersion(Connection txn, Settings s,
@Nullable MigrationListener listener) throws DbException {
- Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE);
int dataSchemaVersion = s.getInt(SCHEMA_VERSION_KEY, -1);
if (dataSchemaVersion == -1) throw new DbException();
if (dataSchemaVersion == CODE_SCHEMA_VERSION) return;
@@ -384,7 +415,7 @@ abstract class JdbcDatabase implements Database<Connection> {
if (start == dataSchemaVersion) {
if (LOG.isLoggable(INFO))
LOG.info("Migrating from schema " + start + " to " + end);
- if (listener != null) listener.onMigrationRun();
+ if (listener != null) listener.onDatabaseMigration();
// Apply the migration
m.migrate(txn);
// Store the new schema version
@@ -408,6 +439,19 @@ abstract class JdbcDatabase implements Database<Connection> {
mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
}
+ private void storeLastCompacted(Connection txn) throws DbException {
+ Settings s = new Settings();
+ s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis());
+ mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
+ }
+
+ private void initialiseSettings(Connection txn) throws DbException {
+ Settings s = new Settings();
+ s.putInt(SCHEMA_VERSION_KEY, CODE_SCHEMA_VERSION);
+ s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis());
+ mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
+ }
+
private void tryToClose(@Nullable ResultSet rs) {
try {
if (rs != null) rs.close();
@@ -416,7 +460,7 @@ abstract class JdbcDatabase implements Database<Connection> {
}
}
- private void tryToClose(@Nullable Statement s) {
+ protected void tryToClose(@Nullable Statement s) {
try {
if (s != null) s.close();
} catch (SQLException e) {
@@ -424,6 +468,14 @@ abstract class JdbcDatabase implements Database<Connection> {
}
}
+ protected void tryToClose(@Nullable Connection c) {
+ try {
+ if (c != null) c.close();
+ } catch (SQLException e) {
+ logException(LOG, WARNING, e);
+ }
+ }
+
private void createTables(Connection txn) throws DbException {
Statement s = null;
try {
@@ -489,7 +541,6 @@ abstract class JdbcDatabase implements Database<Connection> {
if (txn == null) {
// Open a new connection
txn = createConnection();
- if (txn == null) throw new DbException();
txn.setAutoCommit(false);
connectionsLock.lock();
try {
diff --git a/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java b/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java
index 130595ab63..f158a690f0 100644
--- a/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java
+++ b/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java
@@ -29,6 +29,7 @@ import javax.inject.Inject;
import static java.util.logging.Level.FINE;
import static java.util.logging.Level.INFO;
import static java.util.logging.Level.WARNING;
+import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.RUNNING;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING;
@@ -159,11 +160,17 @@ class LifecycleManagerImpl implements LifecycleManager, MigrationListener {
}
@Override
- public void onMigrationRun() {
+ public void onDatabaseMigration() {
state = MIGRATING_DATABASE;
eventBus.broadcast(new LifecycleEvent(MIGRATING_DATABASE));
}
+ @Override
+ public void onDatabaseCompaction() {
+ state = COMPACTING_DATABASE;
+ eventBus.broadcast(new LifecycleEvent(COMPACTING_DATABASE));
+ }
+
@Override
public void stopServices() {
try {
diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java
index 9e6a0d2731..94eb4d1969 100644
--- a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java
+++ b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java
@@ -26,8 +26,8 @@ import org.briarproject.bramble.api.transport.KeySetId;
import org.briarproject.bramble.api.transport.OutgoingKeys;
import org.briarproject.bramble.api.transport.TransportKeys;
import org.briarproject.bramble.system.SystemClock;
-import org.briarproject.bramble.test.ArrayClock;
import org.briarproject.bramble.test.BrambleTestCase;
+import org.briarproject.bramble.test.SettableClock;
import org.briarproject.bramble.test.TestDatabaseConfig;
import org.briarproject.bramble.test.TestMessageFactory;
import org.briarproject.bramble.test.TestUtils;
@@ -46,6 +46,7 @@ import java.util.Map.Entry;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
@@ -1818,10 +1819,9 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
@Test
public void testMessageRetransmission() throws Exception {
long now = System.currentTimeMillis();
- long steps[] = {now, now, now + MAX_LATENCY * 2 - 1,
- now + MAX_LATENCY * 2};
+ AtomicLong time = new AtomicLong(now);
Database<Connection> db =
- open(false, new TestMessageFactory(), new ArrayClock(steps));
+ open(false, new TestMessageFactory(), new SettableClock(time));
Connection txn = db.startTransaction();
// Add a contact, a shared group and a shared message
@@ -1847,11 +1847,13 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
// Time: now + MAX_LATENCY * 2 - 1
// The message should not yet be sendable
+ time.set(now + MAX_LATENCY * 2 - 1);
ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY);
assertTrue(ids.isEmpty());
// Time: now + MAX_LATENCY * 2
// The message should have expired and should now be sendable
+ time.set(now + MAX_LATENCY * 2);
ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY);
assertEquals(singletonList(messageId), ids);
@@ -1859,13 +1861,12 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
db.close();
}
-
@Test
public void testFasterMessageRetransmission() throws Exception {
long now = System.currentTimeMillis();
- long steps[] = {now, now, now, now, now + 1};
+ AtomicLong time = new AtomicLong(now);
Database<Connection> db =
- open(false, new TestMessageFactory(), new ArrayClock(steps));
+ open(false, new TestMessageFactory(), new SettableClock(time));
Connection txn = db.startTransaction();
// Add a contact, a shared group and a shared message
@@ -1903,6 +1904,7 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
// Time: now + 1
// The message should no longer be sendable via the faster transport,
// as the ETA is now equal
+ time.set(now + 1);
ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE,
MAX_LATENCY - 1);
assertTrue(ids.isEmpty());
@@ -1911,7 +1913,6 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
db.close();
}
-
private Database<Connection> open(boolean resume) throws Exception {
return open(resume, new TestMessageFactory(), new SystemClock());
}
diff --git a/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java b/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java
index d24196d32b..24ccc1e010 100644
--- a/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java
+++ b/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java
@@ -20,6 +20,7 @@ import org.briarproject.briar.android.navdrawer.NavDrawerActivity;
import javax.annotation.ParametersAreNonnullByDefault;
import javax.inject.Inject;
+import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING_SERVICES;
@@ -34,7 +35,7 @@ public class OpenDatabaseActivity extends BriarActivity
private TextView textView;
private ImageView imageView;
- private boolean showingMigration = false;
+ private boolean showingMigration = false, showingCompaction = false;
@Override
public void onCreate(@Nullable Bundle state) {
@@ -57,6 +58,7 @@ public class OpenDatabaseActivity extends BriarActivity
finishAndStartApp();
} else {
if (state == MIGRATING_DATABASE) showMigration();
+ else if (state == COMPACTING_DATABASE) showCompaction();
eventBus.addListener(this);
}
}
@@ -75,6 +77,8 @@ public class OpenDatabaseActivity extends BriarActivity
runOnUiThreadUnlessDestroyed(this::finishAndStartApp);
else if (state == MIGRATING_DATABASE)
runOnUiThreadUnlessDestroyed(this::showMigration);
+ else if (state == COMPACTING_DATABASE)
+ runOnUiThreadUnlessDestroyed(this::showCompaction);
}
}
@@ -85,6 +89,13 @@ public class OpenDatabaseActivity extends BriarActivity
showingMigration = true;
}
+ private void showCompaction() {
+ if (showingCompaction) return;
+ textView.setText(R.string.startup_compact_database);
+ imageView.setImageResource(R.drawable.startup_migration);
+ showingCompaction = true;
+ }
+
private void finishAndStartApp() {
startActivity(new Intent(this, NavDrawerActivity.class));
supportFinishAfterTransition();
diff --git a/briar-android/src/main/res/values/strings.xml b/briar-android/src/main/res/values/strings.xml
index 08afd7f570..4875cc50c4 100644
--- a/briar-android/src/main/res/values/strings.xml
+++ b/briar-android/src/main/res/values/strings.xml
@@ -54,6 +54,7 @@
<string name="download_briar_button">Download Briar 1.0</string>
<string name="startup_open_database">Decrypting Database…</string>
<string name="startup_migrate_database">Upgrading Database…</string>
+ <string name="startup_compact_database">Compacting Database…</string>
<!-- Navigation Drawer -->
<string name="nav_drawer_open_description">Open the navigation drawer</string>
--
GitLab