From 47c91a96ae80ff5a118a7aec89b72bf10781988d Mon Sep 17 00:00:00 2001 From: akwizgran Date: Wed, 10 Oct 2018 13:40:36 +0100 Subject: [PATCH] Compact the database at startup. --- .../briarproject/bramble/api/StringMap.java | 14 ++++ .../bramble/api/db/MigrationListener.java | 6 +- .../api/lifecycle/LifecycleManager.java | 3 +- .../bramble/test/SettableClock.java | 24 +++++++ .../bramble/db/DatabaseConstants.java | 14 ++++ .../briarproject/bramble/db/H2Database.java | 19 ++++++ .../bramble/db/HyperSqlDatabase.java | 26 ++++++- .../briarproject/bramble/db/JdbcDatabase.java | 67 ++++++++++++++++--- .../lifecycle/LifecycleManagerImpl.java | 9 ++- .../bramble/db/JdbcDatabaseTest.java | 17 ++--- .../android/login/OpenDatabaseActivity.java | 13 +++- briar-android/src/main/res/values/strings.xml | 1 + 12 files changed, 191 insertions(+), 22 deletions(-) create mode 100644 bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java b/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java index aeb9823a1..bda2a94d2 100644 --- a/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java +++ b/bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java @@ -38,4 +38,18 @@ public abstract class StringMap extends Hashtable { public void putInt(String key, int value) { put(key, String.valueOf(value)); } + + public long getLong(String key, long defaultValue) { + String s = get(key); + if (s == null) return defaultValue; + try { + return Long.valueOf(s); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public void putLong(String key, long value) { + put(key, String.valueOf(value)); + } } diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java b/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java index 79e292a75..92c9bf163 100644 --- a/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java +++ b/bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java @@ -6,6 +6,10 @@ public interface MigrationListener { * This is called when a migration is started while opening the database. * It will be called once for each migration being applied. */ - void onMigrationRun(); + void onDatabaseMigration(); + /** + * This is called when compaction is started while opening the database. + */ + void onDatabaseCompaction(); } diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java b/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java index c44cf8879..2ef340a87 100644 --- a/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java +++ b/bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java @@ -34,7 +34,8 @@ public interface LifecycleManager { */ enum LifecycleState { - STARTING, MIGRATING_DATABASE, STARTING_SERVICES, RUNNING, STOPPING; + STARTING, MIGRATING_DATABASE, COMPACTING_DATABASE, STARTING_SERVICES, + RUNNING, STOPPING; public boolean isAfter(LifecycleState state) { return ordinal() > state.ordinal(); diff --git a/bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java b/bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java new file mode 100644 index 000000000..26f885de8 --- /dev/null +++ b/bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java @@ -0,0 +1,24 @@ +package org.briarproject.bramble.test; + +import org.briarproject.bramble.api.system.Clock; + +import java.util.concurrent.atomic.AtomicLong; + +public class SettableClock implements Clock { + + private final AtomicLong time; + + public SettableClock(AtomicLong time) { + this.time = time; + } + + @Override + public long currentTimeMillis() { + return time.get(); + } + + @Override + public void sleep(long milliseconds) throws InterruptedException { + Thread.sleep(milliseconds); + } +} diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java b/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java index 707234b1f..f27d45ced 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java @@ -2,6 +2,8 @@ package org.briarproject.bramble.db; import org.briarproject.bramble.api.settings.Settings; +import static java.util.concurrent.TimeUnit.DAYS; + interface DatabaseConstants { /** @@ -23,4 +25,16 @@ interface DatabaseConstants { */ String SCHEMA_VERSION_KEY = "schemaVersion"; + /** + * The {@link Settings} key under which the time of the last database + * compaction is stored. + */ + String LAST_COMPACTED_KEY = "lastCompacted"; + + /** + * The maximum time between database compactions in milliseconds. When the + * database is opened it will be compacted if more than this amount of time + * has passed since the last compaction. + */ + long MAX_COMPACTION_INTERVAL_MS = DAYS.toMillis(30); } diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java b/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java index 028312d81..85bebb67c 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java @@ -13,6 +13,7 @@ import java.io.File; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; import java.util.Properties; import javax.annotation.Nullable; @@ -106,4 +107,22 @@ class H2Database extends JdbcDatabase { String getUrl() { return url; } + + @Override + protected void compactAndClose() throws DbException { + Connection c = null; + Statement s = null; + try { + c = createConnection(); + closeAllConnections(); + s = c.createStatement(); + s.execute("SHUTDOWN COMPACT"); + s.close(); + c.close(); + } catch (SQLException e) { + tryToClose(s); + tryToClose(c); + throw new DbException(e); + } + } } diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java b/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java index f5419f7cb..e8ba7b33d 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java @@ -61,14 +61,18 @@ class HyperSqlDatabase extends JdbcDatabase { @Override public void close() throws DbException { + Connection c = null; + Statement s = null; try { super.closeAllConnections(); - Connection c = createConnection(); - Statement s = c.createStatement(); + c = createConnection(); + s = c.createStatement(); s.executeQuery("SHUTDOWN"); s.close(); c.close(); } catch (SQLException e) { + tryToClose(s); + tryToClose(c); throw new DbException(e); } } @@ -104,4 +108,22 @@ class HyperSqlDatabase extends JdbcDatabase { String hex = StringUtils.toHexString(key.getBytes()); return DriverManager.getConnection(url + ";crypt_key=" + hex); } + + @Override + protected void compactAndClose() throws DbException { + Connection c = null; + Statement s = null; + try { + super.closeAllConnections(); + c = createConnection(); + s = c.createStatement(); + s.executeQuery("SHUTDOWN COMPACT"); + s.close(); + c.close(); + } catch (SQLException e) { + tryToClose(s); + tryToClose(c); + throw new DbException(e); + } + } } diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java b/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java index 2396ee907..307da37d7 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java @@ -67,9 +67,13 @@ import static org.briarproject.bramble.api.sync.ValidationManager.State.DELIVERE import static org.briarproject.bramble.api.sync.ValidationManager.State.PENDING; import static org.briarproject.bramble.api.sync.ValidationManager.State.UNKNOWN; import static org.briarproject.bramble.db.DatabaseConstants.DB_SETTINGS_NAMESPACE; +import static org.briarproject.bramble.db.DatabaseConstants.LAST_COMPACTED_KEY; +import static org.briarproject.bramble.db.DatabaseConstants.MAX_COMPACTION_INTERVAL_MS; import static org.briarproject.bramble.db.DatabaseConstants.SCHEMA_VERSION_KEY; import static org.briarproject.bramble.db.ExponentialBackoff.calculateExpiry; +import static org.briarproject.bramble.util.LogUtils.logDuration; import static org.briarproject.bramble.util.LogUtils.logException; +import static org.briarproject.bramble.util.LogUtils.now; /** * A generic database implementation that can be used with any JDBC-compatible @@ -317,9 +321,10 @@ abstract class JdbcDatabase implements Database { private int openConnections = 0; // Locking: connectionsLock private boolean closed = false; // Locking: connectionsLock - @Nullable protected abstract Connection createConnection() throws SQLException; + protected abstract void compactAndClose() throws DbException; + private final Lock connectionsLock = new ReentrantLock(); private final Condition connectionsChanged = connectionsLock.newCondition(); @@ -344,13 +349,21 @@ abstract class JdbcDatabase implements Database { throw new DbException(e); } // Open the database and create the tables and indexes if necessary + boolean compact; Connection txn = startTransaction(); try { if (reopen) { - checkSchemaVersion(txn, listener); + Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE); + checkSchemaVersion(txn, s, listener); + long lastCompacted = s.getLong(LAST_COMPACTED_KEY, 0); + long elapsed = clock.currentTimeMillis() - lastCompacted; + if (LOG.isLoggable(INFO)) + LOG.info(elapsed + " ms since last compaction"); + compact = elapsed > MAX_COMPACTION_INTERVAL_MS; } else { createTables(txn); - storeSchemaVersion(txn, CODE_SCHEMA_VERSION); + initialiseSettings(txn); + compact = false; } createIndexes(txn); commitTransaction(txn); @@ -358,6 +371,25 @@ abstract class JdbcDatabase implements Database { abortTransaction(txn); throw e; } + // Compact the database if necessary + if (compact) { + if (listener != null) listener.onDatabaseCompaction(); + long start = now(); + compactAndClose(); + logDuration(LOG, "Compacting database", start); + // Allow the next transaction to reopen the DB + synchronized (connectionsLock) { + closed = false; + } + txn = startTransaction(); + try { + storeLastCompacted(txn); + commitTransaction(txn); + } catch (DbException e) { + abortTransaction(txn); + throw e; + } + } } /** @@ -370,9 +402,8 @@ abstract class JdbcDatabase implements Database { * @throws DataTooOldException if the data uses an older schema than the * current code and cannot be migrated */ - private void checkSchemaVersion(Connection txn, + private void checkSchemaVersion(Connection txn, Settings s, @Nullable MigrationListener listener) throws DbException { - Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE); int dataSchemaVersion = s.getInt(SCHEMA_VERSION_KEY, -1); if (dataSchemaVersion == -1) throw new DbException(); if (dataSchemaVersion == CODE_SCHEMA_VERSION) return; @@ -384,7 +415,7 @@ abstract class JdbcDatabase implements Database { if (start == dataSchemaVersion) { if (LOG.isLoggable(INFO)) LOG.info("Migrating from schema " + start + " to " + end); - if (listener != null) listener.onMigrationRun(); + if (listener != null) listener.onDatabaseMigration(); // Apply the migration m.migrate(txn); // Store the new schema version @@ -408,6 +439,19 @@ abstract class JdbcDatabase implements Database { mergeSettings(txn, s, DB_SETTINGS_NAMESPACE); } + private void storeLastCompacted(Connection txn) throws DbException { + Settings s = new Settings(); + s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis()); + mergeSettings(txn, s, DB_SETTINGS_NAMESPACE); + } + + private void initialiseSettings(Connection txn) throws DbException { + Settings s = new Settings(); + s.putInt(SCHEMA_VERSION_KEY, CODE_SCHEMA_VERSION); + s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis()); + mergeSettings(txn, s, DB_SETTINGS_NAMESPACE); + } + private void tryToClose(@Nullable ResultSet rs) { try { if (rs != null) rs.close(); @@ -416,7 +460,7 @@ abstract class JdbcDatabase implements Database { } } - private void tryToClose(@Nullable Statement s) { + protected void tryToClose(@Nullable Statement s) { try { if (s != null) s.close(); } catch (SQLException e) { @@ -424,6 +468,14 @@ abstract class JdbcDatabase implements Database { } } + protected void tryToClose(@Nullable Connection c) { + try { + if (c != null) c.close(); + } catch (SQLException e) { + logException(LOG, WARNING, e); + } + } + private void createTables(Connection txn) throws DbException { Statement s = null; try { @@ -489,7 +541,6 @@ abstract class JdbcDatabase implements Database { if (txn == null) { // Open a new connection txn = createConnection(); - if (txn == null) throw new DbException(); txn.setAutoCommit(false); connectionsLock.lock(); try { diff --git a/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java b/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java index 130595ab6..f158a690f 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java @@ -29,6 +29,7 @@ import javax.inject.Inject; import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; +import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE; import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE; import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.RUNNING; import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING; @@ -159,11 +160,17 @@ class LifecycleManagerImpl implements LifecycleManager, MigrationListener { } @Override - public void onMigrationRun() { + public void onDatabaseMigration() { state = MIGRATING_DATABASE; eventBus.broadcast(new LifecycleEvent(MIGRATING_DATABASE)); } + @Override + public void onDatabaseCompaction() { + state = COMPACTING_DATABASE; + eventBus.broadcast(new LifecycleEvent(COMPACTING_DATABASE)); + } + @Override public void stopServices() { try { diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java index 9e6a0d273..94eb4d196 100644 --- a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java +++ b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java @@ -26,8 +26,8 @@ import org.briarproject.bramble.api.transport.KeySetId; import org.briarproject.bramble.api.transport.OutgoingKeys; import org.briarproject.bramble.api.transport.TransportKeys; import org.briarproject.bramble.system.SystemClock; -import org.briarproject.bramble.test.ArrayClock; import org.briarproject.bramble.test.BrambleTestCase; +import org.briarproject.bramble.test.SettableClock; import org.briarproject.bramble.test.TestDatabaseConfig; import org.briarproject.bramble.test.TestMessageFactory; import org.briarproject.bramble.test.TestUtils; @@ -46,6 +46,7 @@ import java.util.Map.Entry; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -1818,10 +1819,9 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase { @Test public void testMessageRetransmission() throws Exception { long now = System.currentTimeMillis(); - long steps[] = {now, now, now + MAX_LATENCY * 2 - 1, - now + MAX_LATENCY * 2}; + AtomicLong time = new AtomicLong(now); Database db = - open(false, new TestMessageFactory(), new ArrayClock(steps)); + open(false, new TestMessageFactory(), new SettableClock(time)); Connection txn = db.startTransaction(); // Add a contact, a shared group and a shared message @@ -1847,11 +1847,13 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase { // Time: now + MAX_LATENCY * 2 - 1 // The message should not yet be sendable + time.set(now + MAX_LATENCY * 2 - 1); ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY); assertTrue(ids.isEmpty()); // Time: now + MAX_LATENCY * 2 // The message should have expired and should now be sendable + time.set(now + MAX_LATENCY * 2); ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY); assertEquals(singletonList(messageId), ids); @@ -1859,13 +1861,12 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase { db.close(); } - @Test public void testFasterMessageRetransmission() throws Exception { long now = System.currentTimeMillis(); - long steps[] = {now, now, now, now, now + 1}; + AtomicLong time = new AtomicLong(now); Database db = - open(false, new TestMessageFactory(), new ArrayClock(steps)); + open(false, new TestMessageFactory(), new SettableClock(time)); Connection txn = db.startTransaction(); // Add a contact, a shared group and a shared message @@ -1903,6 +1904,7 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase { // Time: now + 1 // The message should no longer be sendable via the faster transport, // as the ETA is now equal + time.set(now + 1); ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY - 1); assertTrue(ids.isEmpty()); @@ -1911,7 +1913,6 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase { db.close(); } - private Database open(boolean resume) throws Exception { return open(resume, new TestMessageFactory(), new SystemClock()); } diff --git a/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java b/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java index d24196d32..24ccc1e01 100644 --- a/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java +++ b/briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java @@ -20,6 +20,7 @@ import org.briarproject.briar.android.navdrawer.NavDrawerActivity; import javax.annotation.ParametersAreNonnullByDefault; import javax.inject.Inject; +import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE; import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE; import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING_SERVICES; @@ -34,7 +35,7 @@ public class OpenDatabaseActivity extends BriarActivity private TextView textView; private ImageView imageView; - private boolean showingMigration = false; + private boolean showingMigration = false, showingCompaction = false; @Override public void onCreate(@Nullable Bundle state) { @@ -57,6 +58,7 @@ public class OpenDatabaseActivity extends BriarActivity finishAndStartApp(); } else { if (state == MIGRATING_DATABASE) showMigration(); + else if (state == COMPACTING_DATABASE) showCompaction(); eventBus.addListener(this); } } @@ -75,6 +77,8 @@ public class OpenDatabaseActivity extends BriarActivity runOnUiThreadUnlessDestroyed(this::finishAndStartApp); else if (state == MIGRATING_DATABASE) runOnUiThreadUnlessDestroyed(this::showMigration); + else if (state == COMPACTING_DATABASE) + runOnUiThreadUnlessDestroyed(this::showCompaction); } } @@ -85,6 +89,13 @@ public class OpenDatabaseActivity extends BriarActivity showingMigration = true; } + private void showCompaction() { + if (showingCompaction) return; + textView.setText(R.string.startup_compact_database); + imageView.setImageResource(R.drawable.startup_migration); + showingCompaction = true; + } + private void finishAndStartApp() { startActivity(new Intent(this, NavDrawerActivity.class)); supportFinishAfterTransition(); diff --git a/briar-android/src/main/res/values/strings.xml b/briar-android/src/main/res/values/strings.xml index 08afd7f57..4875cc50c 100644 --- a/briar-android/src/main/res/values/strings.xml +++ b/briar-android/src/main/res/values/strings.xml @@ -54,6 +54,7 @@ Download Briar 1.0 Decrypting Database… Upgrading Database… + Compacting Database… Open the navigation drawer -- GitLab