Verified Commit 47c91a96 authored by akwizgran's avatar akwizgran

Compact the database at startup.

parent b464fe16
......@@ -38,4 +38,18 @@ public abstract class StringMap extends Hashtable<String, String> {
public void putInt(String key, int value) {
put(key, String.valueOf(value));
}
public long getLong(String key, long defaultValue) {
String s = get(key);
if (s == null) return defaultValue;
try {
return Long.valueOf(s);
} catch (NumberFormatException e) {
return defaultValue;
}
}
public void putLong(String key, long value) {
put(key, String.valueOf(value));
}
}
......@@ -6,6 +6,10 @@ public interface MigrationListener {
* This is called when a migration is started while opening the database.
* It will be called once for each migration being applied.
*/
void onMigrationRun();
void onDatabaseMigration();
/**
* This is called when compaction is started while opening the database.
*/
void onDatabaseCompaction();
}
......@@ -34,7 +34,8 @@ public interface LifecycleManager {
*/
enum LifecycleState {
STARTING, MIGRATING_DATABASE, STARTING_SERVICES, RUNNING, STOPPING;
STARTING, MIGRATING_DATABASE, COMPACTING_DATABASE, STARTING_SERVICES,
RUNNING, STOPPING;
public boolean isAfter(LifecycleState state) {
return ordinal() > state.ordinal();
......
package org.briarproject.bramble.test;
import org.briarproject.bramble.api.system.Clock;
import java.util.concurrent.atomic.AtomicLong;
public class SettableClock implements Clock {
private final AtomicLong time;
public SettableClock(AtomicLong time) {
this.time = time;
}
@Override
public long currentTimeMillis() {
return time.get();
}
@Override
public void sleep(long milliseconds) throws InterruptedException {
Thread.sleep(milliseconds);
}
}
......@@ -2,6 +2,8 @@ package org.briarproject.bramble.db;
import org.briarproject.bramble.api.settings.Settings;
import static java.util.concurrent.TimeUnit.DAYS;
interface DatabaseConstants {
/**
......@@ -23,4 +25,16 @@ interface DatabaseConstants {
*/
String SCHEMA_VERSION_KEY = "schemaVersion";
/**
* The {@link Settings} key under which the time of the last database
* compaction is stored.
*/
String LAST_COMPACTED_KEY = "lastCompacted";
/**
* The maximum time between database compactions in milliseconds. When the
* database is opened it will be compacted if more than this amount of time
* has passed since the last compaction.
*/
long MAX_COMPACTION_INTERVAL_MS = DAYS.toMillis(30);
}
......@@ -13,6 +13,7 @@ import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
import javax.annotation.Nullable;
......@@ -106,4 +107,22 @@ class H2Database extends JdbcDatabase {
String getUrl() {
return url;
}
@Override
protected void compactAndClose() throws DbException {
Connection c = null;
Statement s = null;
try {
c = createConnection();
closeAllConnections();
s = c.createStatement();
s.execute("SHUTDOWN COMPACT");
s.close();
c.close();
} catch (SQLException e) {
tryToClose(s);
tryToClose(c);
throw new DbException(e);
}
}
}
......@@ -61,14 +61,18 @@ class HyperSqlDatabase extends JdbcDatabase {
@Override
public void close() throws DbException {
Connection c = null;
Statement s = null;
try {
super.closeAllConnections();
Connection c = createConnection();
Statement s = c.createStatement();
c = createConnection();
s = c.createStatement();
s.executeQuery("SHUTDOWN");
s.close();
c.close();
} catch (SQLException e) {
tryToClose(s);
tryToClose(c);
throw new DbException(e);
}
}
......@@ -104,4 +108,22 @@ class HyperSqlDatabase extends JdbcDatabase {
String hex = StringUtils.toHexString(key.getBytes());
return DriverManager.getConnection(url + ";crypt_key=" + hex);
}
@Override
protected void compactAndClose() throws DbException {
Connection c = null;
Statement s = null;
try {
super.closeAllConnections();
c = createConnection();
s = c.createStatement();
s.executeQuery("SHUTDOWN COMPACT");
s.close();
c.close();
} catch (SQLException e) {
tryToClose(s);
tryToClose(c);
throw new DbException(e);
}
}
}
......@@ -67,9 +67,13 @@ import static org.briarproject.bramble.api.sync.ValidationManager.State.DELIVERE
import static org.briarproject.bramble.api.sync.ValidationManager.State.PENDING;
import static org.briarproject.bramble.api.sync.ValidationManager.State.UNKNOWN;
import static org.briarproject.bramble.db.DatabaseConstants.DB_SETTINGS_NAMESPACE;
import static org.briarproject.bramble.db.DatabaseConstants.LAST_COMPACTED_KEY;
import static org.briarproject.bramble.db.DatabaseConstants.MAX_COMPACTION_INTERVAL_MS;
import static org.briarproject.bramble.db.DatabaseConstants.SCHEMA_VERSION_KEY;
import static org.briarproject.bramble.db.ExponentialBackoff.calculateExpiry;
import static org.briarproject.bramble.util.LogUtils.logDuration;
import static org.briarproject.bramble.util.LogUtils.logException;
import static org.briarproject.bramble.util.LogUtils.now;
/**
* A generic database implementation that can be used with any JDBC-compatible
......@@ -317,9 +321,10 @@ abstract class JdbcDatabase implements Database<Connection> {
private int openConnections = 0; // Locking: connectionsLock
private boolean closed = false; // Locking: connectionsLock
@Nullable
protected abstract Connection createConnection() throws SQLException;
protected abstract void compactAndClose() throws DbException;
private final Lock connectionsLock = new ReentrantLock();
private final Condition connectionsChanged = connectionsLock.newCondition();
......@@ -344,13 +349,21 @@ abstract class JdbcDatabase implements Database<Connection> {
throw new DbException(e);
}
// Open the database and create the tables and indexes if necessary
boolean compact;
Connection txn = startTransaction();
try {
if (reopen) {
checkSchemaVersion(txn, listener);
Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE);
checkSchemaVersion(txn, s, listener);
long lastCompacted = s.getLong(LAST_COMPACTED_KEY, 0);
long elapsed = clock.currentTimeMillis() - lastCompacted;
if (LOG.isLoggable(INFO))
LOG.info(elapsed + " ms since last compaction");
compact = elapsed > MAX_COMPACTION_INTERVAL_MS;
} else {
createTables(txn);
storeSchemaVersion(txn, CODE_SCHEMA_VERSION);
initialiseSettings(txn);
compact = false;
}
createIndexes(txn);
commitTransaction(txn);
......@@ -358,6 +371,25 @@ abstract class JdbcDatabase implements Database<Connection> {
abortTransaction(txn);
throw e;
}
// Compact the database if necessary
if (compact) {
if (listener != null) listener.onDatabaseCompaction();
long start = now();
compactAndClose();
logDuration(LOG, "Compacting database", start);
// Allow the next transaction to reopen the DB
synchronized (connectionsLock) {
closed = false;
}
txn = startTransaction();
try {
storeLastCompacted(txn);
commitTransaction(txn);
} catch (DbException e) {
abortTransaction(txn);
throw e;
}
}
}
/**
......@@ -370,9 +402,8 @@ abstract class JdbcDatabase implements Database<Connection> {
* @throws DataTooOldException if the data uses an older schema than the
* current code and cannot be migrated
*/
private void checkSchemaVersion(Connection txn,
private void checkSchemaVersion(Connection txn, Settings s,
@Nullable MigrationListener listener) throws DbException {
Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE);
int dataSchemaVersion = s.getInt(SCHEMA_VERSION_KEY, -1);
if (dataSchemaVersion == -1) throw new DbException();
if (dataSchemaVersion == CODE_SCHEMA_VERSION) return;
......@@ -384,7 +415,7 @@ abstract class JdbcDatabase implements Database<Connection> {
if (start == dataSchemaVersion) {
if (LOG.isLoggable(INFO))
LOG.info("Migrating from schema " + start + " to " + end);
if (listener != null) listener.onMigrationRun();
if (listener != null) listener.onDatabaseMigration();
// Apply the migration
m.migrate(txn);
// Store the new schema version
......@@ -408,6 +439,19 @@ abstract class JdbcDatabase implements Database<Connection> {
mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
}
private void storeLastCompacted(Connection txn) throws DbException {
Settings s = new Settings();
s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis());
mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
}
private void initialiseSettings(Connection txn) throws DbException {
Settings s = new Settings();
s.putInt(SCHEMA_VERSION_KEY, CODE_SCHEMA_VERSION);
s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis());
mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
}
private void tryToClose(@Nullable ResultSet rs) {
try {
if (rs != null) rs.close();
......@@ -416,7 +460,7 @@ abstract class JdbcDatabase implements Database<Connection> {
}
}
private void tryToClose(@Nullable Statement s) {
protected void tryToClose(@Nullable Statement s) {
try {
if (s != null) s.close();
} catch (SQLException e) {
......@@ -424,6 +468,14 @@ abstract class JdbcDatabase implements Database<Connection> {
}
}
protected void tryToClose(@Nullable Connection c) {
try {
if (c != null) c.close();
} catch (SQLException e) {
logException(LOG, WARNING, e);
}
}
private void createTables(Connection txn) throws DbException {
Statement s = null;
try {
......@@ -489,7 +541,6 @@ abstract class JdbcDatabase implements Database<Connection> {
if (txn == null) {
// Open a new connection
txn = createConnection();
if (txn == null) throw new DbException();
txn.setAutoCommit(false);
connectionsLock.lock();
try {
......
......@@ -29,6 +29,7 @@ import javax.inject.Inject;
import static java.util.logging.Level.FINE;
import static java.util.logging.Level.INFO;
import static java.util.logging.Level.WARNING;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.RUNNING;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING;
......@@ -159,11 +160,17 @@ class LifecycleManagerImpl implements LifecycleManager, MigrationListener {
}
@Override
public void onMigrationRun() {
public void onDatabaseMigration() {
state = MIGRATING_DATABASE;
eventBus.broadcast(new LifecycleEvent(MIGRATING_DATABASE));
}
@Override
public void onDatabaseCompaction() {
state = COMPACTING_DATABASE;
eventBus.broadcast(new LifecycleEvent(COMPACTING_DATABASE));
}
@Override
public void stopServices() {
try {
......
......@@ -26,8 +26,8 @@ import org.briarproject.bramble.api.transport.KeySetId;
import org.briarproject.bramble.api.transport.OutgoingKeys;
import org.briarproject.bramble.api.transport.TransportKeys;
import org.briarproject.bramble.system.SystemClock;
import org.briarproject.bramble.test.ArrayClock;
import org.briarproject.bramble.test.BrambleTestCase;
import org.briarproject.bramble.test.SettableClock;
import org.briarproject.bramble.test.TestDatabaseConfig;
import org.briarproject.bramble.test.TestMessageFactory;
import org.briarproject.bramble.test.TestUtils;
......@@ -46,6 +46,7 @@ import java.util.Map.Entry;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
......@@ -1818,10 +1819,9 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
@Test
public void testMessageRetransmission() throws Exception {
long now = System.currentTimeMillis();
long steps[] = {now, now, now + MAX_LATENCY * 2 - 1,
now + MAX_LATENCY * 2};
AtomicLong time = new AtomicLong(now);
Database<Connection> db =
open(false, new TestMessageFactory(), new ArrayClock(steps));
open(false, new TestMessageFactory(), new SettableClock(time));
Connection txn = db.startTransaction();
// Add a contact, a shared group and a shared message
......@@ -1847,11 +1847,13 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
// Time: now + MAX_LATENCY * 2 - 1
// The message should not yet be sendable
time.set(now + MAX_LATENCY * 2 - 1);
ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY);
assertTrue(ids.isEmpty());
// Time: now + MAX_LATENCY * 2
// The message should have expired and should now be sendable
time.set(now + MAX_LATENCY * 2);
ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY);
assertEquals(singletonList(messageId), ids);
......@@ -1859,13 +1861,12 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
db.close();
}
@Test
public void testFasterMessageRetransmission() throws Exception {
long now = System.currentTimeMillis();
long steps[] = {now, now, now, now, now + 1};
AtomicLong time = new AtomicLong(now);
Database<Connection> db =
open(false, new TestMessageFactory(), new ArrayClock(steps));
open(false, new TestMessageFactory(), new SettableClock(time));
Connection txn = db.startTransaction();
// Add a contact, a shared group and a shared message
......@@ -1903,6 +1904,7 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
// Time: now + 1
// The message should no longer be sendable via the faster transport,
// as the ETA is now equal
time.set(now + 1);
ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE,
MAX_LATENCY - 1);
assertTrue(ids.isEmpty());
......@@ -1911,7 +1913,6 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
db.close();
}
private Database<Connection> open(boolean resume) throws Exception {
return open(resume, new TestMessageFactory(), new SystemClock());
}
......
......@@ -20,6 +20,7 @@ import org.briarproject.briar.android.navdrawer.NavDrawerActivity;
import javax.annotation.ParametersAreNonnullByDefault;
import javax.inject.Inject;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE;
import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING_SERVICES;
......@@ -34,7 +35,7 @@ public class OpenDatabaseActivity extends BriarActivity
private TextView textView;
private ImageView imageView;
private boolean showingMigration = false;
private boolean showingMigration = false, showingCompaction = false;
@Override
public void onCreate(@Nullable Bundle state) {
......@@ -57,6 +58,7 @@ public class OpenDatabaseActivity extends BriarActivity
finishAndStartApp();
} else {
if (state == MIGRATING_DATABASE) showMigration();
else if (state == COMPACTING_DATABASE) showCompaction();
eventBus.addListener(this);
}
}
......@@ -75,6 +77,8 @@ public class OpenDatabaseActivity extends BriarActivity
runOnUiThreadUnlessDestroyed(this::finishAndStartApp);
else if (state == MIGRATING_DATABASE)
runOnUiThreadUnlessDestroyed(this::showMigration);
else if (state == COMPACTING_DATABASE)
runOnUiThreadUnlessDestroyed(this::showCompaction);
}
}
......@@ -85,6 +89,13 @@ public class OpenDatabaseActivity extends BriarActivity
showingMigration = true;
}
private void showCompaction() {
if (showingCompaction) return;
textView.setText(R.string.startup_compact_database);
imageView.setImageResource(R.drawable.startup_migration);
showingCompaction = true;
}
private void finishAndStartApp() {
startActivity(new Intent(this, NavDrawerActivity.class));
supportFinishAfterTransition();
......
......@@ -54,6 +54,7 @@
<string name="download_briar_button">Download Briar 1.0</string>
<string name="startup_open_database">Decrypting Database…</string>
<string name="startup_migrate_database">Upgrading Database…</string>
<string name="startup_compact_database">Compacting Database…</string>
<!-- Navigation Drawer -->
<string name="nav_drawer_open_description">Open the navigation drawer</string>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment