diff --git a/mailbox-android/src/main/java/org/briarproject/mailbox/android/AppModule.kt b/mailbox-android/src/main/java/org/briarproject/mailbox/android/AppModule.kt index d4e4de7a15408e80a2b98a5a77e116fd4555511f..d1126699664b698566427a16011841c02c0eadaf 100644 --- a/mailbox-android/src/main/java/org/briarproject/mailbox/android/AppModule.kt +++ b/mailbox-android/src/main/java/org/briarproject/mailbox/android/AppModule.kt @@ -1,9 +1,15 @@ package org.briarproject.mailbox.android +import android.app.Application +import android.content.Context import dagger.Module +import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.components.SingletonComponent import org.briarproject.mailbox.core.CoreModule +import org.briarproject.mailbox.core.db.DatabaseConfig +import java.io.File +import javax.inject.Singleton @Module( includes = [ @@ -12,4 +18,12 @@ import org.briarproject.mailbox.core.CoreModule ] ) @InstallIn(SingletonComponent::class) -internal class AppModule +internal class AppModule { + @Singleton + @Provides + fun provideDatabaseConfig(app: Application) = object : DatabaseConfig { + override fun getDatabaseDirectory(): File { + return app.applicationContext.getDir("db", Context.MODE_PRIVATE) + } + } +} diff --git a/mailbox-cli/src/main/java/org/briarproject/mailbox/cli/JavaCliModule.kt b/mailbox-cli/src/main/java/org/briarproject/mailbox/cli/JavaCliModule.kt index 3638ea3a6125418665f874ae26bc1d29e49c7049..cb2b1a40240c290016c8873ca01576b2dab74498 100644 --- a/mailbox-cli/src/main/java/org/briarproject/mailbox/cli/JavaCliModule.kt +++ b/mailbox-cli/src/main/java/org/briarproject/mailbox/cli/JavaCliModule.kt @@ -1,12 +1,16 @@ package org.briarproject.mailbox.cli import dagger.Module +import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.components.SingletonComponent import org.briarproject.mailbox.core.CoreModule +import org.briarproject.mailbox.core.db.DatabaseConfig import org.briarproject.mailbox.core.event.DefaultEventExecutorModule import org.briarproject.mailbox.core.system.DefaultTaskSchedulerModule import org.briarproject.mailbox.core.tor.JavaTorModule +import java.io.File +import javax.inject.Singleton @Module( includes = [ @@ -17,4 +21,13 @@ import org.briarproject.mailbox.core.tor.JavaTorModule ] ) @InstallIn(SingletonComponent::class) -internal class JavaCliModule +internal class JavaCliModule { + @Singleton + @Provides + fun provideDatabaseConfig() = object : DatabaseConfig { + override fun getDatabaseDirectory(): File { + // TODO: use a correct location + return File("/tmp") + } + } +} diff --git a/mailbox-core/build.gradle b/mailbox-core/build.gradle index fc34695fd31e37055694ca56685ac3531d10e0f6..33f0927ea6db736a0c4093673186d6525bff656e 100644 --- a/mailbox-core/build.gradle +++ b/mailbox-core/build.gradle @@ -24,6 +24,8 @@ dependencies { implementation "io.ktor:ktor-server-netty:$ktorVersion" api "org.slf4j:slf4j-api:1.7.32" + implementation 'com.h2database:h2:1.4.192' // The last version that supports Java 1.6 + testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlin_version" testImplementation "org.junit.jupiter:junit-jupiter-api:$junit_version" testImplementation "org.junit.jupiter:junit-jupiter-params:$junit_version" diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/api/Contact.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/api/Contact.kt new file mode 100644 index 0000000000000000000000000000000000000000..86eded1fdf72cb27b6c025e6396725026c5f25a2 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/api/Contact.kt @@ -0,0 +1,8 @@ +package org.briarproject.mailbox.core.api + +data class Contact( + val id: Int, + val token: String, + val inboxId: String, + val outboxId: String, +) diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DataTooNewException.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DataTooNewException.kt new file mode 100644 index 0000000000000000000000000000000000000000..c6120d1880c2bc6ebe708835a407aafb90c514e7 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DataTooNewException.kt @@ -0,0 +1,6 @@ +package org.briarproject.mailbox.core.db + +/** + * Thrown when the database uses a newer schema than the current code. + */ +class DataTooNewException : DbException() diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DataTooOldException.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DataTooOldException.kt new file mode 100644 index 0000000000000000000000000000000000000000..02d23760f8a89f3c1fd89a3dc1138c4f5538218a --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DataTooOldException.kt @@ -0,0 +1,7 @@ +package org.briarproject.mailbox.core.db + +/** + * Thrown when the database uses an older schema than the current code and + * cannot be migrated. + */ +class DataTooOldException : DbException() diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/Database.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/Database.kt new file mode 100644 index 0000000000000000000000000000000000000000..336cac11ff218437b8dbc3e94287f33117a97b59 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/Database.kt @@ -0,0 +1,55 @@ +package org.briarproject.mailbox.core.db + +import org.briarproject.mailbox.core.api.Contact +import org.briarproject.mailbox.core.settings.Settings +import java.sql.Connection + +interface Database<T> { + + /** + * Opens the database and returns true if the database already existed. + */ + fun open(listener: MigrationListener?): Boolean + + /** + * Prevents new transactions from starting, waits for all current + * transactions to finish, and closes the database. + */ + @Throws(DbException::class) + fun close() + + /** + * Starts a new transaction and returns an object representing it. + */ + @Throws(DbException::class) + fun startTransaction(): T + + /** + * Aborts the given transaction - no changes made during the transaction + * will be applied to the database. + */ + fun abortTransaction(txn: T) + + /** + * Commits the given transaction - all changes made during the transaction + * will be applied to the database. + */ + @Throws(DbException::class) + fun commitTransaction(txn: T) + + @Throws(DbException::class) + fun getSettings(txn: Connection, namespace: String?): Settings + + @Throws(DbException::class) + fun mergeSettings(txn: Connection, s: Settings, namespace: String?) + + @Throws(DbException::class) + fun addContact(txn: T, contact: Contact) + + @Throws(DbException::class) + fun getContact(txn: T, id: Int): Contact? + + @Throws(DbException::class) + fun removeContact(txn: T, id: Int) + +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseComponentImpl.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseComponentImpl.kt index 86804941fc8d92e3f2efb250f18679f4a1280107..e17f7014cce00c48304d7d938c39e52e3c78b8da 100644 --- a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseComponentImpl.kt +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseComponentImpl.kt @@ -1,6 +1,10 @@ package org.briarproject.mailbox.core.db -class DatabaseComponentImpl : DatabaseComponent { +import javax.inject.Inject + +class DatabaseComponentImpl<T> @Inject constructor(database: Database<T>) : DatabaseComponent { + + private val db: Database<T>? = null override fun open(listener: MigrationListener?): Boolean { // TODO: implement this diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseConfig.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseConfig.kt new file mode 100644 index 0000000000000000000000000000000000000000..50101bbad976246f74b5e1b6d8acc07f7cfba2f2 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseConfig.kt @@ -0,0 +1,12 @@ +package org.briarproject.mailbox.core.db + +import java.io.File + +interface DatabaseConfig { + + /** + * Returns the directory where the database stores its data. + */ + fun getDatabaseDirectory(): File + +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseConstants.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseConstants.kt new file mode 100644 index 0000000000000000000000000000000000000000..5fd4c94faa81fbbbda7184a6dda80990c1559f38 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseConstants.kt @@ -0,0 +1,43 @@ +package org.briarproject.mailbox.core.db + +import org.briarproject.mailbox.core.settings.Settings +import java.util.concurrent.TimeUnit.DAYS + +interface DatabaseConstants { + + companion object { + + /** + * The namespace of the [Settings] where the database schema version + * is stored. + */ + const val DB_SETTINGS_NAMESPACE = "db" + + /** + * The [Settings] key under which the database schema version is + * stored. + */ + const val SCHEMA_VERSION_KEY = "schemaVersion" + + /** + * The [Settings] key under which the time of the last database + * compaction is stored. + */ + const val LAST_COMPACTED_KEY = "lastCompacted" + + /** + * The maximum time between database compactions in milliseconds. When the + * database is opened it will be compacted if more than this amount of time + * has passed since the last compaction. + */ + var MAX_COMPACTION_INTERVAL_MS = DAYS.toMillis(30) + + /** + * The [Settings] key under which the flag is stored indicating + * whether the database is marked as dirty. + */ + const val DIRTY_KEY = "dirty" + + } + +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseModule.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseModule.kt index 851fcbe413325f6beda34db552e1f183ae9c830a..84a4e28308ca4866d54a27145043a1a7686b3820 100644 --- a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseModule.kt +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseModule.kt @@ -4,6 +4,8 @@ import dagger.Module import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.components.SingletonComponent +import org.briarproject.mailbox.core.system.Clock +import java.sql.Connection import javax.inject.Singleton @Module @@ -12,8 +14,14 @@ internal class DatabaseModule { @Provides @Singleton - fun provideDatabaseComponent(): DatabaseComponent { - return DatabaseComponentImpl() + fun provideDatabase(config: DatabaseConfig, clock: Clock): Database<Connection> { + return H2Database(config, clock) + } + + @Provides + @Singleton + fun provideDatabaseComponent(db: Database<Connection>): DatabaseComponent { + return DatabaseComponentImpl(db) } } diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseTypes.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseTypes.kt new file mode 100644 index 0000000000000000000000000000000000000000..1536ac2f602473c58937a74d56bda2d07a88abc5 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DatabaseTypes.kt @@ -0,0 +1,28 @@ +package org.briarproject.mailbox.core.db + +class DatabaseTypes( + private val hashType: String, + private val secretType: String, + private val binaryType: String, + private val counterType: String, + private val stringType: String, +) { + /** + * Replaces database type placeholders in a statement with the actual types. + * These placeholders are currently supported: + * * _HASH + * * _SECRET + * * _BINARY + * * _COUNTER + * * _STRING + */ + fun replaceTypes(stmt: String): String { + var s = stmt + s = s.replace("_HASH".toRegex(), hashType) + s = s.replace("_SECRET".toRegex(), secretType) + s = s.replace("_BINARY".toRegex(), binaryType) + s = s.replace("_COUNTER".toRegex(), counterType) + s = s.replace("_STRING".toRegex(), stringType) + return s + } +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbClosedException.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbClosedException.kt new file mode 100644 index 0000000000000000000000000000000000000000..283fc17a4a3b4a6b0fe77283d19762080f9efd3e --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbClosedException.kt @@ -0,0 +1,3 @@ +package org.briarproject.mailbox.core.db + +class DbClosedException : DbException() diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbException.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbException.kt new file mode 100644 index 0000000000000000000000000000000000000000..263669dd1a22a762f9bf237ca05e1de94243662f --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbException.kt @@ -0,0 +1,9 @@ +package org.briarproject.mailbox.core.db + +open class DbException : Exception { + + constructor() + + constructor(t: Throwable?) : super(t) + +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbStateException.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbStateException.kt new file mode 100644 index 0000000000000000000000000000000000000000..05c4985ce8cfb7ffbb3133e265623b40ad6c1801 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/DbStateException.kt @@ -0,0 +1,8 @@ +package org.briarproject.mailbox.core.db + +import java.sql.SQLException + +/** + * Thrown when the database is in an illegal state. + */ +class DbStateException : SQLException() diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/H2Database.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/H2Database.kt new file mode 100644 index 0000000000000000000000000000000000000000..ae7ec1f9ebe644056b91c04880c7631890fdeb83 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/H2Database.kt @@ -0,0 +1,90 @@ +package org.briarproject.mailbox.core.db + +import org.briarproject.mailbox.core.db.JdbcUtils.tryToClose +import org.briarproject.mailbox.core.system.Clock +import org.briarproject.mailbox.core.util.IoUtils.isNonEmptyDirectory +import org.briarproject.mailbox.core.util.LogUtils.logFileOrDir +import org.slf4j.LoggerFactory +import java.io.File +import java.sql.Connection +import java.sql.DriverManager +import java.sql.SQLException +import java.sql.Statement +import java.util.Properties + +class H2Database( + private val config: DatabaseConfig, + val clock: Clock, +) : JdbcDatabase(dbTypes, clock) { + + internal companion object { + private val LOG = LoggerFactory.getLogger(H2Database::class.java) + + private const val HASH_TYPE = "BINARY(32)" + private const val SECRET_TYPE = "BINARY(32)" + private const val BINARY_TYPE = "BINARY" + private const val COUNTER_TYPE = "INT NOT NULL AUTO_INCREMENT" + private const val STRING_TYPE = "VARCHAR" + private val dbTypes = DatabaseTypes( + HASH_TYPE, SECRET_TYPE, BINARY_TYPE, COUNTER_TYPE, STRING_TYPE + ) + } + + private val dbPath: String get() = File(config.getDatabaseDirectory(), "db").absolutePath + private val url: String = ("jdbc:h2:split:$dbPath;MULTI_THREADED=1;WRITE_DELAY=0") + + override fun open(listener: MigrationListener?): Boolean { + val dir = config.getDatabaseDirectory() + if (LOG.isInfoEnabled) { + LOG.info("Contents of account directory before opening DB:") + logFileOrDir(LOG, dir.parentFile) + } + val reopen = isNonEmptyDirectory(dir) + if (LOG.isInfoEnabled) LOG.info("Reopening DB: $reopen") + if (!reopen && dir.mkdirs()) LOG.info("Created database directory") + super.open("org.h2.Driver", reopen, listener) + if (LOG.isInfoEnabled) { + LOG.info("Contents of account directory after opening DB:") + logFileOrDir(LOG, dir.parentFile) + } + return reopen + } + + override fun close() { + // H2 will close the database when the last connection closes + var c: Connection? = null + try { + c = createConnection() + super.closeAllConnections() + setDirty(c, false) + c.close() + } catch (e: SQLException) { + tryToClose(c, LOG) + throw DbException(e) + } + } + + @Throws(DbException::class, SQLException::class) + override fun createConnection(): Connection { + val props = Properties() + return DriverManager.getConnection(url, props) + } + + override fun compactAndClose() { + var c: Connection? = null + var s: Statement? = null + try { + c = createConnection() + closeAllConnections() + s = c.createStatement() + s.execute("SHUTDOWN COMPACT") + s.close() + c.close() + } catch (e: SQLException) { + tryToClose(s, LOG) + tryToClose(c, LOG) + throw DbException(e) + } + } + +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/JdbcDatabase.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/JdbcDatabase.kt new file mode 100644 index 0000000000000000000000000000000000000000..a0433b4879e8653fca233adc7182d118393f1f44 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/JdbcDatabase.kt @@ -0,0 +1,464 @@ +package org.briarproject.mailbox.core.db + +import org.briarproject.mailbox.core.api.Contact +import org.briarproject.mailbox.core.db.DatabaseConstants.Companion.DB_SETTINGS_NAMESPACE +import org.briarproject.mailbox.core.db.DatabaseConstants.Companion.DIRTY_KEY +import org.briarproject.mailbox.core.db.DatabaseConstants.Companion.LAST_COMPACTED_KEY +import org.briarproject.mailbox.core.db.DatabaseConstants.Companion.MAX_COMPACTION_INTERVAL_MS +import org.briarproject.mailbox.core.db.DatabaseConstants.Companion.SCHEMA_VERSION_KEY +import org.briarproject.mailbox.core.db.JdbcUtils.tryToClose +import org.briarproject.mailbox.core.settings.Settings +import org.briarproject.mailbox.core.system.Clock +import org.briarproject.mailbox.core.util.LogUtils.logDuration +import org.briarproject.mailbox.core.util.LogUtils.logException +import org.briarproject.mailbox.core.util.LogUtils.now +import org.slf4j.LoggerFactory +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.SQLException +import java.sql.Statement +import java.util.Arrays +import java.util.LinkedList +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock +import javax.annotation.concurrent.GuardedBy + +abstract class JdbcDatabase(private val dbTypes: DatabaseTypes, private val clock: Clock) : + Database<Connection> { + + internal companion object { + + private val LOG = LoggerFactory.getLogger(JdbcDatabase::class.java) + const val CODE_SCHEMA_VERSION = 1 + + private val CREATE_SETTINGS = """ + CREATE TABLE settings + (namespace _STRING NOT NULL, + settingKey _STRING NOT NULL, + value _STRING NOT NULL, + PRIMARY KEY (namespace, settingKey)) + """.trimIndent() + + private val CREATE_CONTACTS = """ + CREATE TABLE contacts + (contactId INT NOT NULL, + token _STRING NOT NULL, + inbox _STRING NOT NULL, + outbox _STRING NOT NULL) + """.trimIndent() + } + + private val connectionsLock: Lock = ReentrantLock() + private val connectionsChanged = connectionsLock.newCondition() + + @GuardedBy("connectionsLock") + private val connections = LinkedList<Connection>() + + @GuardedBy("connectionsLock") + private var openConnections = 0 + + @GuardedBy("connectionsLock") + private var closed = false + + @Volatile + private var wasDirtyOnInitialisation = false + + fun open(driverClass: String, reopen: Boolean, listener: MigrationListener?) { + // Load the JDBC driver + try { + Class.forName(driverClass) + } catch (e: ClassNotFoundException) { + throw DbException(e) + } + // Open the database and create the tables and indexes if necessary + val compact: Boolean + var txn = startTransaction() + try { + compact = if (reopen) { + val s: Settings = getSettings(txn, DB_SETTINGS_NAMESPACE) + wasDirtyOnInitialisation = isDirty(s) + migrateSchema(txn, s, listener) || isCompactionDue(s) + } else { + wasDirtyOnInitialisation = false + createTables(txn) + initialiseSettings(txn) + false + } + if (LOG.isInfoEnabled) { + LOG.info("db dirty? $wasDirtyOnInitialisation") + } + createIndexes(txn) + commitTransaction(txn) + } catch (e: DbException) { + abortTransaction(txn) + throw e + } + // Compact the database if necessary + if (compact) { + listener?.onDatabaseCompaction() + val start: Long = now() + compactAndClose() + logDuration(LOG, { "Compacting database" }, start) + // Allow the next transaction to reopen the DB + synchronized(connectionsLock) { closed = false } + txn = startTransaction() + try { + storeLastCompacted(txn) + commitTransaction(txn) + } catch (e: DbException) { + abortTransaction(txn) + throw e + } + } + } + + /** + * Compares the schema version stored in the database with the schema + * version used by the current code and applies any suitable migrations to + * the data if necessary. + * + * @return true if any migrations were applied, false if the schema was + * already current + * @throws DataTooNewException if the data uses a newer schema than the + * current code + * @throws DataTooOldException if the data uses an older schema than the + * current code and cannot be migrated + */ + @Throws(DbException::class) + private fun migrateSchema(txn: Connection, s: Settings, listener: MigrationListener?): Boolean { + var dataSchemaVersion = s.getInt(SCHEMA_VERSION_KEY, -1) + if (dataSchemaVersion == -1) throw DbException() + if (dataSchemaVersion == CODE_SCHEMA_VERSION) return false + if (CODE_SCHEMA_VERSION < dataSchemaVersion) throw DataTooNewException() + // Apply any suitable migrations in order + for (m in getMigrations()) { + val start: Int = m.startVersion + val end: Int = m.endVersion + if (start == dataSchemaVersion) { + if (LOG.isInfoEnabled) LOG.info("Migrating from schema $start to $end") + listener?.onDatabaseMigration() + // Apply the migration + m.migrate(txn) + // Store the new schema version + storeSchemaVersion(txn, end) + dataSchemaVersion = end + } + } + if (dataSchemaVersion != CODE_SCHEMA_VERSION) throw DataTooOldException() + return true + } + + // Public access for testing + fun getMigrations(): List<Migration<Connection>> { + return Arrays.asList<Migration<Connection>>( + // Migration1_2(dbTypes), + ) + } + + @Throws(DbException::class, SQLException::class) + protected abstract fun createConnection(): Connection + + @Throws(DbException::class) + protected abstract fun compactAndClose() + + override fun startTransaction(): Connection { + var txn: Connection? + connectionsLock.lock() + txn = try { + if (closed) throw DbClosedException() + connections.poll() + } finally { + connectionsLock.unlock() + } + try { + if (txn == null) { + // Open a new connection + txn = createConnection() + txn.autoCommit = false + connectionsLock.lock() + try { + openConnections++ + } finally { + connectionsLock.unlock() + } + } + } catch (e: SQLException) { + throw DbException(e) + } + return txn + } + + override fun abortTransaction(txn: Connection) { + try { + txn.rollback() + connectionsLock.lock() + try { + connections.add(txn) + connectionsChanged.signalAll() + } finally { + connectionsLock.unlock() + } + } catch (e: SQLException) { + // Try to close the connection + logException(LOG, e) + tryToClose(txn, LOG) + // Whatever happens, allow the database to close + connectionsLock.lock() + try { + openConnections-- + connectionsChanged.signalAll() + } finally { + connectionsLock.unlock() + } + } + } + + override fun commitTransaction(txn: Connection) { + try { + txn.commit() + } catch (e: SQLException) { + throw DbException(e) + } + connectionsLock.lock() + try { + connections.add(txn) + connectionsChanged.signalAll() + } finally { + connectionsLock.unlock() + } + } + + @Throws(SQLException::class) + fun closeAllConnections() { + var interrupted = false + connectionsLock.lock() + try { + closed = true + for (c in connections) c.close() + openConnections -= connections.size + connections.clear() + while (openConnections > 0) { + try { + connectionsChanged.await() + } catch (e: InterruptedException) { + LOG.warn("Interrupted while closing connections") + interrupted = true + } + for (c in connections) c.close() + openConnections -= connections.size + connections.clear() + } + } finally { + connectionsLock.unlock() + } + if (interrupted) Thread.currentThread().interrupt() + } + + private fun isDirty(s: Settings): Boolean { + return s.getBoolean(DIRTY_KEY, false) + } + + @Throws(DbException::class) + fun setDirty(txn: Connection, dirty: Boolean) { + val s = Settings() + s.putBoolean(DIRTY_KEY, dirty) + mergeSettings(txn, s, DB_SETTINGS_NAMESPACE) + } + + private fun isCompactionDue(s: Settings): Boolean { + val lastCompacted = s.getLong(LAST_COMPACTED_KEY, 0) + val elapsed = clock.currentTimeMillis() - lastCompacted + if (LOG.isInfoEnabled) LOG.info("$elapsed ms since last compaction") + return elapsed > MAX_COMPACTION_INTERVAL_MS + } + + @Throws(DbException::class) + private fun storeSchemaVersion(txn: Connection, version: Int) { + val s = Settings() + s.putInt(SCHEMA_VERSION_KEY, version) + mergeSettings(txn, s, DB_SETTINGS_NAMESPACE) + } + + @Throws(DbException::class) + private fun storeLastCompacted(txn: Connection) { + val s = Settings() + s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis()) + mergeSettings(txn, s, DB_SETTINGS_NAMESPACE) + } + + @Throws(DbException::class) + private fun initialiseSettings(txn: Connection) { + val s = Settings() + s.putInt(SCHEMA_VERSION_KEY, CODE_SCHEMA_VERSION) + s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis()) + mergeSettings(txn, s, DB_SETTINGS_NAMESPACE) + } + + @Throws(DbException::class) + private fun createTables(txn: Connection) { + var s: Statement? = null + try { + s = txn.createStatement() + s.executeUpdate(dbTypes.replaceTypes(CREATE_SETTINGS)) + s.executeUpdate(dbTypes.replaceTypes(CREATE_CONTACTS)) + s.close() + } catch (e: SQLException) { + tryToClose(s, LOG) + throw DbException(e) + } + } + + @Throws(DbException::class) + private fun createIndexes(txn: Connection) { + var s: Statement? = null + try { + s = txn.createStatement() + // s.executeUpdate(INDEX_SOMETABLE_BY_SOMECOLUMN) + s.close() + } catch (e: SQLException) { + tryToClose(s, LOG) + throw DbException(e) + } + } + + @Throws(DbException::class) + override fun getSettings(txn: Connection, namespace: String?): Settings { + var ps: PreparedStatement? = null + var rs: ResultSet? = null + return try { + val sql = """ + SELECT settingKey, value FROM settings + WHERE namespace = ? + """.trimIndent() + + ps = txn.prepareStatement(sql) + ps.setString(1, namespace) + rs = ps.executeQuery() + val s = Settings() + while (rs.next()) s[rs.getString(1)] = rs.getString(2) + rs.close() + ps.close() + s + } catch (e: SQLException) { + tryToClose(rs, LOG) + tryToClose(ps, LOG) + throw DbException(e) + } + } + + @Throws(DbException::class) + override fun mergeSettings(txn: Connection, s: Settings, namespace: String?) { + var ps: PreparedStatement? = null + try { + // Update any settings that already exist + var sql = """ + UPDATE settings SET value = ? + WHERE namespace = ? AND settingKey = ? + """.trimIndent() + + ps = txn.prepareStatement(sql) + for ((key, value) in s) { + ps.setString(1, value) + ps.setString(2, namespace) + ps.setString(3, key) + ps.addBatch() + } + var batchAffected = ps.executeBatch() + if (batchAffected.size != s.size) throw DbStateException() + for (rows in batchAffected) { + if (rows < 0) throw DbStateException() + if (rows > 1) throw DbStateException() + } + // Insert any settings that don't already exist + sql = """ + INSERT INTO settings (namespace, settingKey, value) + VALUES (?, ?, ?) + """.trimIndent() + + ps = txn.prepareStatement(sql) + var updateIndex = 0 + var inserted = 0 + for ((key, value) in s) { + if (batchAffected[updateIndex] == 0) { + ps.setString(1, namespace) + ps.setString(2, key) + ps.setString(3, value) + ps.addBatch() + inserted++ + } + updateIndex++ + } + batchAffected = ps.executeBatch() + if (batchAffected.size != inserted) throw DbStateException() + for (rows in batchAffected) if (rows != 1) throw DbStateException() + ps.close() + } catch (e: SQLException) { + tryToClose(ps, LOG) + throw DbException(e) + } + } + + @Throws(DbException::class) + override fun addContact(txn: Connection, contact: Contact) { + var ps: PreparedStatement? = null + try { + val sql = """INSERT INTO contacts (contactId, token, inbox, outbox) + VALUES (?, ?, ?, ?) + """.trimIndent() + ps = txn.prepareStatement(sql) + ps.setInt(1, contact.id) + ps.setString(2, contact.token) + ps.setString(3, contact.inboxId) + ps.setString(4, contact.outboxId) + val affected = ps.executeUpdate() + if (affected != 1) throw DbStateException() + ps.close() + } catch (e: SQLException) { + tryToClose(ps, LOG) + throw DbException(e) + } + } + + @Throws(DbException::class) + override fun getContact(txn: Connection, id: Int): Contact? { + var ps: PreparedStatement? = null + var rs: ResultSet? = null + try { + val sql = """SELECT token, inbox, outbox FROM contacts + WHERE contactId = ? + """.trimIndent() + ps = txn.prepareStatement(sql) + ps.setInt(1, id) + rs = ps.executeQuery() + if (!rs.next()) return null + val token = rs.getString(1) + val inboxId = rs.getString(2) + val outboxId = rs.getString(3) + rs.close() + ps.close() + return Contact(id, token, inboxId, outboxId) + } catch (e: SQLException) { + tryToClose(rs, LOG) + tryToClose(ps, LOG) + throw DbException(e) + } + } + + @Throws(DbException::class) + override fun removeContact(txn: Connection, id: Int) { + var ps: PreparedStatement? = null + try { + val sql = "DELETE FROM contacts WHERE contactId = ?" + ps = txn.prepareStatement(sql) + ps.setInt(1, id) + val affected = ps.executeUpdate() + if (affected != 1) throw DbStateException() + ps.close() + } catch (e: SQLException) { + tryToClose(ps, LOG) + throw DbException(e) + } + } + +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/JdbcUtils.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/JdbcUtils.kt new file mode 100644 index 0000000000000000000000000000000000000000..8f123e79c2a0fd84e07e6616d0f8910cb8a8c54e --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/JdbcUtils.kt @@ -0,0 +1,36 @@ +package org.briarproject.mailbox.core.db + +import org.briarproject.mailbox.core.util.LogUtils.logException +import org.slf4j.Logger +import java.sql.Connection +import java.sql.ResultSet +import java.sql.SQLException +import java.sql.Statement + +object JdbcUtils { + + fun tryToClose(rs: ResultSet?, logger: Logger) { + try { + rs?.close() + } catch (e: SQLException) { + logException(logger, e) + } + } + + fun tryToClose(s: Statement?, logger: Logger) { + try { + s?.close() + } catch (e: SQLException) { + logException(logger, e) + } + } + + fun tryToClose(c: Connection?, logger: Logger) { + try { + c?.close() + } catch (e: SQLException) { + logException(logger, e) + } + } + +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/Migration.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/Migration.kt new file mode 100644 index 0000000000000000000000000000000000000000..0f4f49972036b08aed1bf2eb820fcf36e4310f64 --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/Migration.kt @@ -0,0 +1,16 @@ +package org.briarproject.mailbox.core.db + +interface Migration<T> { + /** + * Returns the schema version from which this migration starts. + */ + val startVersion: Int + + /** + * Returns the schema version at which this migration ends. + */ + val endVersion: Int + + @Throws(DbException::class) + fun migrate(txn: T) +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/MigrationListener.java b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/MigrationListener.java index e12dece37e94b43206a469b8a5521fd73e475d9b..d9283393f2dcbf5cc09db159546b19c47bf76fbb 100644 --- a/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/MigrationListener.java +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/db/MigrationListener.java @@ -12,4 +12,5 @@ public interface MigrationListener { * This is called when compaction is started while opening the database. */ void onDatabaseCompaction(); + } diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/lifecycle/LifecycleManagerImpl.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/lifecycle/LifecycleManagerImpl.kt index abf568985e09ae21edba2a84c101ac21c1a23ab1..d5e659d0c5c455419179f82bb52ea561e8edd515 100644 --- a/mailbox-core/src/main/java/org/briarproject/mailbox/core/lifecycle/LifecycleManagerImpl.kt +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/lifecycle/LifecycleManagerImpl.kt @@ -84,12 +84,10 @@ internal class LifecycleManagerImpl @Inject constructor(private val db: Database state.value = STARTING_SERVICES dbLatch.countDown() for (s in services) { - LOG.info { "Starting service ${s.javaClass.simpleName}" } start = now() s.startService() - logDuration(LOG, { "Starting service ${s.javaClass.simpleName}" }, start) + logDuration(LOG, { "Starting service ${s.javaClass.simpleName}" }, start) } - LOG.info("All services started") state.compareAndSet(STARTING_SERVICES, RUNNING) startupLatch.countDown() SUCCESS diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/settings/Settings.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/settings/Settings.kt new file mode 100644 index 0000000000000000000000000000000000000000..8e988a443a943dfe829b7becab2dd2e66bd40d5e --- /dev/null +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/settings/Settings.kt @@ -0,0 +1,42 @@ +package org.briarproject.mailbox.core.settings + +import java.util.Hashtable + +class Settings : Hashtable<String, String>() { + + fun getBoolean(key: String, defaultValue: Boolean): Boolean { + val s = get(key) ?: return defaultValue + if ("true" == s) return true + return if ("false" == s) false else defaultValue + } + + fun putBoolean(key: String, value: Boolean) { + put(key, value.toString()) + } + + fun getInt(key: String, defaultValue: Int): Int { + val s = get(key) ?: return defaultValue + return try { + Integer.parseInt(s) + } catch (e: NumberFormatException) { + defaultValue + } + } + + fun putInt(key: String, value: Int) { + put(key, value.toString()) + } + + fun getLong(key: String, defaultValue: Long): Long { + val s = get(key) ?: return defaultValue + return try { + java.lang.Long.parseLong(s) + } catch (e: NumberFormatException) { + defaultValue + } + } + + fun putLong(key: String, value: Long) { + put(key, value.toString()) + } +} diff --git a/mailbox-core/src/main/java/org/briarproject/mailbox/core/util/LogUtils.kt b/mailbox-core/src/main/java/org/briarproject/mailbox/core/util/LogUtils.kt index b209ee11343faf826b2855e565bd889943d84893..46e7614f9c2a5b0ec1df63790aa06b9e7f7a5f23 100644 --- a/mailbox-core/src/main/java/org/briarproject/mailbox/core/util/LogUtils.kt +++ b/mailbox-core/src/main/java/org/briarproject/mailbox/core/util/LogUtils.kt @@ -1,6 +1,7 @@ package org.briarproject.mailbox.core.util import org.slf4j.Logger +import java.io.File object LogUtils { @@ -59,4 +60,25 @@ object LogUtils { fun logException(logger: Logger, t: Throwable) { if (logger.isWarnEnabled) logger.warn(t.toString(), t) } + + fun logFileOrDir(logger: Logger, f: File) { + if (logger.isInfoEnabled) { + if (f.isFile) { + logWithType(logger, f, "F") + } else if (f.isDirectory) { + logWithType(logger, f, "D") + val children = f.listFiles() + if (children != null) { + for (child in children) logFileOrDir(logger, child) + } + } else if (f.exists()) { + logWithType(logger, f, "?") + } + } + } + + fun logWithType(logger: Logger, f: File, type: String) { + logger.info("$type ${f.absolutePath} ${f.length()}") + } + } diff --git a/mailbox-core/src/test/java/org/briarproject/mailbox/core/TestModule.kt b/mailbox-core/src/test/java/org/briarproject/mailbox/core/TestModule.kt index 739283752583549163e7f64425f993acf3faa181..a94240522433410182262f34a31f187f95efb7dd 100644 --- a/mailbox-core/src/test/java/org/briarproject/mailbox/core/TestModule.kt +++ b/mailbox-core/src/test/java/org/briarproject/mailbox/core/TestModule.kt @@ -4,10 +4,12 @@ import dagger.Module import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.components.SingletonComponent +import org.briarproject.mailbox.core.db.DatabaseConfig import org.briarproject.mailbox.core.db.DatabaseModule import org.briarproject.mailbox.core.lifecycle.LifecycleModule import org.briarproject.mailbox.core.server.WebServerModule import org.briarproject.mailbox.core.system.Clock +import java.io.File import javax.inject.Singleton @Module( @@ -23,4 +25,12 @@ internal class TestModule { @Singleton @Provides fun provideClock() = Clock { System.currentTimeMillis() } + + @Singleton + @Provides + fun provideDatabaseConfig() = object : DatabaseConfig { + override fun getDatabaseDirectory(): File { + return File("/tmp") + } + } } diff --git a/mailbox-core/src/test/java/org/briarproject/mailbox/core/TestUtils.kt b/mailbox-core/src/test/java/org/briarproject/mailbox/core/TestUtils.kt new file mode 100644 index 0000000000000000000000000000000000000000..88d0f4806ee3dd1b757d968303e9ff5af0e53503 --- /dev/null +++ b/mailbox-core/src/test/java/org/briarproject/mailbox/core/TestUtils.kt @@ -0,0 +1,23 @@ +package org.briarproject.mailbox.core + +import org.briarproject.mailbox.core.util.IoUtils +import java.io.File +import java.util.concurrent.atomic.AtomicInteger + +object TestUtils { + + private val nextTestDir = AtomicInteger( + (Math.random() * 1000 * 1000).toInt() + ) + + fun getTestDirectory(): File { + val name: Int = nextTestDir.getAndIncrement() + return File("test.tmp/$name") + } + + fun deleteTestDirectory(testDir: File) { + IoUtils.deleteFileOrDir(testDir) + testDir.parentFile.delete() // Delete if empty + } + +} diff --git a/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/H2DatabaseTest.kt b/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/H2DatabaseTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..32485e2522e51c4bb5721f807813214621ce2d0c --- /dev/null +++ b/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/H2DatabaseTest.kt @@ -0,0 +1,11 @@ +package org.briarproject.mailbox.core.db + +import org.briarproject.mailbox.core.system.Clock + +class H2DatabaseTest : JdbcDatabaseTest() { + + override fun createDatabase(config: DatabaseConfig, clock: Clock): JdbcDatabase { + return H2Database(config, clock) + } + +} diff --git a/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/JdbcDatabaseTest.kt b/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/JdbcDatabaseTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..03dff95dcc5c629e8ac8869b63be0a2dd24b0c91 --- /dev/null +++ b/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/JdbcDatabaseTest.kt @@ -0,0 +1,95 @@ +package org.briarproject.mailbox.core.db + +import org.briarproject.mailbox.core.TestUtils.deleteTestDirectory +import org.briarproject.mailbox.core.TestUtils.getTestDirectory +import org.briarproject.mailbox.core.api.Contact +import org.briarproject.mailbox.core.system.Clock +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.io.File +import java.sql.Connection +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +abstract class JdbcDatabaseTest { + + private val testDir: File = getTestDirectory() + + protected abstract fun createDatabase( + config: DatabaseConfig, + clock: Clock, + ): JdbcDatabase + + @BeforeEach + open fun setUp() { + assertTrue(testDir.mkdirs()) + } + + @Throws(java.lang.Exception::class) + fun open( + resume: Boolean, + ): Database<Connection> { + val db: Database<Connection> = createDatabase( + TestDatabaseConfig(testDir) + ) { System.currentTimeMillis() } + if (!resume) deleteTestDirectory(testDir) + db.open(null) + return db + } + + @Test + @Throws(Exception::class) + open fun testPersistence() { + // Store some records + var db: Database<Connection> = open(false) + var txn = db.startTransaction() + + val contact1 = Contact( + 1, + "4291ad1d-897d-4db4-9de9-ea3f78c5262e", + "f21467bd-afb0-4c0e-9090-cae45ea1eae9", + "880629fb-3226-41d8-a978-7b28cf44d57d" + ) + val contact2 = Contact( + 2, + "fbbe9a63-2f28-46d4-a465-e6ca57a5d811", + "7931fa7a-077e-403a-8487-63261027d6d2", + "12a61ca3-af0a-41d1-acc1-a0f4625f6e42" + ) + + db.addContact(txn, contact1) + db.addContact(txn, contact2) + + db.commitTransaction(txn) + db.close() + + // Check that the records are still there + db = open(true) + txn = db.startTransaction() + + val contact1Reloaded1 = db.getContact(txn, 1) + val contact2Reloaded1 = db.getContact(txn, 2) + assertEquals(contact1, contact1Reloaded1) + assertEquals(contact2, contact2Reloaded1) + + // Delete one of the records + db.removeContact(txn, 1) + + db.commitTransaction(txn) + db.close() + + // Check that the record is gone + db = open(true) + txn = db.startTransaction() + + val contact1Reloaded2 = db.getContact(txn, 1) + val contact2Reloaded2 = db.getContact(txn, 2) + assertNull(contact1Reloaded2) + assertEquals(contact2, contact2Reloaded2) + + db.commitTransaction(txn) + db.close() + } + +} diff --git a/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/TestDatabaseConfig.kt b/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/TestDatabaseConfig.kt new file mode 100644 index 0000000000000000000000000000000000000000..5aabc040cf6225cfe3562b9269b0da3291e4e6dc --- /dev/null +++ b/mailbox-core/src/test/java/org/briarproject/mailbox/core/db/TestDatabaseConfig.kt @@ -0,0 +1,13 @@ +package org.briarproject.mailbox.core.db + +import java.io.File + +class TestDatabaseConfig(testDir: File) : DatabaseConfig { + + private val dbDir: File = File(testDir, "db") + + override fun getDatabaseDirectory(): File { + return dbDir + } + +}