narayana/conversations-classic
3
2
Fork 0
Code Issues 3 Pull requests 2 Actions Packages Projects Releases 4 Wiki Activity

add database migration for new fts scheme

Browse source
This commit is contained in:
Daniel Gultsch 2021-09-07 16:47:40 +02:00
parent 754773be55
commit 8b817b3bd8
2 changed files with 42 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java
View file

@ -11,6 +11,8 @@ import android.os.SystemClock;
import android.util.Base64; import android.util.Base64;
import android.util.Log; import android.util.Log;
import com.google.common.base.Stopwatch;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.whispersystems.libsignal.IdentityKey; import org.whispersystems.libsignal.IdentityKey;
@ -62,7 +64,9 @@ import eu.siacs.conversations.xmpp.mam.MamReference;
public class DatabaseBackend extends SQLiteOpenHelper { public class DatabaseBackend extends SQLiteOpenHelper {
private static final String DATABASE_NAME = "history"; private static final String DATABASE_NAME = "history";
private static final int DATABASE_VERSION = 48; private static final int DATABASE_VERSION = 49;
private static boolean requiresMessageIndexRebuild = false;
private static DatabaseBackend instance = null; private static DatabaseBackend instance = null;
private static final String CREATE_CONTATCS_STATEMENT = "create table " private static final String CREATE_CONTATCS_STATEMENT = "create table "
+ Contact.TABLENAME + "(" + Contact.ACCOUNT + " TEXT, " + Contact.TABLENAME + "(" + Contact.ACCOUNT + " TEXT, "
@ -187,6 +191,17 @@ public class DatabaseBackend extends SQLiteOpenHelper {
return values; return values;
} }
public static boolean requiresMessageIndexRebuild() {
return requiresMessageIndexRebuild;
}
public void rebuildMessagesIndex() {
final SQLiteDatabase db = getWritableDatabase();
final Stopwatch stopwatch = Stopwatch.createStarted();
db.execSQL(COPY_PREEXISTING_ENTRIES);
Log.d(Config.LOGTAG,"rebuilt message index in "+ stopwatch.stop().toString());
}
public static synchronized DatabaseBackend getInstance(Context context) { public static synchronized DatabaseBackend getInstance(Context context) {
if (instance == null) { if (instance == null) {
instance = new DatabaseBackend(context); instance = new DatabaseBackend(context);
@ -520,14 +535,6 @@ public class DatabaseBackend extends SQLiteOpenHelper {
db.execSQL(CREATE_RESOLVER_RESULTS_TABLE); db.execSQL(CREATE_RESOLVER_RESULTS_TABLE);
} }
if (oldVersion < 41 && newVersion >= 41) {
db.execSQL(CREATE_MESSAGE_INDEX_TABLE);
db.execSQL(CREATE_MESSAGE_INSERT_TRIGGER);
db.execSQL(CREATE_MESSAGE_UPDATE_TRIGGER);
db.execSQL(CREATE_MESSAGE_DELETE_TRIGGER);
db.execSQL(COPY_PREEXISTING_ENTRIES);
}
if (oldVersion < 42 && newVersion >= 42) { if (oldVersion < 42 && newVersion >= 42) {
db.execSQL("DROP TRIGGER IF EXISTS after_message_delete"); db.execSQL("DROP TRIGGER IF EXISTS after_message_delete");
} }
@ -554,10 +561,10 @@ public class DatabaseBackend extends SQLiteOpenHelper {
if (oldVersion < 46 && newVersion >= 46) { if (oldVersion < 46 && newVersion >= 46) {
final long start = SystemClock.elapsedRealtime(); final long start = SystemClock.elapsedRealtime();
db.rawQuery("PRAGMA secure_delete = FALSE", null).close(); db.rawQuery("PRAGMA secure_delete = FALSE", null).close();
db.execSQL("update "+Message.TABLENAME+" set "+Message.EDITED+"=NULL"); db.execSQL("update " + Message.TABLENAME + " set " + Message.EDITED + "=NULL");
db.rawQuery("PRAGMA secure_delete=ON", null).close(); db.rawQuery("PRAGMA secure_delete=ON", null).close();
final long diff = SystemClock.elapsedRealtime() - start; final long diff = SystemClock.elapsedRealtime() - start;
Log.d(Config.LOGTAG,"deleted old edit information in "+diff+"ms"); Log.d(Config.LOGTAG, "deleted old edit information in " + diff + "ms");
} }
if (oldVersion < 47 && newVersion >= 47) { if (oldVersion < 47 && newVersion >= 47) {
db.execSQL("ALTER TABLE " + Contact.TABLENAME + " ADD COLUMN " + Contact.PRESENCE_NAME + " TEXT"); db.execSQL("ALTER TABLE " + Contact.TABLENAME + " ADD COLUMN " + Contact.PRESENCE_NAME + " TEXT");
@ -565,6 +572,19 @@ public class DatabaseBackend extends SQLiteOpenHelper {
if (oldVersion < 48 && newVersion >= 48) { if (oldVersion < 48 && newVersion >= 48) {
db.execSQL("ALTER TABLE " + Contact.TABLENAME + " ADD COLUMN " + Contact.RTP_CAPABILITY + " TEXT"); db.execSQL("ALTER TABLE " + Contact.TABLENAME + " ADD COLUMN " + Contact.RTP_CAPABILITY + " TEXT");
} }
if (oldVersion < 49 && newVersion >= 49) {
db.beginTransaction();
db.execSQL("DROP TRIGGER IF EXISTS after_message_insert;");
db.execSQL("DROP TRIGGER IF EXISTS after_message_update;");
db.execSQL("DROP TABLE IF EXISTS messages_index;");
db.execSQL(CREATE_MESSAGE_INDEX_TABLE);
db.execSQL(CREATE_MESSAGE_INSERT_TRIGGER);
db.execSQL(CREATE_MESSAGE_UPDATE_TRIGGER);
db.execSQL(CREATE_MESSAGE_DELETE_TRIGGER);
requiresMessageIndexRebuild = true;
db.setTransactionSuccessful();
db.endTransaction();
}
} }
private void canonicalizeJids(SQLiteDatabase db) { private void canonicalizeJids(SQLiteDatabase db) {
@ -779,7 +799,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
try { try {
list.add(0, Message.fromCursor(cursor, conversation)); list.add(0, Message.fromCursor(cursor, conversation));
} catch (Exception e) { } catch (Exception e) {
Log.e(Config.LOGTAG,"unable to restore message"); Log.e(Config.LOGTAG, "unable to restore message");
} }
} }
cursor.close(); cursor.close();
@ -790,14 +810,12 @@ public class DatabaseBackend extends SQLiteOpenHelper {
final SQLiteDatabase db = this.getReadableDatabase(); final SQLiteDatabase db = this.getReadableDatabase();
final StringBuilder SQL = new StringBuilder(); final StringBuilder SQL = new StringBuilder();
final String[] selectionArgs; final String[] selectionArgs;
// TODO: change "ON messages_index.uuid=messages.uuid" to SQL.append("SELECT " + Message.TABLENAME + ".*," + Conversation.TABLENAME + "." + Conversation.CONTACTJID + "," + Conversation.TABLENAME + "." + Conversation.ACCOUNT + "," + Conversation.TABLENAME + "." + Conversation.MODE + " FROM " + Message.TABLENAME + " JOIN " + Conversation.TABLENAME + " ON " + Message.TABLENAME + "." + Message.CONVERSATION + "=" + Conversation.TABLENAME + "." + Conversation.UUID + " JOIN messages_index ON messages_index.rowid=messages.rowid WHERE " + Message.ENCRYPTION + " NOT IN(" + Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE + "," + Message.ENCRYPTION_PGP + "," + Message.ENCRYPTION_DECRYPTION_FAILED + "," + Message.ENCRYPTION_AXOLOTL_FAILED + ") AND " + Message.TYPE + " IN(" + Message.TYPE_TEXT + "," + Message.TYPE_PRIVATE + ") AND messages_index.body MATCH ?");
// "ON messages_index.rowid=messages.rowid" when everyone's migrated.
SQL.append("SELECT " + Message.TABLENAME + ".*," + Conversation.TABLENAME + "." + Conversation.CONTACTJID + "," + Conversation.TABLENAME + "." + Conversation.ACCOUNT + "," + Conversation.TABLENAME + "." + Conversation.MODE + " FROM " + Message.TABLENAME + " JOIN " + Conversation.TABLENAME + " ON " + Message.TABLENAME + "." + Message.CONVERSATION + "=" + Conversation.TABLENAME + "." + Conversation.UUID + " JOIN messages_index ON messages_index.uuid=messages.uuid WHERE " + Message.ENCRYPTION + " NOT IN(" + Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE + "," + Message.ENCRYPTION_PGP + "," + Message.ENCRYPTION_DECRYPTION_FAILED + "," + Message.ENCRYPTION_AXOLOTL_FAILED + ") AND " + Message.TYPE + " IN(" + Message.TYPE_TEXT + "," + Message.TYPE_PRIVATE + ") AND messages_index.body MATCH ?");
if (uuid == null) { if (uuid == null) {
selectionArgs = new String[]{FtsUtils.toMatchString(term)}; selectionArgs = new String[]{FtsUtils.toMatchString(term)};
} else { } else {
selectionArgs = new String[]{FtsUtils.toMatchString(term), uuid}; selectionArgs = new String[]{FtsUtils.toMatchString(term), uuid};
SQL.append(" AND "+Conversation.TABLENAME+'.'+Conversation.UUID+"=?"); SQL.append(" AND " + Conversation.TABLENAME + '.' + Conversation.UUID + "=?");
} }
SQL.append(" ORDER BY " + Message.TIME_SENT + " DESC limit " + Config.MAX_SEARCH_RESULTS); SQL.append(" ORDER BY " + Message.TIME_SENT + " DESC limit " + Config.MAX_SEARCH_RESULTS);
Log.d(Config.LOGTAG, "search term: " + FtsUtils.toMatchString(term)); Log.d(Config.LOGTAG, "search term: " + FtsUtils.toMatchString(term));
@ -861,7 +879,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
public List<FilePathInfo> getFilePathInfo() { public List<FilePathInfo> getFilePathInfo() {
final SQLiteDatabase db = this.getReadableDatabase(); final SQLiteDatabase db = this.getReadableDatabase();
final Cursor cursor = db.query(Message.TABLENAME, new String[]{Message.UUID, Message.RELATIVE_FILE_PATH, Message.DELETED}, "type in (1,2,5) and "+Message.RELATIVE_FILE_PATH+" is not null", null, null, null, null); final Cursor cursor = db.query(Message.TABLENAME, new String[]{Message.UUID, Message.RELATIVE_FILE_PATH, Message.DELETED}, "type in (1,2,5) and " + Message.RELATIVE_FILE_PATH + " is not null", null, null, null, null);
final List<FilePathInfo> list = new ArrayList<>(); final List<FilePathInfo> list = new ArrayList<>();
while (cursor != null && cursor.moveToNext()) { while (cursor != null && cursor.moveToNext()) {
list.add(new FilePathInfo(cursor.getString(0), cursor.getString(1), cursor.getInt(2) > 0)); list.add(new FilePathInfo(cursor.getString(0), cursor.getString(1), cursor.getInt(2) > 0));
@ -874,7 +892,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
public List<FilePath> getRelativeFilePaths(String account, Jid jid, int limit) { public List<FilePath> getRelativeFilePaths(String account, Jid jid, int limit) {
SQLiteDatabase db = this.getReadableDatabase(); SQLiteDatabase db = this.getReadableDatabase();
final String SQL = "select uuid,relativeFilePath from messages where type in (1,2,5) and deleted=0 and "+Message.RELATIVE_FILE_PATH+" is not null and conversationUuid=(select uuid from conversations where accountUuid=? and (contactJid=? or contactJid like ?)) order by timeSent desc"; final String SQL = "select uuid,relativeFilePath from messages where type in (1,2,5) and deleted=0 and " + Message.RELATIVE_FILE_PATH + " is not null and conversationUuid=(select uuid from conversations where accountUuid=? and (contactJid=? or contactJid like ?)) order by timeSent desc";
final String[] args = {account, jid.toString(), jid.toString() + "/%"}; final String[] args = {account, jid.toString(), jid.toString() + "/%"};
Cursor cursor = db.rawQuery(SQL + (limit > 0 ? " limit " + limit : ""), args); Cursor cursor = db.rawQuery(SQL + (limit > 0 ? " limit " + limit : ""), args);
List<FilePath> filesPaths = new ArrayList<>(); List<FilePath> filesPaths = new ArrayList<>();
@ -899,7 +917,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
public boolean deleted; public boolean deleted;
private FilePathInfo(String uuid, String path, boolean deleted) { private FilePathInfo(String uuid, String path, boolean deleted) {
super(uuid,path); super(uuid, path);
this.deleted = deleted; this.deleted = deleted;
} }
@ -1043,9 +1061,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
long start = SystemClock.elapsedRealtime(); long start = SystemClock.elapsedRealtime();
final SQLiteDatabase db = this.getWritableDatabase(); final SQLiteDatabase db = this.getWritableDatabase();
db.beginTransaction(); db.beginTransaction();
String[] args = {conversation.getUuid()}; final String[] args = {conversation.getUuid()};
// TODO: remove once everyone has the after_message_delete trigger
db.delete("messages_index", "uuid in (select uuid from messages where conversationUuid=?)", args);
int num = db.delete(Message.TABLENAME, Message.CONVERSATION + "=?", args); int num = db.delete(Message.TABLENAME, Message.CONVERSATION + "=?", args);
db.setTransactionSuccessful(); db.setTransactionSuccessful();
db.endTransaction(); db.endTransaction();
@ -1055,9 +1071,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
public void expireOldMessages(long timestamp) { public void expireOldMessages(long timestamp) {
final String[] args = {String.valueOf(timestamp)}; final String[] args = {String.valueOf(timestamp)};
SQLiteDatabase db = this.getReadableDatabase(); SQLiteDatabase db = this.getReadableDatabase();
// TODO: remove once everyone has the after_message_delete trigger
db.beginTransaction(); db.beginTransaction();
db.delete("messages_index", "uuid in (select uuid from messages where timeSent<?)", args);
db.delete(Message.TABLENAME, "timeSent<?", args); db.delete(Message.TABLENAME, "timeSent<?", args);
db.setTransactionSuccessful(); db.setTransactionSuccessful();
db.endTransaction(); db.endTransaction();

5
src/main/java/eu/siacs/conversations/services/XmppConnectionService.java
View file

@ -1884,7 +1884,10 @@ public class XmppConnectionService extends Service {
long diffConversationsRestore = SystemClock.elapsedRealtime() - startTimeConversationsRestore; long diffConversationsRestore = SystemClock.elapsedRealtime() - startTimeConversationsRestore;
Log.d(Config.LOGTAG, "finished restoring conversations in " + diffConversationsRestore + "ms"); Log.d(Config.LOGTAG, "finished restoring conversations in " + diffConversationsRestore + "ms");
Runnable runnable = () -> { Runnable runnable = () -> {
long deletionDate = getAutomaticMessageDeletionDate(); if (DatabaseBackend.requiresMessageIndexRebuild()) {
DatabaseBackend.getInstance(this).rebuildMessagesIndex();
}
final long deletionDate = getAutomaticMessageDeletionDate();
mLastExpiryRun.set(SystemClock.elapsedRealtime()); mLastExpiryRun.set(SystemClock.elapsedRealtime());
if (deletionDate > 0) { if (deletionDate > 0) {
Log.d(Config.LOGTAG, "deleting messages that are older than " + AbstractGenerator.getTimestamp(deletionDate)); Log.d(Config.LOGTAG, "deleting messages that are older than " + AbstractGenerator.getTimestamp(deletionDate));