implement self healing omemo

after receiving a SignalMessage that can’t be decrypted because of broken sessions
Conversations will attempt to grab a new pre key bundle and send a new PreKeySignalMessage
wrapped in a key transport message.

src/main/java/eu/siacs/conversations/crypto/axolotl/AxolotlService.java

@@ -82,9 +82,11 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 	private final SerialSingleThreadExecutor executor;
 	private int numPublishTriesOnEmptyPep = 0;
 	private boolean pepBroken = false;
+	private final Set<SignalProtocolAddress> healingAttempts = new HashSet<>();
 	private int lastDeviceListNotificationHash = 0;
 	private final HashSet<Integer> cleanedOwnDeviceIds = new HashSet<>();
 	private Set<XmppAxolotlSession> postponedSessions = new HashSet<>(); //sessions stored here will receive after mam catchup treatment
+	private Set<SignalProtocolAddress> postponedHealing = new HashSet<>(); //addresses stored here will need a healing notification after mam catchup
 
 	private AtomicBoolean changeAccessMode = new AtomicBoolean(false);
 
@@ -390,6 +392,7 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 		this.pepBroken = false;
 		this.numPublishTriesOnEmptyPep = 0;
 		this.lastDeviceListNotificationHash = 0;
+		this.healingAttempts.clear();
 	}
 
 	public void clearErrorsInFetchStatusMap(Jid jid) {
@@ -1071,7 +1074,16 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 		}
 	}
 
+	interface OnSessionBuildFromPep {
+		void onSessionBuildSuccessful();
+		void onSessionBuildFailed();
+	}
+
 	private void buildSessionFromPEP(final SignalProtocolAddress address) {
+		buildSessionFromPEP(address, null);
+	}
+
+	private void buildSessionFromPEP(final SignalProtocolAddress address, OnSessionBuildFromPep callback) {
 		Log.i(Config.LOGTAG, AxolotlService.getLogprefix(account) + "Building new session for " + address.toString());
 		if (address.equals(getOwnAxolotlAddress())) {
 			throw new AssertionError("We should NEVER build a session with ourselves. What happened here?!");
@@ -1092,6 +1104,9 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 					Log.e(Config.LOGTAG, AxolotlService.getLogprefix(account) + "preKey IQ packet invalid: " + packet);
 					fetchStatusMap.put(address, FetchStatus.ERROR);
 					finishBuildingSessionsFromPEP(address);
+					if (callback != null) {
+						callback.onSessionBuildFailed();
+					}
 					return;
 				}
 				Random random = new Random();
@@ -1100,6 +1115,9 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 					//should never happen
 					fetchStatusMap.put(address, FetchStatus.ERROR);
 					finishBuildingSessionsFromPEP(address);
+					if (callback != null) {
+						callback.onSessionBuildFailed();
+					}
 					return;
 				}
 
@@ -1114,7 +1132,7 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 					XmppAxolotlSession session = new XmppAxolotlSession(account, axolotlStore, address, bundle.getIdentityKey());
 					sessions.put(address, session);
 					if (Config.X509_VERIFICATION) {
-						verifySessionWithPEP(session);
+						verifySessionWithPEP(session); //TODO; maybe inject callback in here too
 					} else {
 						FingerprintStatus status = getFingerprintTrust(CryptoHelper.bytesToHex(bundle.getIdentityKey().getPublicKey().serialize()));
 						FetchStatus fetchStatus;
@@ -1127,6 +1145,9 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 						}
 						fetchStatusMap.put(address, fetchStatus);
 						finishBuildingSessionsFromPEP(address);
+						if (callback != null) {
+							callback.onSessionBuildSuccessful();
+						}
 					}
 				} catch (UntrustedIdentityException | InvalidKeyException e) {
 					Log.e(Config.LOGTAG, AxolotlService.getLogprefix(account) + "Error building session for " + address + ": "
@@ -1136,6 +1157,9 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 					if (oneOfOurs && cleanedOwnDeviceIds.add(address.getDeviceId())) {
 						removeFromDeviceAnnouncement(address.getDeviceId());
 					}
+					if (callback != null) {
+						callback.onSessionBuildFailed();
+					}
 				}
 			} else {
 				fetchStatusMap.put(address, FetchStatus.ERROR);
@@ -1146,6 +1170,9 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 				if (oneOfOurs && itemNotFound && cleanedOwnDeviceIds.add(address.getDeviceId())) {
 					removeFromDeviceAnnouncement(address.getDeviceId());
 				}
+				if (callback != null) {
+					callback.onSessionBuildFailed();
+				}
 			}
 		});
 	}
@@ -1391,11 +1418,15 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 	}
 
 	private XmppAxolotlSession getReceivingSession(XmppAxolotlMessage message) {
-		SignalProtocolAddress senderAddress = new SignalProtocolAddress(message.getFrom().toString(),
+		SignalProtocolAddress senderAddress = new SignalProtocolAddress(message.getFrom().toString(), message.getSenderDeviceId());
-				message.getSenderDeviceId());
+		return getReceivingSession(senderAddress);
+
+	}
+
+	private XmppAxolotlSession getReceivingSession(SignalProtocolAddress senderAddress) {
 		XmppAxolotlSession session = sessions.get(senderAddress);
 		if (session == null) {
-			Log.d(Config.LOGTAG, AxolotlService.getLogprefix(account) + "Account: " + account.getJid() + " No axolotl session found while parsing received message " + message);
+			//Log.d(Config.LOGTAG, AxolotlService.getLogprefix(account) + "Account: " + account.getJid() + " No axolotl session found while parsing received message " + message);
 			session = recreateUncachedSession(senderAddress);
 			if (session == null) {
 				session = new XmppAxolotlSession(account, axolotlStore, senderAddress);
@@ -1404,7 +1435,7 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 		return session;
 	}
 
-	public XmppAxolotlMessage.XmppAxolotlPlaintextMessage processReceivingPayloadMessage(XmppAxolotlMessage message, boolean postponePreKeyMessageHandling) throws NotEncryptedForThisDeviceException {
+	public XmppAxolotlMessage.XmppAxolotlPlaintextMessage processReceivingPayloadMessage(XmppAxolotlMessage message, boolean postponePreKeyMessageHandling) throws NotEncryptedForThisDeviceException, BrokenSessionException {
 		XmppAxolotlMessage.XmppAxolotlPlaintextMessage plaintextMessage = null;
 
 		XmppAxolotlSession session = getReceivingSession(message);
@@ -1421,8 +1452,10 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 			} else {
 				throw e;
 			}
+		} catch (final BrokenSessionException e) {
+			throw e;
 		} catch (CryptoFailedException e) {
-			Log.w(Config.LOGTAG, getLogprefix(account) + "Failed to decrypt message from " + message.getFrom() + ": " + e.getMessage());
+			Log.w(Config.LOGTAG, getLogprefix(account) + "Failed to decrypt message from " + message.getFrom(), e);
 		}
 
 		if (session.isFresh() && plaintextMessage != null) {
@@ -1432,6 +1465,35 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 		return plaintextMessage;
 	}
 
+	public void reportBrokenSessionException(BrokenSessionException e, boolean postpone) {
+		Log.e(Config.LOGTAG,account.getJid().asBareJid()+": broken session with "+e.getSignalProtocolAddress().toString()+" detected", e);
+		if (postpone) {
+			postponedHealing.add(e.getSignalProtocolAddress());
+		} else {
+			notifyRequiresHealing(e.getSignalProtocolAddress());
+		}
+	}
+
+	private void notifyRequiresHealing(final SignalProtocolAddress signalProtocolAddress) {
+		if (healingAttempts.add(signalProtocolAddress)) {
+			Log.d(Config.LOGTAG,account.getJid().asBareJid()+": attempt to heal "+signalProtocolAddress);
+			buildSessionFromPEP(signalProtocolAddress, new OnSessionBuildFromPep() {
+				@Override
+				public void onSessionBuildSuccessful() {
+					Log.d(Config.LOGTAG, "successfully build new session from pep after detecting broken session");
+					completeSession(getReceivingSession(signalProtocolAddress));
+				}
+
+				@Override
+				public void onSessionBuildFailed() {
+					Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": unable to build new session from pep after detecting broken session");
+				}
+			});
+		} else {
+			Log.d(Config.LOGTAG,account.getJid().asBareJid()+": do not attempt to heal "+signalProtocolAddress+" again");
+		}
+	}
+
 	private void postPreKeyMessageHandling(final XmppAxolotlSession session, int preKeyId, final boolean postpone) {
 		if (postpone) {
 			postponedSessions.add(session);
@@ -1451,6 +1513,11 @@ public class AxolotlService implements OnAdvancedStreamFeaturesLoaded {
 			completeSession(iterator.next());
 			iterator.remove();
 		}
+		Iterator<SignalProtocolAddress> postponedHealingAttemptsIterator = postponedHealing.iterator();
+		while (postponedHealingAttemptsIterator.hasNext()) {
+			notifyRequiresHealing(postponedHealingAttemptsIterator.next());
+			postponedHealingAttemptsIterator.remove();
+		}
 	}
 
 	private void completeSession(XmppAxolotlSession session) {

src/main/java/eu/siacs/conversations/crypto/axolotl/BrokenSessionException.java

@@ -0,0 +1,18 @@
+package eu.siacs.conversations.crypto.axolotl;
+
+import org.whispersystems.libsignal.SignalProtocolAddress;
+
+public class BrokenSessionException extends CryptoFailedException {
+
+    private final SignalProtocolAddress signalProtocolAddress;
+
+    public BrokenSessionException(SignalProtocolAddress address, Exception e) {
+        super(e);
+        this.signalProtocolAddress = address;
+
+    }
+
+    public SignalProtocolAddress getSignalProtocolAddress() {
+        return signalProtocolAddress;
+    }
+}

src/main/java/eu/siacs/conversations/crypto/axolotl/CryptoFailedException.java

@@ -6,6 +6,10 @@ public class CryptoFailedException extends Exception {
 		super(msg);
 	}
 
+	public CryptoFailedException(String msg, Exception e) {
+		super(msg, e);
+	}
+
 	public CryptoFailedException(Exception e){
 		super(e);
 	}

src/main/java/eu/siacs/conversations/crypto/axolotl/XmppAxolotlSession.java

@@ -79,7 +79,7 @@ public class XmppAxolotlSession implements Comparable<XmppAxolotlSession> {
 	}
 
 	@Nullable
-	public byte[] processReceiving(AxolotlKey encryptedKey) throws CryptoFailedException {
+	byte[] processReceiving(AxolotlKey encryptedKey) throws CryptoFailedException {
 		byte[] plaintext;
 		FingerprintStatus status = getTrust();
 		if (!status.isCompromised()) {
@@ -99,14 +99,15 @@ public class XmppAxolotlSession implements Comparable<XmppAxolotlSession> {
                     plaintext = cipher.decrypt(preKeySignalMessage);
                 } else {
                     SignalMessage signalMessage = new SignalMessage(encryptedKey.key);
+                    try {
                         plaintext = cipher.decrypt(signalMessage);
+                    } catch (InvalidMessageException | NoSessionException e) {
+                        throw new BrokenSessionException(this.remoteAddress, e);
+                    }
                     preKeyId = null; //better safe than sorry because we use that to do special after prekey handling
                 }
-			} catch (InvalidVersionException | InvalidKeyException | LegacyMessageException | InvalidMessageException | DuplicateMessageException | NoSessionException | InvalidKeyIdException | UntrustedIdentityException e) {
+			} catch (InvalidVersionException | InvalidKeyException | LegacyMessageException | InvalidMessageException | DuplicateMessageException | InvalidKeyIdException | UntrustedIdentityException e) {
-				if (!(e instanceof DuplicateMessageException)) {
+				throw new CryptoFailedException("Error decrypting SignalMessage", e);
-					e.printStackTrace();
-				}
-				throw new CryptoFailedException("Error decrypting WhisperMessage " + e.getClass().getSimpleName() + ": " + e.getMessage());
 			}
 			if (!status.isActive()) {
 				setTrust(status.toActive());

src/main/java/eu/siacs/conversations/entities/Conversation.java

@@ -702,6 +702,20 @@ public class Conversation extends AbstractEntity implements Blockable, Comparabl
 		}
 	}
 
+	public boolean possibleDuplicate(final String serverMsgId, final String remoteMsgId) {
+		if (serverMsgId == null || remoteMsgId == null) {
+			return false;
+		}
+		synchronized (this.messages) {
+			for(Message message : this.messages) {
+				if (serverMsgId.equals(message.getServerMsgId()) || remoteMsgId.equals(message.getRemoteMsgId())) {
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
 	public MamReference getLastMessageTransmitted() {
 		final MamReference lastClear = getLastClearHistory();
 		MamReference lastReceived = new MamReference(0);

src/main/java/eu/siacs/conversations/entities/Message.java

@@ -47,6 +47,7 @@ public class Message extends AbstractEntity {
 	public static final int ENCRYPTION_DECRYPTION_FAILED = 4;
 	public static final int ENCRYPTION_AXOLOTL = 5;
 	public static final int ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE = 6;
+	public static final int ENCRYPTION_AXOLOTL_FAILED = 7;
 
 	public static final int TYPE_TEXT = 0;
 	public static final int TYPE_IMAGE = 1;
@@ -883,7 +884,7 @@ public class Message extends AbstractEntity {
 		if (encryption == ENCRYPTION_DECRYPTED || encryption == ENCRYPTION_DECRYPTION_FAILED) {
 			return ENCRYPTION_PGP;
 		}
-		if (encryption == ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE) {
+		if (encryption == ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE || encryption == ENCRYPTION_AXOLOTL_FAILED) {
 			return ENCRYPTION_AXOLOTL;
 		}
 		return encryption;

src/main/java/eu/siacs/conversations/parser/MessageParser.java

@@ -16,6 +16,7 @@ import java.util.UUID;
 import eu.siacs.conversations.Config;
 import eu.siacs.conversations.R;
 import eu.siacs.conversations.crypto.axolotl.AxolotlService;
+import eu.siacs.conversations.crypto.axolotl.BrokenSessionException;
 import eu.siacs.conversations.crypto.axolotl.NotEncryptedForThisDeviceException;
 import eu.siacs.conversations.crypto.axolotl.XmppAxolotlMessage;
 import eu.siacs.conversations.entities.Account;
@@ -107,7 +108,7 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece
         return false;
     }
 
-    private Message parseAxolotlChat(Element axolotlMessage, Jid from, Conversation conversation, int status, boolean postpone) {
+    private Message parseAxolotlChat(Element axolotlMessage, Jid from, Conversation conversation, int status, boolean checkedForDuplicates, boolean postpone) {
         final AxolotlService service = conversation.getAccount().getAxolotlService();
         final XmppAxolotlMessage xmppAxolotlMessage;
         try {
@@ -120,6 +121,14 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece
             final XmppAxolotlMessage.XmppAxolotlPlaintextMessage plaintextMessage;
             try {
                 plaintextMessage = service.processReceivingPayloadMessage(xmppAxolotlMessage, postpone);
+            } catch (BrokenSessionException e) {
+                if (checkedForDuplicates) {
+                    service.reportBrokenSessionException(e, postpone);
+                    return new Message(conversation, "", Message.ENCRYPTION_AXOLOTL_FAILED, status);
+                } else {
+                    Log.d(Config.LOGTAG,"ignoring broken session exception because checkForDuplicase failed");
+                    return null;
+                }
             } catch (NotEncryptedForThisDeviceException e) {
                 return new Message(conversation, "", Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE, status);
             }
@@ -424,12 +433,15 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece
                     fallbacksBySourceId = Collections.emptySet();
                     origin = from;
                 }
+
+                final boolean checkedForDuplicates = serverMsgId != null && remoteMsgId != null && !conversation.possibleDuplicate(serverMsgId, remoteMsgId);
+
                 if (origin != null) {
-                    message = parseAxolotlChat(axolotlEncrypted, origin, conversation, status, query != null);
+                    message = parseAxolotlChat(axolotlEncrypted, origin, conversation, status,  checkedForDuplicates,query != null);
                 } else {
                     Message trial = null;
                     for (Jid fallback : fallbacksBySourceId) {
-                        trial = parseAxolotlChat(axolotlEncrypted, fallback, conversation, status, query != null);
+                        trial = parseAxolotlChat(axolotlEncrypted, fallback, conversation, status, checkedForDuplicates && fallbacksBySourceId.size() == 1, query != null);
                         if (trial != null) {
                             Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": decoded muc message using fallback");
                             origin = fallback;
@@ -606,7 +618,7 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece
 
             if (message.getEncryption() == Message.ENCRYPTION_PGP) {
                 notify = conversation.getAccount().getPgpDecryptionService().decrypt(message, notify);
-            } else if (message.getEncryption() == Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE) {
+            } else if (message.getEncryption() == Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE || message.getEncryption() == Message.ENCRYPTION_AXOLOTL_FAILED) {
                 notify = false;
             }
 

src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java

@@ -751,7 +751,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
 
 	public Cursor getMessageSearchCursor(List<String> term) {
 		SQLiteDatabase db = this.getReadableDatabase();
-		String SQL = "SELECT " + Message.TABLENAME + ".*," + Conversation.TABLENAME + '.' + Conversation.CONTACTJID + ',' + Conversation.TABLENAME + '.' + Conversation.ACCOUNT + ',' + Conversation.TABLENAME + '.' + Conversation.MODE + " FROM " + Message.TABLENAME + " join " + Conversation.TABLENAME + " on " + Message.TABLENAME + '.' + Message.CONVERSATION + '=' + Conversation.TABLENAME + '.' + Conversation.UUID + " join messages_index ON messages_index.uuid=messages.uuid where " + Message.ENCRYPTION + " NOT IN(" + Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE + ',' + Message.ENCRYPTION_PGP + ',' + Message.ENCRYPTION_DECRYPTION_FAILED + ") AND " + Message.TYPE + " IN(" + Message.TYPE_TEXT + ',' + Message.TYPE_PRIVATE + ") AND messages_index.body MATCH ? ORDER BY " + Message.TIME_SENT + " DESC limit " + Config.MAX_SEARCH_RESULTS;
+		String SQL = "SELECT " + Message.TABLENAME + ".*," + Conversation.TABLENAME + '.' + Conversation.CONTACTJID + ',' + Conversation.TABLENAME + '.' + Conversation.ACCOUNT + ',' + Conversation.TABLENAME + '.' + Conversation.MODE + " FROM " + Message.TABLENAME + " join " + Conversation.TABLENAME + " on " + Message.TABLENAME + '.' + Message.CONVERSATION + '=' + Conversation.TABLENAME + '.' + Conversation.UUID + " join messages_index ON messages_index.uuid=messages.uuid where " + Message.ENCRYPTION + " NOT IN(" + Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE + ',' + Message.ENCRYPTION_PGP + ',' + Message.ENCRYPTION_DECRYPTION_FAILED + ','+Message.ENCRYPTION_AXOLOTL_FAILED+") AND " + Message.TYPE + " IN(" + Message.TYPE_TEXT + ',' + Message.TYPE_PRIVATE + ") AND messages_index.body MATCH ? ORDER BY " + Message.TIME_SENT + " DESC limit " + Config.MAX_SEARCH_RESULTS;
 		Log.d(Config.LOGTAG, "search term: " + FtsUtils.toMatchString(term));
 		return db.rawQuery(SQL, new String[]{FtsUtils.toMatchString(term)});
 	}

src/main/java/eu/siacs/conversations/ui/ConversationFragment.java

@@ -1096,7 +1096,7 @@ public class ConversationFragment extends XmppFragment implements EditMessage.Ke
         }
         if (m.getType() != Message.TYPE_STATUS) {
 
-            if (m.getEncryption() == Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE) {
+            if (m.getEncryption() == Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE || m.getEncryption() == Message.ENCRYPTION_AXOLOTL_FAILED) {
                 return;
             }
 

src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java

@@ -775,6 +775,8 @@ public class MessageAdapter extends ArrayAdapter<Message> implements CopyTextVie
 			displayInfoMessage(viewHolder, activity.getString(R.string.decryption_failed), darkBackground);
 		} else if (message.getEncryption() == Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE) {
 			displayInfoMessage(viewHolder, activity.getString(R.string.not_encrypted_for_this_device), darkBackground);
+		} else if (message.getEncryption() == Message.ENCRYPTION_AXOLOTL_FAILED) {
+			displayInfoMessage(viewHolder, activity.getString(R.string.omemo_decryption_failed), darkBackground);
 		} else {
 			if (message.isGeoUri()) {
 				displayLocationMessage(viewHolder, message);

src/main/java/eu/siacs/conversations/utils/CryptoHelper.java

@@ -264,6 +264,7 @@ public final class CryptoHelper {
                 return R.string.encryption_choice_otr;
             case Message.ENCRYPTION_AXOLOTL:
             case Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE:
+            case Message.ENCRYPTION_AXOLOTL_FAILED:
                 return R.string.encryption_choice_omemo;
             case Message.ENCRYPTION_NONE:
                 return R.string.encryption_choice_unencrypted;

src/main/java/eu/siacs/conversations/utils/UIHelper.java

@@ -292,6 +292,8 @@ public class UIHelper {
 			return new Pair<>(context.getString(R.string.decryption_failed), true);
 		} else if (message.getEncryption() == Message.ENCRYPTION_AXOLOTL_NOT_FOR_THIS_DEVICE) {
 			return new Pair<>(context.getString(R.string.not_encrypted_for_this_device), true);
+		} else if (message.getEncryption() == Message.ENCRYPTION_AXOLOTL_FAILED) {
+			return new Pair<>(context.getString(R.string.omemo_decryption_failed), true);
 		} else if (message.getType() == Message.TYPE_FILE || message.getType() == Message.TYPE_IMAGE) {
 			return new Pair<>(getFileDescriptionString(context, message), true);
 		} else {

src/main/res/values/strings.xml

@@ -697,6 +697,7 @@
     <string name="medium">Medium</string>
     <string name="large">Large</string>
     <string name="not_encrypted_for_this_device">Message was not encrypted for this device.</string>
+    <string name="omemo_decryption_failed">Failed to decrypt OMEMO message.</string>
     <string name="undo">undo</string>
     <string name="location_disabled">Location sharing is disabled</string>
     <string name="action_fix_to_location">Fix position</string>