go through mam history page by page. load mam dynamically on scroll

src/main/java/eu/siacs/conversations/Config.java

@@ -18,13 +18,15 @@ public final class Config {
 
 	public static final int MESSAGE_MERGE_WINDOW = 20;
 
+	public static final int PAGE_SIZE = 50;
+
 	public static final int  PROGRESS_UI_UPDATE_INTERVAL = 750;
 
 	public static final boolean NO_PROXY_LOOKUP = false; //useful to debug ibb
 
 	private static final long MILLISECONDS_IN_DAY = 24 * 60 * 60 * 1000;
-	public static final long MAX_HISTORY_AGE = 7 * MILLISECONDS_IN_DAY;
+	public static final long MAM_MAX_CATCHUP =  MILLISECONDS_IN_DAY / 2;
-	public static final long MAX_CATCHUP =  MILLISECONDS_IN_DAY / 2;
+	public static final int MAM_MAX_MESSAGES = 500;
 
 	private Config() {
 

src/main/java/eu/siacs/conversations/parser/MessageParser.java

@@ -324,11 +324,8 @@ public class MessageParser extends AbstractParser implements
 		finishedMessage.setCounterpart(counterpart);
 		finishedMessage.setRemoteMsgId(message.getAttribute("id"));
 		finishedMessage.setServerMsgId(result.getAttribute("id"));
-		if (conversation.hasDuplicateMessage(finishedMessage)) {
+		if (query!=null) {
-			Log.d(Config.LOGTAG, "received mam message " + content+ " (duplicate)");
+			query.incrementCount();
-			return null;
-		} else {
-			Log.d(Config.LOGTAG, "received mam message " + content);
 		}
 		return finishedMessage;
 	}

src/main/java/eu/siacs/conversations/services/MessageArchiveService.java

@@ -12,7 +12,6 @@ import eu.siacs.conversations.Config;
 import eu.siacs.conversations.entities.Account;
 import eu.siacs.conversations.entities.Conversation;
 import eu.siacs.conversations.generator.AbstractGenerator;
-import eu.siacs.conversations.parser.AbstractParser;
 import eu.siacs.conversations.xml.Element;
 import eu.siacs.conversations.xmpp.OnAdvancedStreamFeaturesLoaded;
 import eu.siacs.conversations.xmpp.OnIqPacketReceived;
@@ -40,8 +39,8 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 		long endCatchup = account.getXmppConnection().getLastSessionEstablished();
 		if (startCatchup == 0) {
 			return;
-		} else if (endCatchup - startCatchup >= Config.MAX_CATCHUP) {
+		} else if (endCatchup - startCatchup >= Config.MAM_MAX_CATCHUP) {
-			startCatchup = endCatchup - Config.MAX_CATCHUP;
+			startCatchup = endCatchup - Config.MAM_MAX_CATCHUP;
 			List<Conversation> conversations = mXmppConnectionService.getConversations();
 			for (Conversation conversation : conversations) {
 				if (conversation.getMode() == Conversation.MODE_SINGLE && conversation.getAccount() == account && startCatchup > conversation.getLastMessageTransmitted()) {
@@ -67,22 +66,23 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 		return timestamp;
 	}
 
-	public void query(final Conversation conversation) {
+	public Query query(final Conversation conversation) {
-		query(conversation,conversation.getAccount().getXmppConnection().getLastSessionEstablished());
+		return query(conversation,conversation.getAccount().getXmppConnection().getLastSessionEstablished());
 	}
 
-	public void query(final Conversation conversation, long end) {
+	public Query query(final Conversation conversation, long end) {
+		return this.query(conversation,conversation.getLastMessageTransmitted(),end);
+	}
+
+	public Query query(Conversation conversation, long start, long end) {
 		synchronized (this.queries) {
-			final Account account = conversation.getAccount();
-			long start = conversation.getLastMessageTransmitted();
 			if (start > end) {
-				return;
+				return null;
-			} else if (end - start >= Config.MAX_HISTORY_AGE) {
-				start = end - Config.MAX_HISTORY_AGE;
 			}
 			final Query query = new Query(conversation, start, end,PagingOrder.REVERSE);
 			this.queries.add(query);
 			this.execute(query);
+			return query;
 		}
 	}
 
@@ -133,7 +133,11 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 			if (conversation.setLastMessageTransmitted(query.getEnd())) {
 				this.mXmppConnectionService.databaseBackend.updateConversation(conversation);
 			}
+			if (query.hasCallback()) {
+				query.callback();
+			} else {
 				this.mXmppConnectionService.updateConversationUi();
+			}
 		} else {
 			for(Conversation tmp : this.mXmppConnectionService.getConversations()) {
 				if (tmp.getAccount() == query.getAccount()) {
@@ -170,8 +174,10 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 		Element last = set == null ? null : set.findChild("last");
 		Element first = set == null ? null : set.findChild("first");
 		Element relevant = query.getPagingOrder() == PagingOrder.NORMAL ? last : first;
-		if (complete || relevant == null) {
+		boolean abort = (query.getStart() == 0 && query.getTotalCount() >= Config.PAGE_SIZE) || query.getTotalCount() >= Config.MAM_MAX_MESSAGES;
+		if (complete || relevant == null || abort) {
 			this.finalizeQuery(query);
+			Log.d(Config.LOGTAG,query.getAccount().getJid().toBareJid().toString()+": finished mam after "+query.getTotalCount()+" messages");
 		} else {
 			final Query nextQuery;
 			if (query.getPagingOrder() == PagingOrder.NORMAL) {
@@ -210,6 +216,8 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 	}
 
 	public class Query {
+		private int totalCount = 0;
+		private int count = 0;
 		private long start;
 		private long end;
 		private Jid with = null;
@@ -218,6 +226,7 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 		private Account account;
 		private Conversation conversation;
 		private PagingOrder pagingOrder = PagingOrder.NORMAL;
+		private XmppConnectionService.OnMoreMessagesLoaded callback = null;
 
 
 		public Query(Conversation conversation, long start, long end) {
@@ -243,6 +252,8 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 			query.reference = reference;
 			query.conversation = conversation;
 			query.with = with;
+			query.totalCount = totalCount;
+			query.callback = callback;
 			return query;
 		}
 
@@ -278,6 +289,16 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 			return start;
 		}
 
+		public void setCallback(XmppConnectionService.OnMoreMessagesLoaded callback) {
+			this.callback = callback;
+		}
+
+		public void callback() {
+			if (this.callback != null) {
+				this.callback.onMoreMessagesLoaded(count,conversation);
+			}
+		}
+
 		public long getEnd() {
 			return end;
 		}
@@ -290,6 +311,15 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 			return this.account;
 		}
 
+		public void incrementCount() {
+			this.count++;
+			this.totalCount++;
+		}
+
+		public int getTotalCount() {
+			return this.totalCount;
+		}
+
 		@Override
 		public String toString() {
 			StringBuilder builder = new StringBuilder();
@@ -313,5 +343,9 @@ public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
 			}
 			return builder.toString();
 		}
+
+		public boolean hasCallback() {
+			return this.callback != null;
+		}
 	}
 }

src/main/java/eu/siacs/conversations/services/XmppConnectionService.java

@@ -246,8 +246,7 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 			account.pendingConferenceLeaves.clear();
 			fetchRosterFromServer(account);
 			fetchBookmarks(account);
-			sendPresencePacket(account,
+			sendPresencePacket(account,mPresenceGenerator.sendPresence(account));
-					mPresenceGenerator.sendPresence(account));
 			connectMultiModeConversations(account);
 			updateConversationUi();
 		}
@@ -893,11 +892,11 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 				accountLookupTable.put(account.getUuid(), account);
 			}
 			this.conversations.addAll(databaseBackend.getConversations(Conversation.STATUS_AVAILABLE));
-			for (Conversation conv : this.conversations) {
+			for (Conversation conversation : this.conversations) {
-				Account account = accountLookupTable.get(conv.getAccountUuid());
+				Account account = accountLookupTable.get(conversation.getAccountUuid());
-				conv.setAccount(account);
+				conversation.setAccount(account);
-				conv.addAll(0, databaseBackend.getMessages(conv, 50));
+				conversation.addAll(0, databaseBackend.getMessages(conversation, Config.PAGE_SIZE));
-				checkDeletedFiles(conv);
+				checkDeletedFiles(conversation);
 			}
 		}
 	}
@@ -962,17 +961,29 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 		});
 	}
 
-	public int loadMoreMessages(Conversation conversation, long timestamp) {
+	public void loadMoreMessages(Conversation conversation, long timestamp, final OnMoreMessagesLoaded callback) {
 		if (this.getMessageArchiveService().queryInProgress(conversation)) {
-			return 0;
+			Log.d(Config.LOGTAG,"query in progress");
+			return;
+		}
+		List<Message> messages = databaseBackend.getMessages(conversation, 50,timestamp);
+		if (messages.size() == 0 && (conversation.getAccount().getXmppConnection() != null && conversation.getAccount().getXmppConnection().getFeatures().mam())) {
+			Log.d(Config.LOGTAG,"load more messages with mam");
+			MessageArchiveService.Query query = getMessageArchiveService().query(conversation,0,timestamp - 1);
+			if (query != null) {
+				query.setCallback(callback);
+			}
+			return;
 		}
-		List<Message> messages = databaseBackend.getMessages(conversation, 50,
-				timestamp);
 		for (Message message : messages) {
 			message.setConversation(conversation);
 		}
 		conversation.addAll(0, messages);
-		return messages.size();
+		callback.onMoreMessagesLoaded(messages.size(),conversation);
+	}
+
+	public interface OnMoreMessagesLoaded {
+		public void onMoreMessagesLoaded(int count,Conversation conversation);
 	}
 
 	public List<Account> getAccounts() {
@@ -1022,7 +1033,7 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 				} else {
 					conversation.setMode(Conversation.MODE_SINGLE);
 				}
-				conversation.addAll(0, databaseBackend.getMessages(conversation, 50));
+				conversation.addAll(0, databaseBackend.getMessages(conversation, Config.PAGE_SIZE));
 				this.databaseBackend.updateConversation(conversation);
 			} else {
 				String conversationName;
@@ -1244,11 +1255,12 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 		Log.d(Config.LOGTAG, "app switched into background");
 	}
 
-	public void connectMultiModeConversations(Account account) {
+	private void connectMultiModeConversations(Account account) {
 		List<Conversation> conversations = getConversations();
 		for (Conversation conversation : conversations) {
 			if ((conversation.getMode() == Conversation.MODE_MULTI)
 					&& (conversation.getAccount() == account)) {
+				conversation.resetMucOptions();
 				joinMuc(conversation);
 			}
 		}

src/main/java/eu/siacs/conversations/ui/ConversationFragment.java

@@ -9,6 +9,7 @@ import android.content.Intent;
 import android.content.IntentSender;
 import android.content.IntentSender.SendIntentException;
 import android.os.Bundle;
+import android.util.Log;
 import android.view.ContextMenu;
 import android.view.ContextMenu.ContextMenuInfo;
 import android.view.Gravity;
@@ -38,6 +39,7 @@ import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
+import eu.siacs.conversations.Config;
 import eu.siacs.conversations.R;
 import eu.siacs.conversations.crypto.PgpEngine;
 import eu.siacs.conversations.entities.Account;
@@ -104,6 +106,7 @@ public class ConversationFragment extends Fragment {
 	private TextView snackbarMessage;
 	private TextView snackbarAction;
 	private boolean messagesLoaded = false;
+
 	private OnScrollListener mOnScrollListener = new OnScrollListener() {
 
 		@Override
@@ -119,14 +122,30 @@ public class ConversationFragment extends Fragment {
 				if (firstVisibleItem == 0 && messagesLoaded) {
 					long timestamp = ConversationFragment.this.messageList.get(0).getTimeSent();
 					messagesLoaded = false;
-					int size = activity.xmppConnectionService.loadMoreMessages(conversation, timestamp);
+					Log.d(Config.LOGTAG,"load more messages");
-					conversation.populateWithMessages(ConversationFragment.this.messageList);
+					activity.xmppConnectionService.loadMoreMessages(conversation, timestamp, new XmppConnectionService.OnMoreMessagesLoaded() {
+						@Override
+						public void onMoreMessagesLoaded(final int count, Conversation conversation) {
+							if (ConversationFragment.this.conversation != conversation) {
+								return;
+							}
+							activity.runOnUiThread(new Runnable() {
+								@Override
+								public void run() {
+									int firstItem = messagesView.getFirstVisiblePosition();
+									Log.d(Config.LOGTAG, "done loading more messages. first item: " + firstItem);
+									ConversationFragment.this.conversation.populateWithMessages(ConversationFragment.this.messageList);
 									updateStatusMessages();
 									messageListAdapter.notifyDataSetChanged();
-					if (size != 0) {
+									if (count != 0) {
 										messagesLoaded = true;
 									}
-					messagesView.setSelectionFromTop(size + 1, 0);
+									messagesView.setSelectionFromTop(firstItem + count, 0);
+								}
+							});
+						}
+					});
+
 				}
 			}
 		}
@@ -580,6 +599,7 @@ public class ConversationFragment extends Fragment {
 					}
 				}
 				conversation.populateWithMessages(ConversationFragment.this.messageList);
+				this.messagesLoaded = this.messageList.size() > 0;
 				for (Message message : this.messageList) {
 					if (message.getEncryption() == Message.ENCRYPTION_PGP
 							&& (message.getStatus() == Message.STATUS_RECEIVED || message