Merge branch 'feature/mam' into development

Conflicts:
	src/main/java/eu/siacs/conversations/services/XmppConnectionService.java

src/main/java/eu/siacs/conversations/Config.java

@@ -22,6 +22,10 @@ public final class Config {
 
 	public static final boolean NO_PROXY_LOOKUP = false; //useful to debug ibb
 
+	private static final long MILLISECONDS_IN_DAY = 24 * 60 * 60 * 1000;
+	public static final long MAX_HISTORY_AGE = 7 * MILLISECONDS_IN_DAY;
+	public static final long MAX_CATCHUP =  MILLISECONDS_IN_DAY / 2;
+
 	private Config() {
 
 	}

src/main/java/eu/siacs/conversations/crypto/OtrEngine.java

@@ -180,6 +180,7 @@ public class OtrEngine implements OtrEngineHost {
 		packet.setBody(body);
 		packet.addChild("private", "urn:xmpp:carbons:2");
 		packet.addChild("no-copy", "urn:xmpp:hints");
+		packet.addChild("no-store", "urn:xmpp:hints");
 		packet.setType(MessagePacket.TYPE_CHAT);
 		account.getXmppConnection().sendMessagePacket(packet);
 	}

src/main/java/eu/siacs/conversations/entities/AbstractEntity.java

@@ -17,5 +17,4 @@ public abstract class AbstractEntity {
 	public boolean equals(AbstractEntity entity) {
 		return this.getUuid().equals(entity.getUuid());
 	}
-
 }

src/main/java/eu/siacs/conversations/entities/Bookmark.java

@@ -102,9 +102,7 @@ public class Bookmark extends Element implements ListItem {
 	}
 
 	public boolean autojoin() {
-		String autojoin = this.getAttribute("autojoin");
+		return this.getAttributeAsBoolean("autojoin");
-		return (autojoin != null && (autojoin.equalsIgnoreCase("true") || autojoin
-				.equalsIgnoreCase("1")));
 	}
 
 	public String getPassword() {

src/main/java/eu/siacs/conversations/entities/Conversation.java

@@ -3,6 +3,7 @@ package eu.siacs.conversations.entities;
 import android.content.ContentValues;
 import android.database.Cursor;
 import android.os.SystemClock;
+import android.util.Log;
 
 import net.java.otr4j.OtrException;
 import net.java.otr4j.crypto.OtrCryptoEngineImpl;
@@ -16,8 +17,11 @@ import org.json.JSONObject;
 
 import java.security.interfaces.DSAPublicKey;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 
+import eu.siacs.conversations.Config;
 import eu.siacs.conversations.xmpp.jid.InvalidJidException;
 import eu.siacs.conversations.xmpp.jid.Jid;
 
@@ -43,6 +47,7 @@ public class Conversation extends AbstractEntity {
 	public static final String ATTRIBUTE_NEXT_ENCRYPTION = "next_encryption";
 	public static final String ATTRIBUTE_MUC_PASSWORD = "muc_password";
 	public static final String ATTRIBUTE_MUTED_TILL = "muted_till";
+	public static final String ATTRIBUTE_LAST_MESSAGE_TRANSMITTED = "last_message_transmitted";
 
 	private String name;
 	private String contactUuid;
@@ -470,6 +475,31 @@ public class Conversation extends AbstractEntity {
 		}
 	}
 
+	public boolean setLastMessageTransmitted(long value) {
+		long before = getLastMessageTransmitted();
+		if (value - before > 1000) {
+			this.setAttribute(ATTRIBUTE_LAST_MESSAGE_TRANSMITTED, String.valueOf(value));
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	public long getLastMessageTransmitted() {
+		long timestamp = getLongAttribute(ATTRIBUTE_LAST_MESSAGE_TRANSMITTED,0);
+		if (timestamp == 0) {
+			synchronized (this.messages) {
+				for(int i = this.messages.size() - 1; i >= 0; --i) {
+					Message message = this.messages.get(i);
+					if (message.getStatus() == Message.STATUS_RECEIVED) {
+						return message.getTimeSent();
+					}
+				}
+			}
+		}
+		return timestamp;
+	}
+
 	public void setMutedTill(long value) {
 		this.setAttribute(ATTRIBUTE_MUTED_TILL, String.valueOf(value));
 	}
@@ -535,6 +565,26 @@ public class Conversation extends AbstractEntity {
 		}
 	}
 
+	public void sort() {
+		synchronized (this.messages) {
+			for(Message message : this.messages) {
+				message.untie();
+			}
+			Collections.sort(this.messages,new Comparator<Message>() {
+				@Override
+				public int compare(Message left, Message right) {
+					if (left.getTimeSent() < right.getTimeSent()) {
+						return -1;
+					} else if (left.getTimeSent() > right.getTimeSent()) {
+						return 1;
+					} else {
+						return 0;
+					}
+				}
+			});
+		}
+	}
+
 	public class Smp {
 		public static final int STATUS_NONE = 0;
 		public static final int STATUS_CONTACT_REQUESTED = 1;

src/main/java/eu/siacs/conversations/entities/Message.java

@@ -45,6 +45,7 @@ public class Message extends AbstractEntity {
 	public static String STATUS = "status";
 	public static String TYPE = "type";
 	public static String REMOTE_MSG_ID = "remoteMsgId";
+	public static String SERVER_MSG_ID = "serverMsgId";
 	public static String RELATIVE_FILE_PATH = "relativeFilePath";
 	public boolean markable = false;
 	protected String conversationUuid;
@@ -59,6 +60,7 @@ public class Message extends AbstractEntity {
 	protected String relativeFilePath;
 	protected boolean read = true;
 	protected String remoteMsgId = null;
+	protected String serverMsgId = null;
 	protected Conversation conversation = null;
 	protected Downloadable downloadable = null;
 	private Message mNextMessage = null;
@@ -83,13 +85,15 @@ public class Message extends AbstractEntity {
 				status,
 				TYPE_TEXT,
 				null,
+				null,
 				null);
 		this.conversation = conversation;
 	}
 
 	private Message(final String uuid, final String conversationUUid, final Jid counterpart,
 				   final Jid trueCounterpart, final String body, final long timeSent,
-				   final int encryption, final int status, final int type, final String remoteMsgId, final String relativeFilePath) {
+				   final int encryption, final int status, final int type, final String remoteMsgId,
+				   final String relativeFilePath, final String serverMsgId) {
 		this.uuid = uuid;
 		this.conversationUuid = conversationUUid;
 		this.counterpart = counterpart;
@@ -101,6 +105,7 @@ public class Message extends AbstractEntity {
 		this.type = type;
 		this.remoteMsgId = remoteMsgId;
 		this.relativeFilePath = relativeFilePath;
+		this.serverMsgId = serverMsgId;
 	}
 
 	public static Message fromCursor(Cursor cursor) {
@@ -136,7 +141,8 @@ public class Message extends AbstractEntity {
 				cursor.getInt(cursor.getColumnIndex(STATUS)),
 				cursor.getInt(cursor.getColumnIndex(TYPE)),
 				cursor.getString(cursor.getColumnIndex(REMOTE_MSG_ID)),
-				cursor.getString(cursor.getColumnIndex(RELATIVE_FILE_PATH)));
+				cursor.getString(cursor.getColumnIndex(RELATIVE_FILE_PATH)),
+				cursor.getString(cursor.getColumnIndex(SERVER_MSG_ID)));
 	}
 
 	public static Message createStatusMessage(Conversation conversation) {
@@ -168,6 +174,7 @@ public class Message extends AbstractEntity {
 		values.put(TYPE, type);
 		values.put(REMOTE_MSG_ID, remoteMsgId);
 		values.put(RELATIVE_FILE_PATH, relativeFilePath);
+		values.put(SERVER_MSG_ID,serverMsgId);
 		return values;
 	}
 
@@ -248,6 +255,14 @@ public class Message extends AbstractEntity {
 		this.remoteMsgId = id;
 	}
 
+	public String getServerMsgId() {
+		return this.serverMsgId;
+	}
+
+	public void setServerMsgId(String id) {
+		this.serverMsgId = id;
+	}
+
 	public boolean isRead() {
 		return this.read;
 	}
@@ -293,7 +308,15 @@ public class Message extends AbstractEntity {
 	}
 
 	public boolean equals(Message message) {
-		return (this.remoteMsgId != null) && (this.body != null) && (this.counterpart != null) && this.remoteMsgId.equals(message.getRemoteMsgId()) && this.body.equals(message.getBody()) && this.counterpart.equals(message.getCounterpart());
+		if (this.serverMsgId != null && message.getServerMsgId() != null) {
+			return this.serverMsgId.equals(message.getServerMsgId());
+		} else {
+			return this.body != null
+					&& this.counterpart != null
+					&& ((this.remoteMsgId != null && this.remoteMsgId.equals(message.getRemoteMsgId()))
+					|| this.uuid.equals(message.getRemoteMsgId())) && this.body.equals(message.getBody())
+					&& this.counterpart.equals(message.getCounterpart());
+		}
 	}
 
 	public Message next() {
@@ -493,6 +516,11 @@ public class Message extends AbstractEntity {
 		}
 	}
 
+	public void untie() {
+		this.mNextMessage = null;
+		this.mPreviousMessage = null;
+	}
+
 	public class ImageParams {
 		public URL url;
 		public long size = 0;

src/main/java/eu/siacs/conversations/generator/AbstractGenerator.java

@@ -4,9 +4,12 @@ import android.util.Base64;
 
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
 
 import eu.siacs.conversations.services.XmppConnectionService;
 
@@ -23,6 +26,8 @@ public abstract class AbstractGenerator {
 	public final String IDENTITY_NAME = "Conversations 0.9.3";
 	public final String IDENTITY_TYPE = "phone";
 
+	private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US);
+
 	protected XmppConnectionService mXmppConnectionService;
 
 	protected AbstractGenerator(XmppConnectionService service) {
@@ -46,4 +51,9 @@ public abstract class AbstractGenerator {
 		byte[] sha1 = md.digest(s.toString().getBytes());
 		return new String(Base64.encode(sha1, Base64.DEFAULT)).trim();
 	}
+
+	public static String getTimestamp(long time) {
+		DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
+		return DATE_FORMAT.format(time);
+	}
 }

src/main/java/eu/siacs/conversations/generator/IqGenerator.java

@@ -1,11 +1,16 @@
 package eu.siacs.conversations.generator;
 
+import android.util.Log;
+
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import eu.siacs.conversations.Config;
+import eu.siacs.conversations.services.MessageArchiveService;
 import eu.siacs.conversations.services.XmppConnectionService;
 import eu.siacs.conversations.xml.Element;
+import eu.siacs.conversations.xmpp.forms.Data;
 import eu.siacs.conversations.xmpp.jid.Jid;
 import eu.siacs.conversations.xmpp.pep.Avatar;
 import eu.siacs.conversations.xmpp.stanzas.IqPacket;
@@ -94,4 +99,22 @@ public class IqGenerator extends AbstractGenerator {
 		}
 		return packet;
 	}
+
+	public IqPacket queryMessageArchiveManagement(MessageArchiveService.Query mam) {
+		final IqPacket packet = new IqPacket(IqPacket.TYPE_SET);
+		Element query = packet.query("urn:xmpp:mam:0");
+		query.setAttribute("queryid",mam.getQueryId());
+		Data data = new Data();
+		data.setFormType("urn:xmpp:mam:0");
+		if (mam.getWith()!=null) {
+			data.put("with", mam.getWith().toString());
+		}
+		data.put("start",getTimestamp(mam.getStart()));
+		data.put("end",getTimestamp(mam.getEnd()));
+		query.addChild(data);
+		if (mam.getAfter() != null) {
+			query.addChild("set", "http://jabber.org/protocol/rsm").addChild("after").setContent(mam.getAfter());
+		}
+		return packet;
+	}
 }

src/main/java/eu/siacs/conversations/parser/AbstractParser.java

@@ -24,40 +24,31 @@ public abstract class AbstractParser {
 
 	protected long getTimestamp(Element packet) {
 		long now = System.currentTimeMillis();
-		ArrayList<String> stamps = new ArrayList<>();
+		Element delay = packet.findChild("delay");
-		for (Element child : packet.getChildren()) {
+		if (delay == null) {
-			if (child.getName().equals("delay")) {
-				stamps.add(child.getAttribute("stamp").replace("Z", "+0000"));
-			}
-		}
-		Collections.sort(stamps);
-		if (stamps.size() >= 1) {
-			try {
-				String stamp = stamps.get(stamps.size() - 1);
-				if (stamp.contains(".")) {
-					Date date = new SimpleDateFormat(
-							"yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.US)
-							.parse(stamp);
-					if (now < date.getTime()) {
-						return now;
-					} else {
-						return date.getTime();
-					}
-				} else {
-					Date date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ",
-							Locale.US).parse(stamp);
-					if (now < date.getTime()) {
-						return now;
-					} else {
-						return date.getTime();
-					}
-				}
-			} catch (ParseException e) {
-				return now;
-			}
-		} else {
 			return now;
 		}
+		String stamp = delay.getAttribute("stamp");
+		if (stamp == null) {
+			return now;
+		}
+		try {
+			long time = parseTimestamp(stamp).getTime();
+			return now < time ? now : time;
+		} catch (ParseException e) {
+			return now;
+		}
+	}
+
+	public static Date parseTimestamp(String timestamp) throws ParseException {
+		timestamp = timestamp.replace("Z", "+0000");
+		SimpleDateFormat dateFormat;
+		if (timestamp.contains(".")) {
+			dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.US);
+		} else {
+			dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ",Locale.US);
+		}
+		return dateFormat.parse(timestamp);
 	}
 
 	protected void updateLastseen(final Element packet, final Account account,
@@ -66,8 +57,7 @@ public abstract class AbstractParser {
         try {
             from = Jid.fromString(packet.getAttribute("from")).toBareJid();
         } catch (final InvalidJidException e) {
-            // TODO: Handle this?
+			return;
-            from = null;
         }
         String presence = from == null || from.isBareJid() ? "" : from.getResourcepart();
 		Contact contact = account.getRoster().getContact(from);

src/main/java/eu/siacs/conversations/parser/MessageParser.java

@@ -10,11 +10,11 @@ import eu.siacs.conversations.entities.Account;
 import eu.siacs.conversations.entities.Contact;
 import eu.siacs.conversations.entities.Conversation;
 import eu.siacs.conversations.entities.Message;
+import eu.siacs.conversations.services.MessageArchiveService;
 import eu.siacs.conversations.services.XmppConnectionService;
 import eu.siacs.conversations.utils.CryptoHelper;
 import eu.siacs.conversations.xml.Element;
 import eu.siacs.conversations.xmpp.OnMessagePacketReceived;
-import eu.siacs.conversations.xmpp.jid.InvalidJidException;
 import eu.siacs.conversations.xmpp.jid.Jid;
 import eu.siacs.conversations.xmpp.pep.Avatar;
 import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
@@ -273,6 +273,66 @@ public class MessageParser extends AbstractParser implements
 		return finishedMessage;
 	}
 
+	private Message parseMamMessage(MessagePacket packet, final Account account) {
+		final Element result = packet.findChild("result","urn:xmpp:mam:0");
+		if (result == null ) {
+			return null;
+		}
+		final Element forwarded = result.findChild("forwarded","urn:xmpp:forward:0");
+		if (forwarded == null) {
+			return null;
+		}
+		final Element message = forwarded.findChild("message");
+		if (message == null) {
+			return null;
+		}
+		final Element body = message.findChild("body");
+		if (body == null || message.hasChild("private","urn:xmpp:carbons:2") || message.hasChild("no-copy","urn:xmpp:hints")) {
+			return null;
+		}
+		int encryption;
+		String content = getPgpBody(message);
+		if (content != null) {
+			encryption = Message.ENCRYPTION_PGP;
+		} else {
+			encryption = Message.ENCRYPTION_NONE;
+			content = body.getContent();
+		}
+		if (content == null) {
+			return null;
+		}
+		final long timestamp = getTimestamp(forwarded);
+		final Jid to = message.getAttributeAsJid("to");
+		final Jid from = message.getAttributeAsJid("from");
+		final MessageArchiveService.Query query = this.mXmppConnectionService.getMessageArchiveService().findQuery(result.getAttribute("queryid"));
+		Jid counterpart;
+		int status;
+		Conversation conversation;
+		if (from!=null && to != null && from.toBareJid().equals(account.getJid().toBareJid())) {
+			status = Message.STATUS_SEND;
+			conversation = this.mXmppConnectionService.findOrCreateConversation(account,to.toBareJid(),false,query);
+			counterpart = to;
+		} else if (from !=null && to != null) {
+			status = Message.STATUS_RECEIVED;
+			conversation = this.mXmppConnectionService.findOrCreateConversation(account,from.toBareJid(),false,query);
+			counterpart = from;
+		} else {
+			return null;
+		}
+		Message finishedMessage = new Message(conversation,content,encryption,status);
+		finishedMessage.setTime(timestamp);
+		finishedMessage.setCounterpart(counterpart);
+		finishedMessage.setRemoteMsgId(message.getAttribute("id"));
+		finishedMessage.setServerMsgId(result.getAttribute("id"));
+		if (conversation.hasDuplicateMessage(finishedMessage)) {
+			Log.d(Config.LOGTAG, "received mam message " + content+ " (duplicate)");
+			return null;
+		} else {
+			Log.d(Config.LOGTAG, "received mam message " + content);
+		}
+		return finishedMessage;
+	}
+
 	private void parseError(final MessagePacket packet, final Account account) {
 		final Jid from = packet.getFrom();
 		mXmppConnectionService.markMessage(account, from.toBareJid(),
@@ -446,6 +506,17 @@ public class MessageParser extends AbstractParser implements
 						message.markUnread();
 					}
 				}
+			} else if (packet.hasChild("result","urn:xmpp:mam:0")) {
+				message = parseMamMessage(packet, account);
+				if (message != null) {
+					Conversation conversation = message.getConversation();
+					conversation.add(message);
+					mXmppConnectionService.databaseBackend.createMessage(message);
+				}
+				return;
+			} else if (packet.hasChild("fin","urn:xmpp:mam:0")) {
+				Element fin = packet.findChild("fin","urn:xmpp:mam:0");
+				mXmppConnectionService.getMessageArchiveService().processFin(fin);
 			} else {
 				parseNonMessage(packet, account);
 			}
@@ -487,12 +558,16 @@ public class MessageParser extends AbstractParser implements
 		}
 		Conversation conversation = message.getConversation();
 		conversation.add(message);
+		if (account.getXmppConnection() != null && account.getXmppConnection().getFeatures().advancedStreamFeaturesLoaded()) {
+			if (conversation.setLastMessageTransmitted(System.currentTimeMillis())) {
+				mXmppConnectionService.updateConversation(conversation);
+			}
+		}
 
 		if (message.getStatus() == Message.STATUS_RECEIVED
 				&& conversation.getOtrSession() != null
 				&& !conversation.getOtrSession().getSessionID().getUserID()
 				.equals(message.getCounterpart().getResourcepart())) {
-			Log.d(Config.LOGTAG, "ending because of reasons");
 			conversation.endOtrIfNeeded();
 		}
 
@@ -505,7 +580,7 @@ public class MessageParser extends AbstractParser implements
 		if (message.trusted() && message.bodyContainsDownloadable()) {
 			this.mXmppConnectionService.getHttpConnectionManager()
 					.createNewConnection(message);
-		} else {
+		} else if (!message.isRead()) {
 			mXmppConnectionService.getNotificationService().push(message);
 		}
 		mXmppConnectionService.updateConversationUi();

src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java

@@ -22,7 +22,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
 	private static DatabaseBackend instance = null;
 
 	private static final String DATABASE_NAME = "history";
-	private static final int DATABASE_VERSION = 11;
+	private static final int DATABASE_VERSION = 12;
 
 	private static String CREATE_CONTATCS_STATEMENT = "create table "
 			+ Contact.TABLENAME + "(" + Contact.ACCOUNT + " TEXT, "
@@ -65,6 +65,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
 				+ Message.BODY + " TEXT, " + Message.ENCRYPTION + " NUMBER, "
 				+ Message.STATUS + " NUMBER," + Message.TYPE + " NUMBER, "
 				+ Message.RELATIVE_FILE_PATH + " TEXT, "
+				+ Message.SERVER_MSG_ID + " TEXT, "
 				+ Message.REMOTE_MSG_ID + " TEXT, FOREIGN KEY("
 				+ Message.CONVERSATION + ") REFERENCES "
 				+ Conversation.TABLENAME + "(" + Conversation.UUID
@@ -121,6 +122,10 @@ public class DatabaseBackend extends SQLiteOpenHelper {
 			db.execSQL("delete from "+Contact.TABLENAME);
 			db.execSQL("update "+Account.TABLENAME+" set "+Account.ROSTERVERSION+" = NULL");
 		}
+		if (oldVersion < 12 && newVersion >= 12) {
+			db.execSQL("ALTER TABLE " + Message.TABLENAME + " ADD COLUMN "
+					+ Message.SERVER_MSG_ID + " TEXT");
+		}
 	}
 
 	public static synchronized DatabaseBackend getInstance(Context context) {

src/main/java/eu/siacs/conversations/services/MessageArchiveService.java

@@ -0,0 +1,240 @@
+package eu.siacs.conversations.services;
+
+import android.util.Log;
+
+import java.math.BigInteger;
+import java.util.HashSet;
+import java.util.List;
+
+import eu.siacs.conversations.Config;
+import eu.siacs.conversations.entities.Account;
+import eu.siacs.conversations.entities.Conversation;
+import eu.siacs.conversations.generator.AbstractGenerator;
+import eu.siacs.conversations.parser.AbstractParser;
+import eu.siacs.conversations.xml.Element;
+import eu.siacs.conversations.xmpp.OnAdvancedStreamFeaturesLoaded;
+import eu.siacs.conversations.xmpp.OnIqPacketReceived;
+import eu.siacs.conversations.xmpp.jid.Jid;
+import eu.siacs.conversations.xmpp.stanzas.IqPacket;
+
+public class MessageArchiveService implements OnAdvancedStreamFeaturesLoaded {
+
+	private final XmppConnectionService mXmppConnectionService;
+
+	private final HashSet<Query> queries = new HashSet<Query>();
+
+	public MessageArchiveService(final XmppConnectionService service) {
+		this.mXmppConnectionService = service;
+	}
+
+	public void catchup(final Account account) {
+		long startCatchup = getLastMessageTransmitted(account);
+		long endCatchup = account.getXmppConnection().getLastSessionEstablished();
+		if (startCatchup == 0) {
+			return;
+		} else if (endCatchup - startCatchup >= Config.MAX_CATCHUP) {
+			startCatchup = endCatchup - Config.MAX_CATCHUP;
+			List<Conversation> conversations = mXmppConnectionService.getConversations();
+			for (Conversation conversation : conversations) {
+				if (conversation.getMode() == Conversation.MODE_SINGLE && conversation.getAccount() == account && startCatchup > conversation.getLastMessageTransmitted()) {
+					this.query(conversation,startCatchup);
+				}
+			}
+		}
+		final Query query = new Query(account, startCatchup, endCatchup);
+		this.queries.add(query);
+		this.execute(query);
+	}
+
+	private long getLastMessageTransmitted(final Account account) {
+		long timestamp = 0;
+		for(final Conversation conversation : mXmppConnectionService.getConversations()) {
+			if (conversation.getAccount() == account) {
+				long tmp = conversation.getLastMessageTransmitted();
+				if (tmp > timestamp) {
+					timestamp = tmp;
+				}
+			}
+		}
+		return timestamp;
+	}
+
+	public void query(final Conversation conversation) {
+		query(conversation,conversation.getAccount().getXmppConnection().getLastSessionEstablished());
+	}
+
+	public void query(final Conversation conversation, long end) {
+		synchronized (this.queries) {
+			final Account account = conversation.getAccount();
+			long start = conversation.getLastMessageTransmitted();
+			if (start > end) {
+				return;
+			} else if (end - start >= Config.MAX_HISTORY_AGE) {
+				start = end - Config.MAX_HISTORY_AGE;
+			}
+			final Query query = new Query(conversation, start, end);
+			this.queries.add(query);
+			this.execute(query);
+		}
+	}
+
+	private void execute(final Query query) {
+		Log.d(Config.LOGTAG,query.getAccount().getJid().toBareJid().toString()+": running mam query "+query.toString());
+		IqPacket packet = this.mXmppConnectionService.getIqGenerator().queryMessageArchiveManagement(query);
+			this.mXmppConnectionService.sendIqPacket(query.getAccount(), packet, new OnIqPacketReceived() {
+				@Override
+				public void onIqPacketReceived(Account account, IqPacket packet) {
+					if (packet.getType() == IqPacket.TYPE_ERROR) {
+						Log.d(Config.LOGTAG,account.getJid().toBareJid().toString()+": error executing mam: "+packet.toString());
+						finalizeQuery(query);
+					}
+				}
+			});
+	}
+
+	private void finalizeQuery(Query query) {
+		synchronized (this.queries) {
+			this.queries.remove(query);
+		}
+		final Conversation conversation = query.getConversation();
+		if (conversation != null) {
+			conversation.sort();
+			if (conversation.setLastMessageTransmitted(query.getEnd())) {
+				this.mXmppConnectionService.databaseBackend.updateConversation(conversation);
+			}
+			this.mXmppConnectionService.updateConversationUi();
+		} else {
+			for(Conversation tmp : this.mXmppConnectionService.getConversations()) {
+				if (tmp.getAccount() == query.getAccount()) {
+					tmp.sort();
+					if (tmp.setLastMessageTransmitted(query.getEnd())) {
+						this.mXmppConnectionService.databaseBackend.updateConversation(tmp);
+					}
+				}
+			}
+		}
+	}
+
+	public void processFin(Element fin) {
+		if (fin == null) {
+			return;
+		}
+		Query query = findQuery(fin.getAttribute("queryid"));
+		if (query == null) {
+			return;
+		}
+		boolean complete = fin.getAttributeAsBoolean("complete");
+		Element set = fin.findChild("set","http://jabber.org/protocol/rsm");
+		Element last = set == null ? null : set.findChild("last");
+		if (complete || last == null) {
+			this.finalizeQuery(query);
+		} else {
+			final Query nextQuery = query.next(last == null ? null : last.getContent());
+			this.execute(nextQuery);
+			synchronized (this.queries) {
+				this.queries.remove(query);
+				this.queries.add(nextQuery);
+			}
+		}
+	}
+
+	public Query findQuery(String id) {
+		if (id == null) {
+			return null;
+		}
+		synchronized (this.queries) {
+			for(Query query : this.queries) {
+				if (query.getQueryId().equals(id)) {
+					return query;
+				}
+			}
+			return null;
+		}
+	}
+
+	@Override
+	public void onAdvancedStreamFeaturesAvailable(Account account) {
+		if (account.getXmppConnection() != null && account.getXmppConnection().getFeatures().mam()) {
+			this.catchup(account);
+		}
+	}
+
+	public class Query {
+		private long start;
+		private long end;
+		private Jid with = null;
+		private String queryId;
+		private String after = null;
+		private Account account;
+		private Conversation conversation;
+
+		public Query(Conversation conversation, long start, long end) {
+			this(conversation.getAccount(), start, end);
+			this.conversation = conversation;
+			this.with = conversation.getContactJid().toBareJid();
+		}
+
+		public Query(Account account, long start, long end) {
+			this.account = account;
+			this.start = start;
+			this.end = end;
+			this.queryId = new BigInteger(50, mXmppConnectionService.getRNG()).toString(32);
+		}
+
+		public Query next(String after) {
+			Query query = new Query(this.account,this.start,this.end);
+			query.after = after;
+			query.conversation = conversation;
+			query.with = with;
+			return query;
+		}
+
+		public String getAfter() {
+			return after;
+		}
+
+		public String getQueryId() {
+			return queryId;
+		}
+
+		public Jid getWith() {
+			return with;
+		}
+
+		public long getStart() {
+			return start;
+		}
+
+		public long getEnd() {
+			return end;
+		}
+
+		public Conversation getConversation() {
+			return conversation;
+		}
+
+		public Account getAccount() {
+			return this.account;
+		}
+
+		@Override
+		public String toString() {
+			StringBuilder builder = new StringBuilder();
+			builder.append("with=");
+			if (this.with==null) {
+				builder.append("*");
+			} else {
+				builder.append(with.toString());
+			}
+			builder.append(", start=");
+			builder.append(AbstractGenerator.getTimestamp(this.start));
+			builder.append(", end=");
+			builder.append(AbstractGenerator.getTimestamp(this.end));
+			if (this.after!=null) {
+				builder.append(", after=");
+				builder.append(this.after);
+			}
+			return builder.toString();
+		}
+	}
+}

src/main/java/eu/siacs/conversations/services/XmppConnectionService.java

@@ -32,20 +32,14 @@ import net.java.otr4j.session.SessionStatus;
 import org.openintents.openpgp.util.OpenPgpApi;
 import org.openintents.openpgp.util.OpenPgpServiceConnection;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
 import java.math.BigInteger;
 import java.security.SecureRandom;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.Date;
 import java.util.Hashtable;
 import java.util.List;
 import java.util.Locale;
-import java.util.TimeZone;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 import de.duenndns.ssl.MemorizingTrustManager;
@@ -147,6 +141,7 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 	private HttpConnectionManager mHttpConnectionManager = new HttpConnectionManager(
 			this);
 	private AvatarService mAvatarService = new AvatarService(this);
+	private MessageArchiveService mMessageArchiveService = new MessageArchiveService(this);
 	private OnConversationUpdate mOnConversationUpdate = null;
 	private Integer convChangedListenerCount = 0;
 	private OnAccountUpdate mOnAccountUpdate = null;
@@ -209,6 +204,7 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 			getNotificationService().updateErrorNotification();
 		}
 	};
+
 	private int accountChangedListenerCount = 0;
 	private OnRosterUpdate mOnRosterUpdate = null;
 	private int rosterChangedListenerCount = 0;
@@ -260,15 +256,17 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 
 		@Override
 		public void onMessageAcknowledged(Account account, String uuid) {
-			for (Conversation conversation : getConversations()) {
+			for (final Conversation conversation : getConversations()) {
 				if (conversation.getAccount() == account) {
-					for (Message message : conversation.getMessages()) {
+					for (final Message message : conversation.getMessages()) {
-						if ((message.getStatus() == Message.STATUS_UNSEND || message
+						final int s = message.getStatus();
-									.getStatus() == Message.STATUS_WAITING)
+						if ((s == Message.STATUS_UNSEND || s == Message.STATUS_WAITING) && message.getUuid().equals(uuid)) {
-								&& message.getUuid().equals(uuid)) {
 							markMessage(message, Message.STATUS_SEND);
+							if (conversation.setLastMessageTransmitted(System.currentTimeMillis())) {
+								databaseBackend.updateConversation(conversation);
+							}
 							return;
-								}
+						}
 					}
 				}
 			}
@@ -590,8 +588,8 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 		connection.setOnUnregisteredIqPacketReceivedListener(this.mIqParser);
 		connection.setOnJinglePacketReceivedListener(this.jingleListener);
 		connection.setOnBindListener(this.mOnBindListener);
-		connection
+		connection.setOnMessageAcknowledgeListener(this.mOnMessageAcknowledgedListener);
-			.setOnMessageAcknowledgeListener(this.mOnMessageAcknowledgedListener);
+		connection.addOnAdvancedStreamFeaturesAvailableListener(this.mMessageArchiveService);
 		return connection;
 	}
 
@@ -995,8 +993,11 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 		return null;
 	}
 
-	public Conversation findOrCreateConversation(final Account account, final Jid jid,
+	public Conversation findOrCreateConversation(final Account account, final Jid jid,final boolean muc) {
-			final boolean muc) {
+		return this.findOrCreateConversation(account,jid,muc,null);
+	}
+
+	public Conversation findOrCreateConversation(final Account account, final Jid jid,final boolean muc, final MessageArchiveService.Query query) {
 		synchronized (this.conversations) {
 			Conversation conversation = find(account, jid);
 			if (conversation != null) {
@@ -1030,6 +1031,13 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 				}
 				this.databaseBackend.createConversation(conversation);
 			}
+			if (query == null) {
+				this.mMessageArchiveService.query(conversation);
+			} else {
+				if (query.getConversation() == null) {
+					this.mMessageArchiveService.query(conversation,query.getStart());
+				}
+			}
 			this.conversations.add(conversation);
 			updateConversationUi();
 			return conversation;
@@ -1256,27 +1264,16 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 			PresencePacket packet = new PresencePacket();
 			packet.setFrom(conversation.getAccount().getJid());
 			packet.setTo(joinJid);
-			Element x = new Element("x");
+			Element x = packet.addChild("x","http://jabber.org/protocol/muc");
-			x.setAttribute("xmlns", "http://jabber.org/protocol/muc");
 			if (conversation.getMucOptions().getPassword() != null) {
-				Element password = x.addChild("password");
+				x.addChild("password").setContent(conversation.getMucOptions().getPassword());
-				password.setContent(conversation.getMucOptions().getPassword());
 			}
+			x.addChild("history").setAttribute("since",PresenceGenerator.getTimestamp(conversation.getLastMessageTransmitted()));
 			String sig = account.getPgpSignature();
 			if (sig != null) {
 				packet.addChild("status").setContent("online");
 				packet.addChild("x", "jabber:x:signed").setContent(sig);
 			}
-			if (conversation.getMessages().size() != 0) {
-				final SimpleDateFormat mDateFormat = new SimpleDateFormat(
-						"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US);
-				mDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-				Date date = new Date(conversation.getLatestMessage()
-						.getTimeSent() + 1000);
-				x.addChild("history").setAttribute("since",
-						mDateFormat.format(date));
-			}
-			packet.addChild(x);
 			sendPresencePacket(account, packet);
 			if (!joinJid.equals(conversation.getContactJid())) {
 				conversation.setContactJid(joinJid);
@@ -1320,7 +1317,7 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 
 				@Override
 				public void onFailure() {
-					callback.error(R.string.nick_in_use,conversation);
+					callback.error(R.string.nick_in_use, conversation);
 				}
 			});
 
@@ -2054,6 +2051,10 @@ public class XmppConnectionService extends Service implements OnPhoneContactsLoa
 		return this.mJingleConnectionManager;
 	}
 
+	public MessageArchiveService getMessageArchiveService() {
+		return this.mMessageArchiveService;
+	}
+
 	public List<Contact> findContacts(Jid jid) {
 		ArrayList<Contact> contacts = new ArrayList<>();
 		for (Account account : getAccounts()) {

src/main/java/eu/siacs/conversations/xml/Element.java

@@ -159,4 +159,9 @@ public class Element {
 	public void setAttribute(String name, int value) {
 		this.setAttribute(name, Integer.toString(value));
 	}
+
+	public boolean getAttributeAsBoolean(String name) {
+		String attr = getAttribute(name);
+		return (attr != null && (attr.equalsIgnoreCase("true") || attr.equalsIgnoreCase("1")));
+	}
 }

src/main/java/eu/siacs/conversations/xmpp/OnAdvancedStreamFeaturesLoaded.java

@@ -0,0 +1,7 @@
+package eu.siacs.conversations.xmpp;
+
+import eu.siacs.conversations.entities.Account;
+
+public interface OnAdvancedStreamFeaturesLoaded {
+	public void onAdvancedStreamFeaturesAvailable(final Account account);
+}

src/main/java/eu/siacs/conversations/xmpp/XmppConnection.java

@@ -107,6 +107,7 @@ public class XmppConnection implements Runnable {
 	private OnMessagePacketReceived messageListener = null;
 	private OnStatusChanged statusListener = null;
 	private OnBindListener bindListener = null;
+	private ArrayList<OnAdvancedStreamFeaturesLoaded> advancedStreamFeaturesLoadedListeners = new ArrayList<>();
 	private OnMessageAcknowledged acknowledgedListener = null;
 	private XmppConnectionService mXmppConnectionService = null;
 
@@ -771,6 +772,9 @@ public class XmppConnection implements Runnable {
 
 					if (account.getServer().equals(server.toDomainJid())) {
 						enableAdvancedStreamFeatures();
+						for(OnAdvancedStreamFeaturesLoaded listener : advancedStreamFeaturesLoadedListeners) {
+							listener.onAdvancedStreamFeaturesAvailable(account);
+						}
 					}
 				}
 			});
@@ -943,6 +947,12 @@ public class XmppConnection implements Runnable {
 		this.acknowledgedListener = listener;
 	}
 
+	public void addOnAdvancedStreamFeaturesAvailableListener(OnAdvancedStreamFeaturesLoaded listener) {
+		if (!this.advancedStreamFeaturesLoadedListeners.contains(listener)) {
+			this.advancedStreamFeaturesLoadedListeners.add(listener);
+		}
+	}
+
 	public void disconnect(boolean force) {
 		Log.d(Config.LOGTAG, account.getJid().toBareJid() + ": disconnecting");
 		try {
@@ -1087,6 +1097,10 @@ public class XmppConnection implements Runnable {
 			return hasDiscoFeature(account.getServer(), "urn:xmpp:mam:0");
 		}
 
+		public boolean advancedStreamFeaturesLoaded() {
+			return disco.containsKey(account.getServer().toString());
+		}
+
 		public boolean rosterVersioning() {
 			return connection.streamFeatures != null && connection.streamFeatures.hasChild("ver");
 		}

src/main/java/eu/siacs/conversations/xmpp/forms/Data.java

@@ -37,6 +37,7 @@ public class Data extends Element {
 		Field field = getFieldByName(name);
 		if (field == null) {
 			field = new Field(name);
+			this.addChild(field);
 		}
 		field.setValue(value);
 	}
@@ -45,6 +46,7 @@ public class Data extends Element {
 		Field field = getFieldByName(name);
 		if (field == null) {
 			field = new Field(name);
+			this.addChild(field);
 		}
 		field.setValues(values);
 	}
@@ -72,4 +74,12 @@ public class Data extends Element {
 		data.setChildren(element.getChildren());
 		return data;
 	}
+
+	public void setFormType(String formType) {
+		this.put("FORM_TYPE",formType);
+	}
+
+	public String getFormType() {
+		return this.getAttribute("FORM_TYPE");
+	}
 }