-- Prosody IM

-- Copyright (C) 2008-2017 Matthew Wild

-- Copyright (C) 2008-2017 Waqas Hussain

-- Copyright (C) 2011-2021 Kim Alvefur

--

-- This project is MIT/X11 licensed. Please see the

-- COPYING file in the source package for more information.

--

-- XEP-0313: Message Archive Management for Prosody

--

local xmlns_mam     = "urn:xmpp:mam:2";

local xmlns_mam_ext = "urn:xmpp:mam:2#extended";

local xmlns_delay   = "urn:xmpp:delay";

local xmlns_forward = "urn:xmpp:forward:0";

local xmlns_st_id   = "urn:xmpp:sid:0";

local um = require "core.usermanager";

local st = require "util.stanza";

local rsm = require "util.rsm";

local get_prefs = module:require"mamprefs".get;

local set_prefs = module:require"mamprefs".set;

local prefs_to_stanza = module:require"mamprefsxml".tostanza;

local prefs_from_stanza = module:require"mamprefsxml".fromstanza;

local jid_bare = require "util.jid".bare;

local jid_split = require "util.jid".split;

local jid_resource = require "util.jid".resource;

local jid_prepped_split = require "util.jid".prepped_split;

local dataform = require "util.dataforms".new;

local get_form_type = require "util.dataforms".get_type;

local host = module.host;

local rm_load_roster = require "core.rostermanager".load_roster;

local is_stanza = st.is_stanza;

local tostring = tostring;

local time_now = os.time;

local m_min = math.min;

local timestamp, datestamp = import( "util.datetime", "datetime", "date");

local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);

local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://jabber.org/protocol/chatstates" });

local archive_store = module:get_option_string("archive_store", "archive");

local archive = module:open_store(archive_store, "archive");

local cleanup_after = module:get_option_string("archive_expires_after", "1w");

local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);

local archive_truncate = math.floor(archive_item_limit * 0.99);

if not archive.find then

	error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"

		.."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");

end

local use_total = module:get_option_boolean("mam_include_total", true);

function schedule_cleanup()

	-- replaced later if cleanup is enabled

end

-- Handle prefs.

module:hook("iq/self/"..xmlns_mam..":prefs", function(event)

	local origin, stanza = event.origin, event.stanza;

	local user = origin.username;

	if stanza.attr.type == "set" then

		local new_prefs = stanza:get_child("prefs", xmlns_mam);

		local prefs = prefs_from_stanza(new_prefs);

		local ok, err = set_prefs(user, prefs);

		if not ok then

			origin.send(st.error_reply(stanza, "cancel", "internal-server-error", "Error storing preferences: "..tostring(err)));

			return true;

		end

	end

	local prefs = prefs_to_stanza(get_prefs(user, true));

	local reply = st.reply(stanza):add_child(prefs);

	origin.send(reply);

	return true;

end);

local query_form = dataform {

	{ name = "FORM_TYPE"; type = "hidden"; value = xmlns_mam };

	{ name = "with"; type = "jid-single" };

	{ name = "start"; type = "text-single"; datatype = "xs:dateTime" };

	{ name = "end"; type = "text-single"; datatype = "xs:dateTime" };

};

if archive.caps and archive.caps.full_id_range then

	table.insert(query_form, { name = "before-id"; type = "text-single"; });

	table.insert(query_form, { name = "after-id"; type = "text-single"; });

end

if archive.caps and archive.caps.ids then

	table.insert(query_form, { name = "ids"; type = "list-multi"; });

end

-- Serve form

module:hook("iq-get/self/"..xmlns_mam..":query", function(event)

	local origin, stanza = event.origin, event.stanza;

	get_prefs(origin.username, true);

	origin.send(st.reply(stanza):query(xmlns_mam):add_child(query_form:form()));

	return true;

end);

-- Handle archive queries

module:hook("iq-set/self/"..xmlns_mam..":query", function(event)

	local origin, stanza = event.origin, event.stanza;

	local query = stanza.tags[1];

	local qid = query.attr.queryid;

	origin.mam_requested = true;

	get_prefs(origin.username, true);

	-- Search query parameters

	local qwith, qstart, qend, qbefore, qafter, qids;

	local form = query:get_child("x", "jabber:x:data");

	if form then

		local form_type, err = get_form_type(form);

		if not form_type then

			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));

			return true;

		elseif form_type ~= xmlns_mam then

			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));

			return true;

		end

		form, err = query_form:data(form);

		if err then

			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));

			return true;

		end

		qwith, qstart, qend = form["with"], form["start"], form["end"];

		qbefore, qafter = form["before-id"], form["after-id"];

		qids = form["ids"];

		qwith = qwith and jid_bare(qwith); -- dataforms does jidprep

	end

	-- RSM stuff

	local qset = rsm.get(query);

	local qmax = m_min(qset and qset.max or default_max_items, max_max_items);

	local reverse = qset and qset.before or false;

	local before, after = qset and qset.before or qbefore, qset and qset.after or qafter;

	if type(before) ~= "string" then before = nil; end

	module:log("debug", "Archive query by %s id=%s with=%s when=%s...%s rsm=%q",

		origin.username,

		qid or stanza.attr.id,

		qwith or "*",

		qstart and timestamp(qstart) or "",

		qend and timestamp(qend) or "",

		qset);

	-- A reverse query needs to be flipped

	local flip = reverse;

	-- A flip-page query needs to be the opposite of that.

	if query:get_child("flip-page") then flip = not flip end

	-- Load all the data!

	local data, err = archive:find(origin.username, {

		start = qstart; ["end"] = qend; -- Time range

		with = qwith;

		limit = qmax == 0 and 0 or qmax + 1;

		before = before; after = after;

		ids = qids;

		reverse = reverse;

		total = use_total or qmax == 0;

	});

	if not data then

		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);

		if err == "item-not-found" then

			origin.send(st.error_reply(stanza, "modify", "item-not-found"));

		else

			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));

		end

		return true;

	end

	local total = tonumber(err);

	local msg_reply_attr = { to = stanza.attr.from, from = stanza.attr.to };

	local results = {};

	-- Wrap it in stuff and deliver

	local first, last;

	local count = 0;

	local complete = "true";

	for id, item, when in data do

		count = count + 1;

		if count > qmax then

			-- We requested qmax+1 items. If that many items are retrieved then

			-- there are more results to page through, so:

			complete = nil;

			break;

		end

		local fwd_st = st.message(msg_reply_attr)

			:tag("result", { xmlns = xmlns_mam, queryid = qid, id = id })

				:tag("forwarded", { xmlns = xmlns_forward })

					:tag("delay", { xmlns = xmlns_delay, stamp = timestamp(when) }):up();

		if not is_stanza(item) then

			item = st.deserialize(item);

		end

		item.attr.xmlns = "jabber:client";

		fwd_st:add_child(item);

		if not first then first = id; end

		last = id;

		if flip then

			results[count] = fwd_st;

		else

			origin.send(fwd_st);

		end

	end

	if flip then

		for i = #results, 1, -1 do

			origin.send(results[i]);

		end

	end

	if reverse then

		first, last = last, first;

	end

	origin.send(st.reply(stanza)

		:tag("fin", { xmlns = xmlns_mam, complete = complete })

			:add_child(rsm.generate {

				first = first, last = last, count = total }));

	-- That's all folks!

	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);

	return true;

end);

module:hook("iq-get/self/"..xmlns_mam..":metadata", function (event)

	local origin, stanza = event.origin, event.stanza;

	local reply = st.reply(stanza):tag("metadata", { xmlns = xmlns_mam });

	do

		local first = archive:find(origin.username, { limit = 1 });

		if not first then

			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));

			return true;

		end

		local id, _, when = first();

		if id then

			reply:tag("start", { id = id, timestamp = timestamp(when) }):up();

		end

	end

	do

		local last = archive:find(origin.username, { limit = 1, reverse = true });

		if not last then

			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));

			return true;

		end

		local id, _, when = last();

		if id then

			reply:tag("end", { id = id, timestamp = timestamp(when) }):up();

		end

	end

	origin.send(reply);

	return true;

end);

local function has_in_roster(user, who)

	local roster = rm_load_roster(user, host);

	module:log("debug", "%s has %s in roster? %s", user, who, roster[who] and "yes" or "no");

	return roster[who];

end

local function shall_store(user, who)

	-- TODO Cache this?

	if not um.user_exists(user, host) then

		module:log("debug", "%s@%s does not exist", user, host)

		return false;

	end

	local prefs = get_prefs(user);

	local rule = prefs[who];

	module:log("debug", "%s's rule for %s is %s", user, who, rule);

	if rule ~= nil then

		return rule;

	end

	-- Below could be done by a metatable

	local default = prefs[false];

	module:log("debug", "%s's default rule is %s", user, default);

	if default == "roster" then

		return has_in_roster(user, who);

	end

	return default;

end

local function strip_stanza_id(stanza, user)

	if stanza:get_child("stanza-id", xmlns_st_id) then

		stanza = st.clone(stanza);

		stanza:maptags(function (tag)

			if tag.name == "stanza-id" and tag.attr.xmlns == xmlns_st_id then

				local by_user, by_host, res = jid_prepped_split(tag.attr.by);

				if not res and by_host == host and by_user == user then

					return nil;

				end

			end

			return tag;

		end);

	end

	return stanza;

end

local function should_store(stanza) --> boolean, reason: string

	local st_type = stanza.attr.type or "normal";

	-- FIXME pass direction of stanza and use that along with bare/full JID addressing

	-- for more accurate MUC / type=groupchat check

	if st_type == "headline" then

		-- Headline messages are ephemeral by definition

		return false, "headline";

	end

	if st_type == "error" then

		-- Errors not sent sent from a local client

		-- Why would a client send an error anyway?

		if jid_resource(stanza.attr.to) then

			-- Store delivery failure notifications so you know if your own messages

			-- were not delivered.

			return true, "bounce";

		else

			-- Skip errors for messages that come from your account, such as PEP

			-- notifications.

			return false, "bounce";

		end

	end

	if st_type == "groupchat" then

		-- MUC messages always go to the full JID, usually archived by the MUC

		return false, "groupchat";

	end

	if stanza:get_child("no-store", "urn:xmpp:hints")

	or stanza:get_child("no-permanent-store", "urn:xmpp:hints") then

		-- XXX Experimental XEP

		return false, "hint";

	end

	if stanza:get_child("store", "urn:xmpp:hints") then

		return true, "hint";

	end

	if stanza:get_child("body") then

		return true, "body";

	end

	if stanza:get_child("subject") then

		-- XXX Who would send a message with a subject but without a body?

		return true, "subject";

	end

	if stanza:get_child("encryption", "urn:xmpp:eme:0") then

		-- Since we can't know what an encrypted message contains, we assume it's important

		-- XXX Experimental XEP

		return true, "encrypted";

	end

	if stanza:get_child(nil, "urn:xmpp:receipts") then

		-- If it's important enough to ask for a receipt then it's important enough to archive

		-- and the same applies to the receipt

		return true, "receipt";

	end

	if stanza:get_child(nil, "urn:xmpp:chat-markers:0") then

		-- XXX Experimental XEP