plugins/mod_storage_internal.lua
author Matthew Wild <mwild1@gmail.com>
Mon, 20 Feb 2023 18:10:15 +0000
branch0.12
changeset 12898 0598d822614f
parent 11769 1cac469b18d0
child 12981 74b9e05af71e
permissions -rw-r--r--
mod_websocket: Fire pre-session-close event (fixes #1800) This event was added in a7c183bb4e64 and is required to make mod_smacks know that a session was intentionally closed and shouldn't be hibernated (see fcea4d9e7502). Because this was missing from mod_websocket's session.close(), mod_smacks would always attempt to hibernate websocket sessions even if they closed cleanly. That mod_websocket has its own copy of session.close() is something to fix another day (probably not in the stable branch). So for now this commit makes the minimal change to get things working again. Thanks to Damian and the Jitsi team for reporting.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
     1
local cache = require "util.cache";
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
     2
local datamanager = require "core.storagemanager".olddm;
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
     3
local array = require "util.array";
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
     4
local datetime = require "util.datetime";
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
     5
local st = require "util.stanza";
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
     6
local now = require "util.time".now;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
     7
local id = require "util.id".medium;
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
     8
local jid_join = require "util.jid".join;
11282
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
     9
local set = require "util.set";
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    10
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    11
local host = module.host;
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    12
9892
5669bb11ac7b mod_storage_internal: Increase default quota to 10 000
Kim Alvefur <zash@zash.se>
parents: 9889
diff changeset
    13
local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    14
local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    15
5122
b41c33dc7c36 mod_storage_*: Don't explicitly set driver name, to ease copying/renaming modules.
Waqas Hussain <waqas20@gmail.com>
parents: 5121
diff changeset
    16
local driver = {};
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    17
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    18
function driver:open(store, typ)
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    19
	local mt = self[typ or "keyval"]
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    20
	if not mt then
6283
7cf6d3a2c855 mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents: 5153
diff changeset
    21
		return nil, "unsupported-store";
7cf6d3a2c855 mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents: 5153
diff changeset
    22
	end
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    23
	return setmetatable({ store = store, type = typ }, mt);
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    24
end
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    25
8023
342ce07836de mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents: 8022
diff changeset
    26
function driver:stores(username) -- luacheck: ignore 212/self
8022
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    27
	return datamanager.stores(username, host);
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    28
end
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    29
8023
342ce07836de mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents: 8022
diff changeset
    30
function driver:purge(user) -- luacheck: ignore 212/self
8022
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    31
	return datamanager.purge(user, host);
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    32
end
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    33
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    34
local keyval = { };
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    35
driver.keyval = { __index = keyval };
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    36
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    37
function keyval:get(user)
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    38
	return datamanager.load(user, host, self.store);
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    39
end
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    40
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    41
function keyval:set(user, data)
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    42
	return datamanager.store(user, host, self.store, data);
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    43
end
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    44
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    45
function keyval:users()
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    46
	return datamanager.users(host, self.store, self.type);
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    47
end
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    48
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    49
local archive = {};
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    50
driver.archive = { __index = archive };
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    51
9888
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    52
archive.caps = {
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    53
	total = true;
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    54
	quota = archive_item_limit;
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    55
	truncate = true;
11279
b8fada57faf0 mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents: 10930
diff changeset
    56
	full_id_range = true;
11282
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
    57
	ids = true;
9888
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    58
};
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    59
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    60
function archive:append(username, key, value, when, with)
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    61
	when = when or now();
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    62
	if not st.is_stanza(value) then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    63
		return nil, "unsupported-datatype";
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    64
	end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    65
	value = st.preserialize(st.clone(value));
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    66
	value.when = when;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    67
	value.with = with;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    68
	value.attr.stamp = datetime.datetime(when);
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    69
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    70
	local cache_key = jid_join(username, host, self.store);
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    71
	local item_count = archive_item_count_cache:get(cache_key);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    72
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    73
	if key then
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    74
		local items, err = datamanager.list_load(username, host, self.store);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    75
		if not items and err then return items, err; end
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    76
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    77
		-- Check the quota
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    78
		item_count = items and #items or 0;
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    79
		archive_item_count_cache:set(cache_key, item_count);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    80
		if item_count >= archive_item_limit then
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    81
			module:log("debug", "%s reached or over quota, not adding to store", username);
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    82
			return nil, "quota-limit";
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    83
		end
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    84
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    85
		if items then
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    86
			-- Filter out any item with the same key as the one being added
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    87
			items = array(items);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    88
			items:filter(function (item)
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    89
				return item.key ~= key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    90
			end);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    91
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    92
			value.key = key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    93
			items:push(value);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    94
			local ok, err = datamanager.list_store(username, host, self.store, items);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    95
			if not ok then return ok, err; end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    96
			archive_item_count_cache:set(cache_key, #items);
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    97
			return key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    98
		end
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    99
	else
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   100
		if not item_count then -- Item count not cached?
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   101
			-- We need to load the list to get the number of items currently stored
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   102
			local items, err = datamanager.list_load(username, host, self.store);
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   103
			if not items and err then return items, err; end
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   104
			item_count = items and #items or 0;
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   105
			archive_item_count_cache:set(cache_key, item_count);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   106
		end
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   107
		if item_count >= archive_item_limit then
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   108
			module:log("debug", "%s reached or over quota, not adding to store", username);
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   109
			return nil, "quota-limit";
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   110
		end
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   111
		key = id();
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   112
	end
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   113
9899
6bd65bff03b5 mod_storage_internal: Include store name when reporting quota status
Kim Alvefur <zash@zash.se>
parents: 9892
diff changeset
   114
	module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   115
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   116
	value.key = key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   117
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   118
	local ok, err = datamanager.list_append(username, host, self.store, value);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   119
	if not ok then return ok, err; end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   120
	archive_item_count_cache:set(cache_key, item_count+1);
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   121
	return key;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   122
end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   123
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   124
function archive:find(username, query)
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   125
	local items, err = datamanager.list_load(username, host, self.store);
8176
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   126
	if not items then
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   127
		if err then
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   128
			return items, err;
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   129
		elseif query then
10028
4a0d990253a0 mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents: 10022
diff changeset
   130
			if query.before or query.after then
4a0d990253a0 mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents: 10022
diff changeset
   131
				return nil, "item-not-found";
4a0d990253a0 mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents: 10022
diff changeset
   132
			end
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   133
			if query.total then
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   134
				return function () end, 0;
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   135
			end
8176
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   136
		end
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   137
		return function () end;
8176
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   138
	end
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   139
	local count = nil;
10930
c55bd98a54f8 mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents: 10848
diff changeset
   140
	local i, last_key = 0;
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   141
	if query then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   142
		items = array(items);
8092
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   143
		if query.key then
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   144
			items:filter(function (item)
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   145
				return item.key == query.key;
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   146
			end);
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   147
		end
11282
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   148
		if query.ids then
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   149
			local ids = set.new(query.ids);
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   150
			items:filter(function (item)
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   151
				return ids:contains(item.key);
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   152
			end);
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   153
		end
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   154
		if query.with then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   155
			items:filter(function (item)
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   156
				return item.with == query.with;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   157
			end);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   158
		end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   159
		if query.start then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   160
			items:filter(function (item)
10847
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   161
				local when = item.when or datetime.parse(item.attr.stamp);
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   162
				return when >= query.start;
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   163
			end);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   164
		end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   165
		if query["end"] then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   166
			items:filter(function (item)
10847
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   167
				local when = item.when or datetime.parse(item.attr.stamp);
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   168
				return when <= query["end"];
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   169
			end);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   170
		end
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   171
		if query.total then
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   172
			count = #items;
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   173
		end
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   174
		if query.reverse then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   175
			items:reverse();
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   176
			if query.before then
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   177
				local found = false;
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   178
				for j = 1, #items do
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   179
					if (items[j].key or tostring(j)) == query.before then
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   180
						found = true;
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   181
						i = j;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   182
						break;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   183
					end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   184
				end
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   185
				if not found then
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   186
					return nil, "item-not-found";
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   187
				end
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   188
			end
11279
b8fada57faf0 mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents: 10930
diff changeset
   189
			last_key = query.after;
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   190
		elseif query.after then
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   191
			local found = false;
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   192
			for j = 1, #items do
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   193
				if (items[j].key or tostring(j)) == query.after then
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   194
					found = true;
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   195
					i = j;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   196
					break;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   197
				end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   198
			end
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   199
			if not found then
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   200
				return nil, "item-not-found";
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   201
			end
11279
b8fada57faf0 mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents: 10930
diff changeset
   202
			last_key = query.before;
b8fada57faf0 mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents: 10930
diff changeset
   203
		elseif query.before then
b8fada57faf0 mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents: 10930
diff changeset
   204
			last_key = query.before;
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   205
		end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   206
		if query.limit and #items - i > query.limit then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   207
			items[i+query.limit+1] = nil;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   208
		end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   209
	end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   210
	return function ()
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   211
		i = i + 1;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   212
		local item = items[i];
10930
c55bd98a54f8 mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents: 10848
diff changeset
   213
		if not item or (last_key and item.key == last_key) then
c55bd98a54f8 mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents: 10848
diff changeset
   214
			return;
c55bd98a54f8 mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents: 10848
diff changeset
   215
		end
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   216
		local key = item.key or tostring(i);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   217
		local when = item.when or datetime.parse(item.attr.stamp);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   218
		local with = item.with;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   219
		item.key, item.when, item.with = nil, nil, nil;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   220
		item.attr.stamp = nil;
11769
1cac469b18d0 mod_storage_internal: Stop storing XEP-0091 timestamp
Kim Alvefur <zash@zash.se>
parents: 11282
diff changeset
   221
		-- COMPAT Stored data may still contain legacy XEP-0091 timestamp
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   222
		item.attr.stamp_legacy = nil;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   223
		item = st.deserialize(item);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   224
		return key, item, when, with;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   225
	end, count;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   226
end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   227
10841
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   228
function archive:get(username, wanted_key)
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   229
	local iter, err = self:find(username, { key = wanted_key })
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   230
	if not iter then return iter, err; end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   231
	for key, stanza, when, with in iter do
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   232
		if key == wanted_key then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   233
			return stanza, when, with;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   234
		end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   235
	end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   236
	return nil, "item-not-found";
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   237
end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   238
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   239
function archive:set(username, key, new_value, new_when, new_with)
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   240
	local items, err = datamanager.list_load(username, host, self.store);
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   241
	if not items then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   242
		if err then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   243
			return items, err;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   244
		else
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   245
			return nil, "item-not-found";
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   246
		end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   247
	end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   248
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   249
	for i = 1, #items do
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   250
		local old_item = items[i];
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   251
		if old_item.key == key then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   252
			local item = st.preserialize(st.clone(new_value));
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   253
10846
5a6ba2f38e2b mod_storage_internal: Fix keeping old timestamp in archive map API
Kim Alvefur <zash@zash.se>
parents: 10841
diff changeset
   254
			local when = new_when or old_item.when or datetime.parse(old_item.attr.stamp);
10841
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   255
			item.key = key;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   256
			item.when = when;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   257
			item.with = new_with or old_item.with;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   258
			item.attr.stamp = datetime.datetime(when);
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   259
			items[i] = item;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   260
			return datamanager.list_store(username, host, self.store, items);
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   261
		end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   262
	end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   263
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   264
	return nil, "item-not-found";
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   265
end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   266
8026
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   267
function archive:dates(username)
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   268
	local items, err = datamanager.list_load(username, host, self.store);
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   269
	if not items then return items, err; end
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   270
	return array(items):pluck("when"):map(datetime.date):unique();
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   271
end
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   272
9907
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   273
function archive:summary(username, query)
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   274
	local iter, err = self:find(username, query)
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   275
	if not iter then return iter, err; end
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   276
	local counts = {};
10226
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   277
	local earliest = {};
10225
068692cb9e78 mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10224
diff changeset
   278
	local latest = {};
10227
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   279
	local body = {};
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   280
	for _, stanza, when, with in iter do
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   281
		counts[with] = (counts[with] or 0) + 1;
10226
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   282
		if earliest[with] == nil then
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   283
			earliest[with] = when;
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   284
		end
10225
068692cb9e78 mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10224
diff changeset
   285
		latest[with] = when;
10227
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   286
		body[with] = stanza:get_child_text("body") or body[with];
9907
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   287
	end
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   288
	return {
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   289
		counts = counts;
10226
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   290
		earliest = earliest;
10225
068692cb9e78 mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10224
diff changeset
   291
		latest = latest;
10227
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   292
		body = body;
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   293
	};
9907
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   294
end
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   295
10005
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   296
function archive:users()
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   297
	return datamanager.users(host, self.store, "list");
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   298
end
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   299
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   300
function archive:delete(username, query)
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   301
	local cache_key = jid_join(username, host, self.store);
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   302
	if not query or next(query) == nil then
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   303
		archive_item_count_cache:set(cache_key, nil);
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   304
		return datamanager.list_store(username, host, self.store, nil);
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   305
	end
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   306
	local items, err = datamanager.list_load(username, host, self.store);
8146
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   307
	if not items then
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   308
		if err then
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   309
			return items, err;
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   310
		end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   311
		archive_item_count_cache:set(cache_key, 0);
8146
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   312
		-- Store is empty
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   313
		return 0;
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   314
	end
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   315
	items = array(items);
8145
13e075549cb3 mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
   316
	local count_before = #items;
8313
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   317
	if query then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   318
		if query.key then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   319
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   320
				return item.key ~= query.key;
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   321
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   322
		end
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   323
		if query.with then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   324
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   325
				return item.with ~= query.with;
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   326
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   327
		end
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   328
		if query.start then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   329
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   330
				return item.when < query.start;
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   331
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   332
		end
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   333
		if query["end"] then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   334
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   335
				return item.when > query["end"];
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   336
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   337
		end
9108
e735c9865f42 mod_storage_internal: Only apply truncate if there are more items than requested
Kim Alvefur <zash@zash.se>
parents: 8405
diff changeset
   338
		if query.truncate and #items > query.truncate then
8396
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   339
			if query.reverse then
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   340
				-- Before: { 1, 2, 3, 4, 5, }
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   341
				-- After: { 1, 2, 3 }
8397
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   342
				for i = #items, query.truncate + 1, -1 do
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   343
					items[i] = nil;
8396
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   344
				end
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   345
			else
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   346
				-- Before: { 1, 2, 3, 4, 5, }
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   347
				-- After: { 3, 4, 5 }
8397
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   348
				local offset = #items - query.truncate;
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   349
				for i = 1, #items do
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   350
					items[i] = items[i+offset];
8396
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   351
				end
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   352
			end
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   353
		end
8313
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   354
	end
8145
13e075549cb3 mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
   355
	local count = count_before - #items;
8405
469afa02947b mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents: 8397
diff changeset
   356
	if count == 0 then
469afa02947b mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents: 8397
diff changeset
   357
		return 0; -- No changes, skip write
469afa02947b mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents: 8397
diff changeset
   358
	end
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   359
	local ok, err = datamanager.list_store(username, host, self.store, items);
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   360
	if not ok then return ok, err; end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   361
	archive_item_count_cache:set(cache_key, #items);
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   362
	return count;
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   363
end
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   364
5121
b5a5643f8572 core.storagemanager, mod_storage_*: "data-driver" -> "storage-provider", to allow using module:provides().
Waqas Hussain <waqas20@gmail.com>
parents: 5039
diff changeset
   365
module:provides("storage", driver);