plugins/mod_storage_internal.lua
author Kim Alvefur <zash@zash.se>
Wed, 12 May 2021 01:25:44 +0200
changeset 13139 3fd24e1945b0
parent 12981 74b9e05af71e
child 13140 396db0e7084f
permissions -rw-r--r--
mod_storage_internal: Lazy-load archive items while iterating Very large list files previously ran into limits of the Lua parser, or just caused Prosody to freeze while parsing. Using the new index we can parse individual items one at a time. This probably won't reduce overall CPU usage, probably the opposite, but it will reduce the number of items in memory at once and allow collection of items after we iterated past them.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
12981
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     1
local cache = require "prosody.util.cache";
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     2
local datamanager = require "prosody.core.storagemanager".olddm;
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     3
local array = require "prosody.util.array";
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     4
local datetime = require "prosody.util.datetime";
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     5
local st = require "prosody.util.stanza";
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     6
local now = require "prosody.util.time".now;
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     7
local id = require "prosody.util.id".medium;
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     8
local jid_join = require "prosody.util.jid".join;
74b9e05af71e plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 11769
diff changeset
     9
local set = require "prosody.util.set";
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
    10
local it = require "prosody.util.iterators";
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    11
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    12
local host = module.host;
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    13
9892
5669bb11ac7b mod_storage_internal: Increase default quota to 10 000
Kim Alvefur <zash@zash.se>
parents: 9889
diff changeset
    14
local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    15
local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    16
5122
b41c33dc7c36 mod_storage_*: Don't explicitly set driver name, to ease copying/renaming modules.
Waqas Hussain <waqas20@gmail.com>
parents: 5121
diff changeset
    17
local driver = {};
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    18
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    19
function driver:open(store, typ)
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    20
	local mt = self[typ or "keyval"]
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    21
	if not mt then
6283
7cf6d3a2c855 mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents: 5153
diff changeset
    22
		return nil, "unsupported-store";
7cf6d3a2c855 mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents: 5153
diff changeset
    23
	end
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    24
	return setmetatable({ store = store, type = typ }, mt);
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    25
end
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    26
8023
342ce07836de mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents: 8022
diff changeset
    27
function driver:stores(username) -- luacheck: ignore 212/self
8022
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    28
	return datamanager.stores(username, host);
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    29
end
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    30
8023
342ce07836de mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents: 8022
diff changeset
    31
function driver:purge(user) -- luacheck: ignore 212/self
8022
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    32
	return datamanager.purge(user, host);
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    33
end
925098aad268 mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents: 8021
diff changeset
    34
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    35
local keyval = { };
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    36
driver.keyval = { __index = keyval };
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    37
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    38
function keyval:get(user)
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    39
	return datamanager.load(user, host, self.store);
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    40
end
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    41
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    42
function keyval:set(user, data)
4085
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    43
	return datamanager.store(user, host, self.store, data);
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    44
end
7699cef04740 storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
    45
8021
9545d0a9401f mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents: 6283
diff changeset
    46
function keyval:users()
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    47
	return datamanager.users(host, self.store, self.type);
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    48
end
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5133
diff changeset
    49
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    50
local archive = {};
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    51
driver.archive = { __index = archive };
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    52
9888
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    53
archive.caps = {
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    54
	total = true;
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    55
	quota = archive_item_limit;
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    56
	truncate = true;
11279
b8fada57faf0 mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents: 10930
diff changeset
    57
	full_id_range = true;
11282
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
    58
	ids = true;
9888
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    59
};
9751c17f5281 mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents: 9887
diff changeset
    60
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    61
function archive:append(username, key, value, when, with)
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    62
	when = when or now();
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    63
	if not st.is_stanza(value) then
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    64
		return nil, "unsupported-datatype";
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    65
	end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    66
	value = st.preserialize(st.clone(value));
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    67
	value.when = when;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    68
	value.with = with;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
    69
	value.attr.stamp = datetime.datetime(when);
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    70
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    71
	local cache_key = jid_join(username, host, self.store);
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    72
	local item_count = archive_item_count_cache:get(cache_key);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    73
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    74
	if key then
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    75
		local items, err = datamanager.list_load(username, host, self.store);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    76
		if not items and err then return items, err; end
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    77
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    78
		-- Check the quota
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    79
		item_count = items and #items or 0;
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    80
		archive_item_count_cache:set(cache_key, item_count);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    81
		if item_count >= archive_item_limit then
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    82
			module:log("debug", "%s reached or over quota, not adding to store", username);
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    83
			return nil, "quota-limit";
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    84
		end
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    85
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    86
		if items then
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    87
			-- Filter out any item with the same key as the one being added
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    88
			items = array(items);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    89
			items:filter(function (item)
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    90
				return item.key ~= key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    91
			end);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
    92
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    93
			value.key = key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    94
			items:push(value);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    95
			local ok, err = datamanager.list_store(username, host, self.store, items);
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    96
			if not ok then return ok, err; end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
    97
			archive_item_count_cache:set(cache_key, #items);
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    98
			return key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
    99
		end
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   100
	else
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   101
		if not item_count then -- Item count not cached?
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   102
			-- We need to load the list to get the number of items currently stored
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   103
			local items, err = datamanager.list_load(username, host, self.store);
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   104
			if not items and err then return items, err; end
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   105
			item_count = items and #items or 0;
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   106
			archive_item_count_cache:set(cache_key, item_count);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   107
		end
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   108
		if item_count >= archive_item_limit then
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   109
			module:log("debug", "%s reached or over quota, not adding to store", username);
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   110
			return nil, "quota-limit";
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   111
		end
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   112
		key = id();
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   113
	end
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   114
9899
6bd65bff03b5 mod_storage_internal: Include store name when reporting quota status
Kim Alvefur <zash@zash.se>
parents: 9892
diff changeset
   115
	module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store);
9887
f76bd399267c mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents: 9108
diff changeset
   116
8312
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   117
	value.key = key;
5281c479955a mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents: 8176
diff changeset
   118
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   119
	local ok, err = datamanager.list_append(username, host, self.store, value);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   120
	if not ok then return ok, err; end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   121
	archive_item_count_cache:set(cache_key, item_count+1);
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   122
	return key;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   123
end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   124
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   125
function archive:find(username, query)
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   126
	local list, err = datamanager.list_open(username, host, self.store);
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   127
	if not list then
8176
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   128
		if err then
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   129
			return list, err;
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   130
		elseif query then
10028
4a0d990253a0 mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents: 10022
diff changeset
   131
			if query.before or query.after then
4a0d990253a0 mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents: 10022
diff changeset
   132
				return nil, "item-not-found";
4a0d990253a0 mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents: 10022
diff changeset
   133
			end
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   134
			if query.total then
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   135
				return function()
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   136
				end, 0;
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   137
			end
8176
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   138
		end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   139
		return function()
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   140
		end;
8176
3ff99d49082f mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents: 8146
diff changeset
   141
	end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   142
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   143
	local i = 0;
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   144
	local iter = function()
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   145
		i = i + 1;
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   146
		return list[i]
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   147
	end
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   148
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   149
	if query then
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   150
		if query.reverse then
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   151
			i = #list + 1
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   152
			iter = function()
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   153
				i = i - 1
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   154
				return list[i]
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   155
			end
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   156
		end
8092
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   157
		if query.key then
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   158
			iter = it.filter(function(item)
8092
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   159
				return item.key == query.key;
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   160
			end, iter);
8092
4ba8cb75d925 mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents: 8026
diff changeset
   161
		end
11282
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   162
		if query.ids then
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   163
			local ids = set.new(query.ids);
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   164
			iter = it.filter(function(item)
11282
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   165
				return ids:contains(item.key);
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   166
			end, iter);
11282
c3907f05bed4 mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents: 11279
diff changeset
   167
		end
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   168
		if query.with then
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   169
			iter = it.filter(function(item)
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   170
				return item.with == query.with;
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   171
			end, iter);
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   172
		end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   173
		if query.start then
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   174
			iter = it.filter(function(item)
10847
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   175
				local when = item.when or datetime.parse(item.attr.stamp);
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   176
				return when >= query.start;
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   177
			end, iter);
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   178
		end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   179
		if query["end"] then
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   180
			iter = it.filter(function(item)
10847
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   181
				local when = item.when or datetime.parse(item.attr.stamp);
8fcd46ee9bf5 mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents: 9108
diff changeset
   182
				return when <= query["end"];
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   183
			end, iter);
10001
7c4631d7b6fb mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents: 9907
diff changeset
   184
		end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   185
		if query.after then
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   186
			local found = false;
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   187
			iter = it.filter(function(item)
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   188
				local found_after = found;
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   189
				if item.key == query.after then
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   190
					found = true
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   191
				end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   192
				return found_after;
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   193
			end, iter);
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   194
		end
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   195
		if query.before then
10022
7408b9473729 mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents: 10005
diff changeset
   196
			local found = false;
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   197
			iter = it.filter(function(item)
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   198
				if item.key == query.before then
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   199
					found = true
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   200
				end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   201
				return not found;
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   202
			end, iter);
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   203
		end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   204
		if query.limit then
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   205
			iter = it.head(query.limit, iter);
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   206
		end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   207
	end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   208
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   209
	return function()
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   210
		local item = iter();
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   211
		if item == nil then
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   212
			return
10930
c55bd98a54f8 mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents: 10848
diff changeset
   213
		end
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   214
		local key = item.key;
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   215
		local when = item.when or item.attr and datetime.parse(item.attr.stamp);
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   216
		local with = item.with;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   217
		item.key, item.when, item.with = nil, nil, nil;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   218
		item.attr.stamp = nil;
11769
1cac469b18d0 mod_storage_internal: Stop storing XEP-0091 timestamp
Kim Alvefur <zash@zash.se>
parents: 11282
diff changeset
   219
		-- COMPAT Stored data may still contain legacy XEP-0091 timestamp
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   220
		item.attr.stamp_legacy = nil;
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   221
		item = st.deserialize(item);
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   222
		return key, item, when, with;
13139
3fd24e1945b0 mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents: 12981
diff changeset
   223
	end
8024
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   224
end
83f18982bcfd mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents: 8023
diff changeset
   225
10841
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   226
function archive:get(username, wanted_key)
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   227
	local iter, err = self:find(username, { key = wanted_key })
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   228
	if not iter then return iter, err; end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   229
	for key, stanza, when, with in iter do
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   230
		if key == wanted_key then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   231
			return stanza, when, with;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   232
		end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   233
	end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   234
	return nil, "item-not-found";
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   235
end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   236
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   237
function archive:set(username, key, new_value, new_when, new_with)
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   238
	local items, err = datamanager.list_load(username, host, self.store);
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   239
	if not items then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   240
		if err then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   241
			return items, err;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   242
		else
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   243
			return nil, "item-not-found";
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   244
		end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   245
	end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   246
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   247
	for i = 1, #items do
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   248
		local old_item = items[i];
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   249
		if old_item.key == key then
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   250
			local item = st.preserialize(st.clone(new_value));
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   251
10846
5a6ba2f38e2b mod_storage_internal: Fix keeping old timestamp in archive map API
Kim Alvefur <zash@zash.se>
parents: 10841
diff changeset
   252
			local when = new_when or old_item.when or datetime.parse(old_item.attr.stamp);
10841
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   253
			item.key = key;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   254
			item.when = when;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   255
			item.with = new_with or old_item.with;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   256
			item.attr.stamp = datetime.datetime(when);
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   257
			items[i] = item;
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   258
			return datamanager.list_store(username, host, self.store, items);
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   259
		end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   260
	end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   261
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   262
	return nil, "item-not-found";
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   263
end
f23363380599 mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents: 10227
diff changeset
   264
8026
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   265
function archive:dates(username)
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   266
	local items, err = datamanager.list_load(username, host, self.store);
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   267
	if not items then return items, err; end
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   268
	return array(items):pluck("when"):map(datetime.date):unique();
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   269
end
5a9d491cc714 mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents: 8025
diff changeset
   270
9907
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   271
function archive:summary(username, query)
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   272
	local iter, err = self:find(username, query)
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   273
	if not iter then return iter, err; end
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   274
	local counts = {};
10226
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   275
	local earliest = {};
10225
068692cb9e78 mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10224
diff changeset
   276
	local latest = {};
10227
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   277
	local body = {};
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   278
	for _, stanza, when, with in iter do
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   279
		counts[with] = (counts[with] or 0) + 1;
10226
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   280
		if earliest[with] == nil then
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   281
			earliest[with] = when;
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   282
		end
10225
068692cb9e78 mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10224
diff changeset
   283
		latest[with] = when;
10227
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   284
		body[with] = stanza:get_child_text("body") or body[with];
9907
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   285
	end
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   286
	return {
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   287
		counts = counts;
10226
51f145094648 mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10225
diff changeset
   288
		earliest = earliest;
10225
068692cb9e78 mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents: 10224
diff changeset
   289
		latest = latest;
10227
d185c4961ee0 mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents: 10226
diff changeset
   290
		body = body;
10224
1e2b444acb72 mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents: 10028
diff changeset
   291
	};
9907
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   292
end
2c5546cc5c70 mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents: 9899
diff changeset
   293
10005
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   294
function archive:users()
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   295
	return datamanager.users(host, self.store, "list");
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   296
end
55f5588d71c6 mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents: 10001
diff changeset
   297
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   298
function archive:delete(username, query)
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   299
	local cache_key = jid_join(username, host, self.store);
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   300
	if not query or next(query) == nil then
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   301
		archive_item_count_cache:set(cache_key, nil);
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   302
		return datamanager.list_store(username, host, self.store, nil);
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   303
	end
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   304
	local items, err = datamanager.list_load(username, host, self.store);
8146
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   305
	if not items then
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   306
		if err then
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   307
			return items, err;
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   308
		end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   309
		archive_item_count_cache:set(cache_key, 0);
8146
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   310
		-- Store is empty
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   311
		return 0;
c4c159953c72 mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents: 8145
diff changeset
   312
	end
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   313
	items = array(items);
8145
13e075549cb3 mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
   314
	local count_before = #items;
8313
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   315
	if query then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   316
		if query.key then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   317
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   318
				return item.key ~= query.key;
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   319
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   320
		end
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   321
		if query.with then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   322
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   323
				return item.with ~= query.with;
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   324
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   325
		end
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   326
		if query.start then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   327
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   328
				return item.when < query.start;
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   329
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   330
		end
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   331
		if query["end"] then
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   332
			items:filter(function (item)
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   333
				return item.when > query["end"];
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   334
			end);
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   335
		end
9108
e735c9865f42 mod_storage_internal: Only apply truncate if there are more items than requested
Kim Alvefur <zash@zash.se>
parents: 8405
diff changeset
   336
		if query.truncate and #items > query.truncate then
8396
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   337
			if query.reverse then
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   338
				-- Before: { 1, 2, 3, 4, 5, }
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   339
				-- After: { 1, 2, 3 }
8397
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   340
				for i = #items, query.truncate + 1, -1 do
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   341
					items[i] = nil;
8396
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   342
				end
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   343
			else
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   344
				-- Before: { 1, 2, 3, 4, 5, }
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   345
				-- After: { 3, 4, 5 }
8397
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   346
				local offset = #items - query.truncate;
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   347
				for i = 1, #items do
4892c22403d5 mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents: 8396
diff changeset
   348
					items[i] = items[i+offset];
8396
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   349
				end
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   350
			end
b6a7b83f8d87 mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents: 8313
diff changeset
   351
		end
8313
1759491b53db mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents: 8312
diff changeset
   352
	end
8145
13e075549cb3 mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
   353
	local count = count_before - #items;
8405
469afa02947b mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents: 8397
diff changeset
   354
	if count == 0 then
469afa02947b mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents: 8397
diff changeset
   355
		return 0; -- No changes, skip write
469afa02947b mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents: 8397
diff changeset
   356
	end
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   357
	local ok, err = datamanager.list_store(username, host, self.store, items);
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   358
	if not ok then return ok, err; end
9889
64e16d1e91f6 mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents: 9888
diff changeset
   359
	archive_item_count_cache:set(cache_key, #items);
8025
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   360
	return count;
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   361
end
05e201468f27 mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents: 8024
diff changeset
   362
5121
b5a5643f8572 core.storagemanager, mod_storage_*: "data-driver" -> "storage-provider", to allow using module:provides().
Waqas Hussain <waqas20@gmail.com>
parents: 5039
diff changeset
   363
module:provides("storage", driver);