author | Kim Alvefur <zash@zash.se> |
Fri, 31 Dec 2021 14:01:12 +0100 | |
changeset 12144 | 1a4c61253932 |
parent 11769 | 1cac469b18d0 |
child 12981 | 74b9e05af71e |
permissions | -rw-r--r-- |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
1 |
local cache = require "util.cache"; |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
2 |
local datamanager = require "core.storagemanager".olddm; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
3 |
local array = require "util.array"; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
4 |
local datetime = require "util.datetime"; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
5 |
local st = require "util.stanza"; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
6 |
local now = require "util.time".now; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
7 |
local id = require "util.id".medium; |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
8 |
local jid_join = require "util.jid".join; |
11282
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
9 |
local set = require "util.set"; |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
10 |
|
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
11 |
local host = module.host; |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
12 |
|
9892
5669bb11ac7b
mod_storage_internal: Increase default quota to 10 000
Kim Alvefur <zash@zash.se>
parents:
9889
diff
changeset
|
13 |
local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000); |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
14 |
local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
15 |
|
5122
b41c33dc7c36
mod_storage_*: Don't explicitly set driver name, to ease copying/renaming modules.
Waqas Hussain <waqas20@gmail.com>
parents:
5121
diff
changeset
|
16 |
local driver = {}; |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
17 |
|
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
18 |
function driver:open(store, typ) |
8021
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
19 |
local mt = self[typ or "keyval"] |
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
20 |
if not mt then |
6283
7cf6d3a2c855
mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents:
5153
diff
changeset
|
21 |
return nil, "unsupported-store"; |
7cf6d3a2c855
mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents:
5153
diff
changeset
|
22 |
end |
8021
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
23 |
return setmetatable({ store = store, type = typ }, mt); |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
24 |
end |
8021
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
25 |
|
8023
342ce07836de
mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
26 |
function driver:stores(username) -- luacheck: ignore 212/self |
8022
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
27 |
return datamanager.stores(username, host); |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
28 |
end |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
29 |
|
8023
342ce07836de
mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
30 |
function driver:purge(user) -- luacheck: ignore 212/self |
8022
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
31 |
return datamanager.purge(user, host); |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
32 |
end |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
33 |
|
8021
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
34 |
local keyval = { }; |
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
35 |
driver.keyval = { __index = keyval }; |
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
36 |
|
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
37 |
function keyval:get(user) |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
38 |
return datamanager.load(user, host, self.store); |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
39 |
end |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
40 |
|
8021
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
41 |
function keyval:set(user, data) |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
42 |
return datamanager.store(user, host, self.store, data); |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
43 |
end |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
44 |
|
8021
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
45 |
function keyval:users() |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
46 |
return datamanager.users(host, self.store, self.type); |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
47 |
end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
48 |
|
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
49 |
local archive = {}; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
50 |
driver.archive = { __index = archive }; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
51 |
|
9888
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9887
diff
changeset
|
52 |
archive.caps = { |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9887
diff
changeset
|
53 |
total = true; |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9887
diff
changeset
|
54 |
quota = archive_item_limit; |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9887
diff
changeset
|
55 |
truncate = true; |
11279
b8fada57faf0
mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents:
10930
diff
changeset
|
56 |
full_id_range = true; |
11282
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
57 |
ids = true; |
9888
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9887
diff
changeset
|
58 |
}; |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9887
diff
changeset
|
59 |
|
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
60 |
function archive:append(username, key, value, when, with) |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
61 |
when = when or now(); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
62 |
if not st.is_stanza(value) then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
63 |
return nil, "unsupported-datatype"; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
64 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
65 |
value = st.preserialize(st.clone(value)); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
66 |
value.when = when; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
67 |
value.with = with; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
68 |
value.attr.stamp = datetime.datetime(when); |
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
69 |
|
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
70 |
local cache_key = jid_join(username, host, self.store); |
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
71 |
local item_count = archive_item_count_cache:get(cache_key); |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
72 |
|
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
73 |
if key then |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
74 |
local items, err = datamanager.list_load(username, host, self.store); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
75 |
if not items and err then return items, err; end |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
76 |
|
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
77 |
-- Check the quota |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
78 |
item_count = items and #items or 0; |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
79 |
archive_item_count_cache:set(cache_key, item_count); |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
80 |
if item_count >= archive_item_limit then |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
81 |
module:log("debug", "%s reached or over quota, not adding to store", username); |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
82 |
return nil, "quota-limit"; |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
83 |
end |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
84 |
|
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
85 |
if items then |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
86 |
-- Filter out any item with the same key as the one being added |
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
87 |
items = array(items); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
88 |
items:filter(function (item) |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
89 |
return item.key ~= key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
90 |
end); |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
91 |
|
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
92 |
value.key = key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
93 |
items:push(value); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
94 |
local ok, err = datamanager.list_store(username, host, self.store, items); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
95 |
if not ok then return ok, err; end |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
96 |
archive_item_count_cache:set(cache_key, #items); |
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
97 |
return key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
98 |
end |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
99 |
else |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
100 |
if not item_count then -- Item count not cached? |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
101 |
-- We need to load the list to get the number of items currently stored |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
102 |
local items, err = datamanager.list_load(username, host, self.store); |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
103 |
if not items and err then return items, err; end |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
104 |
item_count = items and #items or 0; |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
105 |
archive_item_count_cache:set(cache_key, item_count); |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
106 |
end |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
107 |
if item_count >= archive_item_limit then |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
108 |
module:log("debug", "%s reached or over quota, not adding to store", username); |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
109 |
return nil, "quota-limit"; |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
110 |
end |
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
111 |
key = id(); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
112 |
end |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
113 |
|
9899
6bd65bff03b5
mod_storage_internal: Include store name when reporting quota status
Kim Alvefur <zash@zash.se>
parents:
9892
diff
changeset
|
114 |
module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store); |
9887
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9108
diff
changeset
|
115 |
|
8312
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
116 |
value.key = key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8176
diff
changeset
|
117 |
|
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
118 |
local ok, err = datamanager.list_append(username, host, self.store, value); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
119 |
if not ok then return ok, err; end |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
120 |
archive_item_count_cache:set(cache_key, item_count+1); |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
121 |
return key; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
122 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
123 |
|
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
124 |
function archive:find(username, query) |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
125 |
local items, err = datamanager.list_load(username, host, self.store); |
8176
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8146
diff
changeset
|
126 |
if not items then |
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8146
diff
changeset
|
127 |
if err then |
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8146
diff
changeset
|
128 |
return items, err; |
10001
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
129 |
elseif query then |
10028
4a0d990253a0
mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents:
10022
diff
changeset
|
130 |
if query.before or query.after then |
4a0d990253a0
mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents:
10022
diff
changeset
|
131 |
return nil, "item-not-found"; |
4a0d990253a0
mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents:
10022
diff
changeset
|
132 |
end |
10001
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
133 |
if query.total then |
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
134 |
return function () end, 0; |
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
135 |
end |
8176
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8146
diff
changeset
|
136 |
end |
10001
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
137 |
return function () end; |
8176
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8146
diff
changeset
|
138 |
end |
10001
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
139 |
local count = nil; |
10930
c55bd98a54f8
mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents:
10848
diff
changeset
|
140 |
local i, last_key = 0; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
141 |
if query then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
142 |
items = array(items); |
8092
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8026
diff
changeset
|
143 |
if query.key then |
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8026
diff
changeset
|
144 |
items:filter(function (item) |
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8026
diff
changeset
|
145 |
return item.key == query.key; |
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8026
diff
changeset
|
146 |
end); |
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8026
diff
changeset
|
147 |
end |
11282
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
148 |
if query.ids then |
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
149 |
local ids = set.new(query.ids); |
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
150 |
items:filter(function (item) |
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
151 |
return ids:contains(item.key); |
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
152 |
end); |
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11279
diff
changeset
|
153 |
end |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
154 |
if query.with then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
155 |
items:filter(function (item) |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
156 |
return item.with == query.with; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
157 |
end); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
158 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
159 |
if query.start then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
160 |
items:filter(function (item) |
10847
8fcd46ee9bf5
mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents:
9108
diff
changeset
|
161 |
local when = item.when or datetime.parse(item.attr.stamp); |
8fcd46ee9bf5
mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents:
9108
diff
changeset
|
162 |
return when >= query.start; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
163 |
end); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
164 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
165 |
if query["end"] then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
166 |
items:filter(function (item) |
10847
8fcd46ee9bf5
mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents:
9108
diff
changeset
|
167 |
local when = item.when or datetime.parse(item.attr.stamp); |
8fcd46ee9bf5
mod_storage_internal: Fix error in time limited queries on items without 'when' field, fixes #1557
Kim Alvefur <zash@zash.se>
parents:
9108
diff
changeset
|
168 |
return when <= query["end"]; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
169 |
end); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
170 |
end |
10001
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
171 |
if query.total then |
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
172 |
count = #items; |
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
173 |
end |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
174 |
if query.reverse then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
175 |
items:reverse(); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
176 |
if query.before then |
10022
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
177 |
local found = false; |
10001
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
178 |
for j = 1, #items do |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
179 |
if (items[j].key or tostring(j)) == query.before then |
10022
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
180 |
found = true; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
181 |
i = j; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
182 |
break; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
183 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
184 |
end |
10022
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
185 |
if not found then |
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
186 |
return nil, "item-not-found"; |
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
187 |
end |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
188 |
end |
11279
b8fada57faf0
mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents:
10930
diff
changeset
|
189 |
last_key = query.after; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
190 |
elseif query.after then |
10022
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
191 |
local found = false; |
10001
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9907
diff
changeset
|
192 |
for j = 1, #items do |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
193 |
if (items[j].key or tostring(j)) == query.after then |
10022
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
194 |
found = true; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
195 |
i = j; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
196 |
break; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
197 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
198 |
end |
10022
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
199 |
if not found then |
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
200 |
return nil, "item-not-found"; |
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10005
diff
changeset
|
201 |
end |
11279
b8fada57faf0
mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents:
10930
diff
changeset
|
202 |
last_key = query.before; |
b8fada57faf0
mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents:
10930
diff
changeset
|
203 |
elseif query.before then |
b8fada57faf0
mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents:
10930
diff
changeset
|
204 |
last_key = query.before; |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
205 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
206 |
if query.limit and #items - i > query.limit then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
207 |
items[i+query.limit+1] = nil; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
208 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
209 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
210 |
return function () |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
211 |
i = i + 1; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
212 |
local item = items[i]; |
10930
c55bd98a54f8
mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents:
10848
diff
changeset
|
213 |
if not item or (last_key and item.key == last_key) then |
c55bd98a54f8
mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents:
10848
diff
changeset
|
214 |
return; |
c55bd98a54f8
mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents:
10848
diff
changeset
|
215 |
end |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
216 |
local key = item.key or tostring(i); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
217 |
local when = item.when or datetime.parse(item.attr.stamp); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
218 |
local with = item.with; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
219 |
item.key, item.when, item.with = nil, nil, nil; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
220 |
item.attr.stamp = nil; |
11769
1cac469b18d0
mod_storage_internal: Stop storing XEP-0091 timestamp
Kim Alvefur <zash@zash.se>
parents:
11282
diff
changeset
|
221 |
-- COMPAT Stored data may still contain legacy XEP-0091 timestamp |
8024
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
222 |
item.attr.stamp_legacy = nil; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
223 |
item = st.deserialize(item); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
224 |
return key, item, when, with; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
225 |
end, count; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
226 |
end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
227 |
|
10841
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
228 |
function archive:get(username, wanted_key) |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
229 |
local iter, err = self:find(username, { key = wanted_key }) |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
230 |
if not iter then return iter, err; end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
231 |
for key, stanza, when, with in iter do |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
232 |
if key == wanted_key then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
233 |
return stanza, when, with; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
234 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
235 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
236 |
return nil, "item-not-found"; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
237 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
238 |
|
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
239 |
function archive:set(username, key, new_value, new_when, new_with) |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
240 |
local items, err = datamanager.list_load(username, host, self.store); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
241 |
if not items then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
242 |
if err then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
243 |
return items, err; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
244 |
else |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
245 |
return nil, "item-not-found"; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
246 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
247 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
248 |
|
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
249 |
for i = 1, #items do |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
250 |
local old_item = items[i]; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
251 |
if old_item.key == key then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
252 |
local item = st.preserialize(st.clone(new_value)); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
253 |
|
10846
5a6ba2f38e2b
mod_storage_internal: Fix keeping old timestamp in archive map API
Kim Alvefur <zash@zash.se>
parents:
10841
diff
changeset
|
254 |
local when = new_when or old_item.when or datetime.parse(old_item.attr.stamp); |
10841
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
255 |
item.key = key; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
256 |
item.when = when; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
257 |
item.with = new_with or old_item.with; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
258 |
item.attr.stamp = datetime.datetime(when); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
259 |
items[i] = item; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
260 |
return datamanager.list_store(username, host, self.store, items); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
261 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
262 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
263 |
|
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
264 |
return nil, "item-not-found"; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
265 |
end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10227
diff
changeset
|
266 |
|
8026
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8025
diff
changeset
|
267 |
function archive:dates(username) |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8025
diff
changeset
|
268 |
local items, err = datamanager.list_load(username, host, self.store); |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8025
diff
changeset
|
269 |
if not items then return items, err; end |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8025
diff
changeset
|
270 |
return array(items):pluck("when"):map(datetime.date):unique(); |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8025
diff
changeset
|
271 |
end |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8025
diff
changeset
|
272 |
|
9907
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9899
diff
changeset
|
273 |
function archive:summary(username, query) |
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9899
diff
changeset
|
274 |
local iter, err = self:find(username, query) |
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9899
diff
changeset
|
275 |
if not iter then return iter, err; end |
10224
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10028
diff
changeset
|
276 |
local counts = {}; |
10226
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10225
diff
changeset
|
277 |
local earliest = {}; |
10225
068692cb9e78
mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10224
diff
changeset
|
278 |
local latest = {}; |
10227
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10226
diff
changeset
|
279 |
local body = {}; |
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10226
diff
changeset
|
280 |
for _, stanza, when, with in iter do |
10224
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10028
diff
changeset
|
281 |
counts[with] = (counts[with] or 0) + 1; |
10226
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10225
diff
changeset
|
282 |
if earliest[with] == nil then |
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10225
diff
changeset
|
283 |
earliest[with] = when; |
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10225
diff
changeset
|
284 |
end |
10225
068692cb9e78
mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10224
diff
changeset
|
285 |
latest[with] = when; |
10227
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10226
diff
changeset
|
286 |
body[with] = stanza:get_child_text("body") or body[with]; |
9907
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9899
diff
changeset
|
287 |
end |
10224
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10028
diff
changeset
|
288 |
return { |
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10028
diff
changeset
|
289 |
counts = counts; |
10226
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10225
diff
changeset
|
290 |
earliest = earliest; |
10225
068692cb9e78
mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10224
diff
changeset
|
291 |
latest = latest; |
10227
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10226
diff
changeset
|
292 |
body = body; |
10224
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10028
diff
changeset
|
293 |
}; |
9907
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9899
diff
changeset
|
294 |
end |
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9899
diff
changeset
|
295 |
|
10005
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
10001
diff
changeset
|
296 |
function archive:users() |
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
10001
diff
changeset
|
297 |
return datamanager.users(host, self.store, "list"); |
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
10001
diff
changeset
|
298 |
end |
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
10001
diff
changeset
|
299 |
|
8025
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
300 |
function archive:delete(username, query) |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
301 |
local cache_key = jid_join(username, host, self.store); |
8025
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
302 |
if not query or next(query) == nil then |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
303 |
archive_item_count_cache:set(cache_key, nil); |
8025
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
304 |
return datamanager.list_store(username, host, self.store, nil); |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
305 |
end |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
306 |
local items, err = datamanager.list_load(username, host, self.store); |
8146
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8145
diff
changeset
|
307 |
if not items then |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8145
diff
changeset
|
308 |
if err then |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8145
diff
changeset
|
309 |
return items, err; |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8145
diff
changeset
|
310 |
end |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
311 |
archive_item_count_cache:set(cache_key, 0); |
8146
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8145
diff
changeset
|
312 |
-- Store is empty |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8145
diff
changeset
|
313 |
return 0; |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8145
diff
changeset
|
314 |
end |
8025
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
315 |
items = array(items); |
8145
13e075549cb3
mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
316 |
local count_before = #items; |
8313
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
317 |
if query then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
318 |
if query.key then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
319 |
items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
320 |
return item.key ~= query.key; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
321 |
end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
322 |
end |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
323 |
if query.with then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
324 |
items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
325 |
return item.with ~= query.with; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
326 |
end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
327 |
end |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
328 |
if query.start then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
329 |
items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
330 |
return item.when < query.start; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
331 |
end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
332 |
end |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
333 |
if query["end"] then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
334 |
items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
335 |
return item.when > query["end"]; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
336 |
end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
337 |
end |
9108
e735c9865f42
mod_storage_internal: Only apply truncate if there are more items than requested
Kim Alvefur <zash@zash.se>
parents:
8405
diff
changeset
|
338 |
if query.truncate and #items > query.truncate then |
8396
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
339 |
if query.reverse then |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
340 |
-- Before: { 1, 2, 3, 4, 5, } |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
341 |
-- After: { 1, 2, 3 } |
8397
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8396
diff
changeset
|
342 |
for i = #items, query.truncate + 1, -1 do |
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8396
diff
changeset
|
343 |
items[i] = nil; |
8396
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
344 |
end |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
345 |
else |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
346 |
-- Before: { 1, 2, 3, 4, 5, } |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
347 |
-- After: { 3, 4, 5 } |
8397
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8396
diff
changeset
|
348 |
local offset = #items - query.truncate; |
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8396
diff
changeset
|
349 |
for i = 1, #items do |
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8396
diff
changeset
|
350 |
items[i] = items[i+offset]; |
8396
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
351 |
end |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
352 |
end |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8313
diff
changeset
|
353 |
end |
8313
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8312
diff
changeset
|
354 |
end |
8145
13e075549cb3
mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
355 |
local count = count_before - #items; |
8405
469afa02947b
mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents:
8397
diff
changeset
|
356 |
if count == 0 then |
469afa02947b
mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents:
8397
diff
changeset
|
357 |
return 0; -- No changes, skip write |
469afa02947b
mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents:
8397
diff
changeset
|
358 |
end |
8025
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
359 |
local ok, err = datamanager.list_store(username, host, self.store, items); |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
360 |
if not ok then return ok, err; end |
9889
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
361 |
archive_item_count_cache:set(cache_key, #items); |
8025
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
362 |
return count; |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
363 |
end |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8024
diff
changeset
|
364 |
|
5121
b5a5643f8572
core.storagemanager, mod_storage_*: "data-driver" -> "storage-provider", to allow using module:provides().
Waqas Hussain <waqas20@gmail.com>
parents:
5039
diff
changeset
|
365 |
module:provides("storage", driver); |