author | Kim Alvefur <zash@zash.se> |
Fri, 14 May 2021 05:49:35 +0200 | |
changeset 13141 | b417a49cc31b |
parent 13138 | 638f627e707f |
child 13184 | 48622b89f570 |
permissions | -rw-r--r-- |
1523
841d61be198f
Remove version number from copyright headers
Matthew Wild <mwild1@gmail.com>
parents:
1462
diff
changeset
|
1 |
-- Prosody IM |
2923
b7049746bd29
Update copyright headers for 2010
Matthew Wild <mwild1@gmail.com>
parents:
2159
diff
changeset
|
2 |
-- Copyright (C) 2008-2010 Matthew Wild |
b7049746bd29
Update copyright headers for 2010
Matthew Wild <mwild1@gmail.com>
parents:
2159
diff
changeset
|
3 |
-- Copyright (C) 2008-2010 Waqas Hussain |
4452
7de17ca4de14
util/datamanager: Use package.config to figure out directory seperator
James Callahan <james@chatid.com>
parents:
4112
diff
changeset
|
4 |
-- |
758 | 5 |
-- This project is MIT/X11 licensed. Please see the |
6 |
-- COPYING file in the source package for more information. |
|
519
cccd610a0ef9
Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents:
456
diff
changeset
|
7 |
-- |
cccd610a0ef9
Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents:
456
diff
changeset
|
8 |
|
cccd610a0ef9
Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents:
456
diff
changeset
|
9 |
|
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
10 |
local string = string; |
0 | 11 |
local format = string.format; |
5051
71253db26fda
util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents:
5049
diff
changeset
|
12 |
local setmetatable = setmetatable; |
71253db26fda
util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents:
5049
diff
changeset
|
13 |
local ipairs = ipairs; |
0 | 14 |
local char = string.char; |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
15 |
local pcall = pcall; |
12979
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12392
diff
changeset
|
16 |
local log = require "prosody.util.logger".init("datamanager"); |
0 | 17 |
local io_open = io.open; |
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
18 |
local os_remove = os.remove; |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
19 |
local os_rename = os.rename; |
5051
71253db26fda
util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents:
5049
diff
changeset
|
20 |
local tonumber = tonumber; |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
21 |
local floor = math.floor; |
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
22 |
local next = next; |
7435
92f721226753
util.datamanager: Import tostring and type (fix global access)
Kim Alvefur <zash@zash.se>
parents:
7205
diff
changeset
|
23 |
local type = type; |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
24 |
local t_insert = table.insert; |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
25 |
local t_concat = table.concat; |
12979
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12392
diff
changeset
|
26 |
local envloadfile = require"prosody.util.envload".envloadfile; |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
27 |
local envload = require"prosody.util.envload".envload; |
12979
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12392
diff
changeset
|
28 |
local serialize = require "prosody.util.serialization".serialize; |
3086
931acb1188b1
util.datamanager: When failing to load a file, and the file exists, return nil, error.
Waqas Hussain <waqas20@gmail.com>
parents:
2925
diff
changeset
|
29 |
local lfs = require "lfs"; |
9997
02a41315d275
Fix various spelling mistakes [codespell]
Kim Alvefur <zash@zash.se>
parents:
8558
diff
changeset
|
30 |
-- Extract directory separator from package.config (an undocumented string that comes with lua) |
7998
b99404a340fc
util.datamanager: Split long line [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7932
diff
changeset
|
31 |
local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package.config not in standard form" ) |
b99404a340fc
util.datamanager: Split long line [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7932
diff
changeset
|
32 |
|
4093
36555949bd16
util.datamanager: Use prosody.paths.data as the initial value for data_path
Matthew Wild <mwild1@gmail.com>
parents:
3723
diff
changeset
|
33 |
local prosody = prosody; |
2444
267d6482bac6
util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents:
2328
diff
changeset
|
34 |
|
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
35 |
local raw_mkdir = lfs.mkdir; |
8014
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
36 |
local atomic_append; |
7928
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
37 |
local ENOENT = 2; |
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
38 |
pcall(function() |
12979
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12392
diff
changeset
|
39 |
local pposix = require "prosody.util.pposix"; |
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
40 |
raw_mkdir = pposix.mkdir or raw_mkdir; -- Doesn't trample on umask |
8014
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
41 |
atomic_append = pposix.atomic_append; |
7930
7132abcf669e
util.datamanager: Import value of ENOENT from util.pposix (if available)
Kim Alvefur <zash@zash.se>
parents:
7928
diff
changeset
|
42 |
ENOENT = pposix.ENOENT or ENOENT; |
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
43 |
end); |
5045
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
44 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
45 |
local _ENV = nil; |
8558
4f0f5b49bb03
vairious: Add annotation when an empty environment is set [luacheck]
Kim Alvefur <zash@zash.se>
parents:
8302
diff
changeset
|
46 |
-- luacheck: std none |
0 | 47 |
|
48 |
---- utils ----- |
|
8302
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
49 |
local encode, decode, store_encode; |
2444
267d6482bac6
util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents:
2328
diff
changeset
|
50 |
do |
7735
aba3dd84d9f0
util.datamanager: Use the 'base' argument to tonumber() to indicate hexadecimal
Kim Alvefur <zash@zash.se>
parents:
7677
diff
changeset
|
51 |
local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber(k, 16)); return t[k]; end }); |
0 | 52 |
|
53 |
decode = function (s) |
|
7737
6a52415ed68a
util.datamanager: Shorter pattern
Kim Alvefur <zash@zash.se>
parents:
7736
diff
changeset
|
54 |
return s and (s:gsub("%%(%x%x)", urlcodes)); |
0 | 55 |
end |
56 |
||
57 |
encode = function (s) |
|
628
3712d36b6d25
Fixed URL encoding to generate %0x instead of %x
Waqas Hussain <waqas20@gmail.com>
parents:
615
diff
changeset
|
58 |
return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); |
0 | 59 |
end |
8302
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
60 |
|
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
61 |
-- Special encode function for store names, which historically were unencoded. |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
62 |
-- All currently known stores use a-z and underscore, so this one preserves underscores. |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
63 |
store_encode = function (s) |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
64 |
return s and (s:gsub("[^_%w]", function (c) return format("%%%02x", c:byte()); end)); |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
65 |
end |
0 | 66 |
end |
67 |
||
8014
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
68 |
if not atomic_append then |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
69 |
function atomic_append(f, data) |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
70 |
local pos = f:seek(); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
71 |
if not f:write(data) or not f:flush() then |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
72 |
f:seek("set", pos); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
73 |
f:write((" "):rep(#data)); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
74 |
f:flush(); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
75 |
return nil, "write-failed"; |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
76 |
end |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
77 |
return true; |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
78 |
end |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
79 |
end |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7999
diff
changeset
|
80 |
|
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
81 |
local _mkdir = {}; |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
82 |
local function mkdir(path) |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
83 |
path = path:gsub("/", path_separator); -- TODO as an optimization, do this during path creation rather than here |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
84 |
if not _mkdir[path] then |
2444
267d6482bac6
util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents:
2328
diff
changeset
|
85 |
raw_mkdir(path); |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
86 |
_mkdir[path] = true; |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
87 |
end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
88 |
return path; |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
89 |
end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
90 |
|
4108
e3e3aa286334
util.datamanager: Handle gracefully the lack of prosody.paths.data
Matthew Wild <mwild1@gmail.com>
parents:
4093
diff
changeset
|
91 |
local data_path = (prosody and prosody.paths and prosody.paths.data) or "."; |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
92 |
local callbacks = {}; |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
93 |
|
0 | 94 |
------- API ------------- |
95 |
||
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
96 |
local function set_data_path(path) |
1097
c5b33640a5f0
util.datamanager: Lower log level of 'Setting data path' to 'debug'
Matthew Wild <mwild1@gmail.com>
parents:
932
diff
changeset
|
97 |
log("debug", "Setting data path to: %s", path); |
452
613c5c6bdce4
Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents:
267
diff
changeset
|
98 |
data_path = path; |
613c5c6bdce4
Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents:
267
diff
changeset
|
99 |
end |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
100 |
|
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
101 |
local function callback(username, host, datastore, data) |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
102 |
for _, f in ipairs(callbacks) do |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
103 |
username, host, datastore, data = f(username, host, datastore, data); |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
104 |
if username == false then break; end |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
105 |
end |
4452
7de17ca4de14
util/datamanager: Use package.config to figure out directory seperator
James Callahan <james@chatid.com>
parents:
4112
diff
changeset
|
106 |
|
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
107 |
return username, host, datastore, data; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
108 |
end |
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
109 |
local function add_callback(func) |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
110 |
if not callbacks[func] then -- Would you really want to set the same callback more than once? |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
111 |
callbacks[func] = true; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
112 |
callbacks[#callbacks+1] = func; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
113 |
return true; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
114 |
end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
115 |
end |
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
116 |
local function remove_callback(func) |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
117 |
if callbacks[func] then |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
118 |
for i, f in ipairs(callbacks) do |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
119 |
if f == func then |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
120 |
callbacks[i] = nil; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
121 |
callbacks[f] = nil; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
122 |
return true; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
123 |
end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
124 |
end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
125 |
end |
932
1ed3e5fe165a
Added: datamanager: Allow a callback to be installed which selectively prevents disk writes
Waqas Hussain <waqas20@gmail.com>
parents:
915
diff
changeset
|
126 |
end |
452
613c5c6bdce4
Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents:
267
diff
changeset
|
127 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
128 |
local function getpath(username, host, datastore, ext, create) |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
129 |
ext = ext or "dat"; |
2328
2804d81206d8
util.datamanager: Store data stores with no host in '_global' folder
Matthew Wild <mwild1@gmail.com>
parents:
2159
diff
changeset
|
130 |
host = (host and encode(host)) or "_global"; |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
131 |
username = username and encode(username); |
8302
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8095
diff
changeset
|
132 |
datastore = store_encode(datastore); |
84
d0a0bac6815e
Added: Datastore support for hosts and global data in addition to users
Waqas Hussain <waqas20@gmail.com>
parents:
0
diff
changeset
|
133 |
if username then |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
134 |
if create then mkdir(mkdir(mkdir(data_path).."/"..host).."/"..datastore); end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
135 |
return format("%s/%s/%s/%s.%s", data_path, host, datastore, username, ext); |
5243
07e8256efcda
util.datamanager: Remove dead code path
Kim Alvefur <zash@zash.se>
parents:
5153
diff
changeset
|
136 |
else |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
137 |
if create then mkdir(mkdir(data_path).."/"..host); end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
138 |
return format("%s/%s/%s.%s", data_path, host, datastore, ext); |
84
d0a0bac6815e
Added: Datastore support for hosts and global data in addition to users
Waqas Hussain <waqas20@gmail.com>
parents:
0
diff
changeset
|
139 |
end |
0 | 140 |
end |
141 |
||
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
142 |
local function load(username, host, datastore) |
7928
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
143 |
local data, err, errno = envloadfile(getpath(username, host, datastore), {}); |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
144 |
if not data then |
7928
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
145 |
if errno == ENOENT then |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
146 |
-- No such file, ok to ignore |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
147 |
return nil; |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
148 |
end |
7999
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7998
diff
changeset
|
149 |
log("error", "Failed to load %s storage ('%s') for user: %s@%s", datastore, err, username or "nil", host or "nil"); |
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7998
diff
changeset
|
150 |
return nil, "Error reading storage"; |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
151 |
end |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
152 |
|
0 | 153 |
local success, ret = pcall(data); |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
154 |
if not success then |
4993
5243b74a4cbb
Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents:
4452
diff
changeset
|
155 |
log("error", "Unable to load %s storage ('%s') for user: %s@%s", datastore, ret, username or "nil", host or "nil"); |
3086
931acb1188b1
util.datamanager: When failing to load a file, and the file exists, return nil, error.
Waqas Hussain <waqas20@gmail.com>
parents:
2925
diff
changeset
|
156 |
return nil, "Error reading storage"; |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
157 |
end |
0 | 158 |
return ret; |
159 |
end |
|
160 |
||
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
161 |
local function atomic_store(filename, data) |
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
162 |
local scratch = filename.."~"; |
10535
810abf34919a
util.datamanager: Ignore unused 'errno' variable [luacheck]
Kim Alvefur <zash@zash.se>
parents:
9997
diff
changeset
|
163 |
local f, ok, msg, errno; -- luacheck: ignore errno |
810abf34919a
util.datamanager: Ignore unused 'errno' variable [luacheck]
Kim Alvefur <zash@zash.se>
parents:
9997
diff
changeset
|
164 |
-- TODO return util.error with code=errno? |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
165 |
|
7928
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
166 |
f, msg, errno = io_open(scratch, "w"); |
7205
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
167 |
if not f then |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
168 |
return nil, msg; |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
169 |
end |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
170 |
|
7205
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
171 |
ok, msg = f:write(data); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
172 |
if not ok then |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
173 |
f:close(); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
174 |
os_remove(scratch); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
175 |
return nil, msg; |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
176 |
end |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
177 |
|
7205
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
178 |
ok, msg = f:close(); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
179 |
if not ok then |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
180 |
os_remove(scratch); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
181 |
return nil, msg; |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
182 |
end |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
183 |
|
7205
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7204
diff
changeset
|
184 |
return os_rename(scratch, filename); |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
185 |
end |
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
186 |
|
6578
bdaadf70a48f
util.datamanager: Check that the global 'prosody' exists before using it (fixes nil indexing in use outside of prosody)
Kim Alvefur <zash@zash.se>
parents:
5441
diff
changeset
|
187 |
if prosody and prosody.platform ~= "posix" then |
5065
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
188 |
-- os.rename does not overwrite existing files on Windows |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
189 |
-- TODO We could use Transactional NTFS on Vista and above |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
190 |
function atomic_store(filename, data) |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
191 |
local f, err = io_open(filename, "w"); |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
192 |
if not f then return f, err; end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
193 |
local ok, msg = f:write(data); |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
194 |
if not ok then f:close(); return ok, msg; end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
195 |
return f:close(); |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
196 |
end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
197 |
end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
198 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
199 |
local function store(username, host, datastore, data) |
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
200 |
if not data then |
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
201 |
data = {}; |
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
202 |
end |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
203 |
|
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
204 |
username, host, datastore, data = callback(username, host, datastore, data); |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
205 |
if username == false then |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
206 |
return true; -- Don't save this data at all |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
207 |
end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
208 |
|
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
209 |
-- save the datastore |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
210 |
local d = "return " .. serialize(data) .. ";\n"; |
5441
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
211 |
local mkdir_cache_cleared; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
212 |
repeat |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
213 |
local ok, msg = atomic_store(getpath(username, host, datastore, nil, true), d); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
214 |
if not ok then |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
215 |
if not mkdir_cache_cleared then -- We may need to recreate a removed directory |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
216 |
_mkdir = {}; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
217 |
mkdir_cache_cleared = true; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
218 |
else |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
219 |
log("error", "Unable to write to %s storage ('%s') for user: %s@%s", datastore, msg, username or "nil", host or "nil"); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
220 |
return nil, "Error saving to storage"; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
221 |
end |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
222 |
end |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
223 |
if next(data) == nil then -- try to delete empty datastore |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
224 |
log("debug", "Removing empty %s datastore for user %s@%s", datastore, username or "nil", host or "nil"); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
225 |
os_remove(getpath(username, host, datastore)); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
226 |
end |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
227 |
-- we write data even when we are deleting because lua doesn't have a |
12392
50fcd3879482
spelling: non-existing mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents:
12391
diff
changeset
|
228 |
-- platform independent way of checking for nonexisting files |
5441
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
229 |
until ok; |
0 | 230 |
return true; |
231 |
end |
|
232 |
||
7002
0ad66d12113a
util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents:
7001
diff
changeset
|
233 |
-- Append a blob of data to a file |
6995
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6780
diff
changeset
|
234 |
local function append(username, host, datastore, ext, data) |
7003
2b57f77985a3
util.datamanager: Make sure only strings are passed as data to append()
Kim Alvefur <zash@zash.se>
parents:
7002
diff
changeset
|
235 |
if type(data) ~= "string" then return; end |
6997
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
236 |
local filename = getpath(username, host, datastore, ext, true); |
6999
644b1bddc676
util.datamanager: No shadowing of variable [luacheck]
Kim Alvefur <zash@zash.se>
parents:
6998
diff
changeset
|
237 |
|
8016
72cfbe377326
util.datamanager: Rearrange locals
Kim Alvefur <zash@zash.se>
parents:
8014
diff
changeset
|
238 |
local f = io_open(filename, "r+"); |
5066
4be7093edde9
util.datamanager: Try to open in read+write mode, then retry with write mode if that fails (usually because it doesn't exist)
Kim Alvefur <zash@zash.se>
parents:
5057
diff
changeset
|
239 |
if not f then |
7931
9d014aca766f
util.datamanager: Use atomic store function when writing the first entry of a list
Kim Alvefur <zash@zash.se>
parents:
7930
diff
changeset
|
240 |
return atomic_store(filename, data); |
7002
0ad66d12113a
util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents:
7001
diff
changeset
|
241 |
-- File did probably not exist, let's create it |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
242 |
end |
7002
0ad66d12113a
util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents:
7001
diff
changeset
|
243 |
|
5045
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
244 |
local pos = f:seek("end"); |
7000
0ab228bc21c6
util.datamanager: Handle potential issues from fallocate
Kim Alvefur <zash@zash.se>
parents:
6999
diff
changeset
|
245 |
|
8016
72cfbe377326
util.datamanager: Rearrange locals
Kim Alvefur <zash@zash.se>
parents:
8014
diff
changeset
|
246 |
local ok, msg = atomic_append(f, data); |
7000
0ab228bc21c6
util.datamanager: Handle potential issues from fallocate
Kim Alvefur <zash@zash.se>
parents:
6999
diff
changeset
|
247 |
|
7001
86607fe755b6
util.datamanager: Handle potential error from :write() call
Kim Alvefur <zash@zash.se>
parents:
7000
diff
changeset
|
248 |
if not ok then |
86607fe755b6
util.datamanager: Handle potential error from :write() call
Kim Alvefur <zash@zash.se>
parents:
7000
diff
changeset
|
249 |
f:close(); |
7004
2743759ca1b5
util.datamanager: Return extra location info
Kim Alvefur <zash@zash.se>
parents:
7003
diff
changeset
|
250 |
return ok, msg, "write"; |
6995
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6780
diff
changeset
|
251 |
end |
6997
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
252 |
|
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
253 |
ok, msg = f:close(); |
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
254 |
if not ok then |
8017
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8016
diff
changeset
|
255 |
return ok, msg, "close"; |
6997
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
256 |
end |
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
257 |
|
7004
2743759ca1b5
util.datamanager: Return extra location info
Kim Alvefur <zash@zash.se>
parents:
7003
diff
changeset
|
258 |
return true, pos; |
6995
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6780
diff
changeset
|
259 |
end |
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6780
diff
changeset
|
260 |
|
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
261 |
local index_fmt, index_item_size, index_magic; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
262 |
if string.packsize then |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
263 |
index_fmt = "T"; -- offset to the end of the item, length can be derived from two index items |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
264 |
index_item_size = string.packsize(index_fmt); |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
265 |
index_magic = string.pack(index_fmt, 7767639 + 1); -- Magic string: T9 for "prosody", version number |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
266 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
267 |
|
6995
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6780
diff
changeset
|
268 |
local function list_append(username, host, datastore, data) |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
269 |
if not data then return; end |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
270 |
if callback(username, host, datastore) == false then return true; end |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
271 |
-- save the datastore |
6995
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6780
diff
changeset
|
272 |
|
6996
dc0c6b8dc638
util.datamanager: Overwrite 'data' variable instead of shadownig it [luacheck]
Kim Alvefur <zash@zash.se>
parents:
6995
diff
changeset
|
273 |
data = "item(" .. serialize(data) .. ");\n"; |
8017
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8016
diff
changeset
|
274 |
local ok, msg, where = append(username, host, datastore, "list", data); |
6995
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6780
diff
changeset
|
275 |
if not ok then |
8017
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8016
diff
changeset
|
276 |
log("error", "Unable to write to %s storage ('%s' in %s) for user: %s@%s", |
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8016
diff
changeset
|
277 |
datastore, msg, where, username or "nil", host or "nil"); |
5045
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
278 |
return ok, msg; |
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
279 |
end |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
280 |
if string.packsize then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
281 |
local offset = type(msg) == "number" and msg or 0; |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
282 |
local index_entry = string.pack(index_fmt, offset + #data); |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
283 |
if offset == 0 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
284 |
index_entry = index_magic .. index_entry; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
285 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
286 |
local ok, off = append(username, host, datastore, "lidx", index_entry); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
287 |
off = off or 0; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
288 |
-- If this was the first item, then both the data and index offsets should |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
289 |
-- be zero, otherwise there's some kind of mismatch and we should drop the |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
290 |
-- index and recreate it from scratch |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
291 |
-- TODO Actually rebuild the index in this case? |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
292 |
if not ok or (off == 0 and offset ~= 0) or (off ~= 0 and offset == 0) then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
293 |
os_remove(getpath(username, host, datastore, "lidx")); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
294 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
295 |
end |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
296 |
return true; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
297 |
end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
298 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
299 |
local function list_store(username, host, datastore, data) |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
300 |
if not data then |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
301 |
data = {}; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
302 |
end |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
303 |
if callback(username, host, datastore) == false then return true; end |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
304 |
-- save the datastore |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
305 |
local d = {}; |
8095
0a1c0f1107d2
util.datamanager: Use already known index instead of measuring length each iteration
Kim Alvefur <zash@zash.se>
parents:
8017
diff
changeset
|
306 |
for i, item in ipairs(data) do |
0a1c0f1107d2
util.datamanager: Use already known index instead of measuring length each iteration
Kim Alvefur <zash@zash.se>
parents:
8017
diff
changeset
|
307 |
d[i] = "item(" .. serialize(item) .. ");\n"; |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
308 |
end |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
309 |
os_remove(getpath(username, host, datastore, "lidx")); |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
310 |
local ok, msg = atomic_store(getpath(username, host, datastore, "list", true), t_concat(d)); |
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
311 |
if not ok then |
4993
5243b74a4cbb
Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents:
4452
diff
changeset
|
312 |
log("error", "Unable to write to %s storage ('%s') for user: %s@%s", datastore, msg, username or "nil", host or "nil"); |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
313 |
return; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
314 |
end |
915
0fe5bf7ab81d
util.datamanager: Don't delete data when first entry in table is 'false'. My favourite bug so far.
Matthew Wild <mwild1@gmail.com>
parents:
896
diff
changeset
|
315 |
if next(data) == nil then -- try to delete empty datastore |
1732
f1282fad2f99
datamanager: Fixed logging errors on deletion of datastores not owned by a user@host
Waqas Hussain <waqas20@gmail.com>
parents:
1523
diff
changeset
|
316 |
log("debug", "Removing empty %s datastore for user %s@%s", datastore, username or "nil", host or "nil"); |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
317 |
os_remove(getpath(username, host, datastore, "list")); |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
318 |
end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
319 |
-- we write data even when we are deleting because lua doesn't have a |
12392
50fcd3879482
spelling: non-existing mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents:
12391
diff
changeset
|
320 |
-- platform independent way of checking for nonexisting files |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
321 |
return true; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
322 |
end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
323 |
|
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
324 |
local function build_list_index(username, host, datastore, items) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
325 |
log("debug", "Building index for (%s@%s/%s)", username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
326 |
local filename = getpath(username, host, datastore, "list"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
327 |
local fh, err, errno = io_open(filename); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
328 |
if not fh then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
329 |
return fh, err, errno; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
330 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
331 |
local prev_pos = 0; -- position before reading |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
332 |
local last_item_start = 0; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
333 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
334 |
if items and items[1] then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
335 |
local last_item = items[#items]; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
336 |
last_item_start = fh:seek("set", last_item.start + last_item.length); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
337 |
else |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
338 |
items = {}; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
339 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
340 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
341 |
for line in fh:lines() do |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
342 |
if line:sub(1, 4) == "item" then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
343 |
if prev_pos ~= 0 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
344 |
t_insert(items, { start = last_item_start; length = prev_pos - last_item_start }); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
345 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
346 |
last_item_start = prev_pos |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
347 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
348 |
-- seek position is at the start of the next line within each loop iteration |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
349 |
-- so we need to collect the "current" position at the end of the previous |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
350 |
prev_pos = fh:seek() |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
351 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
352 |
if prev_pos ~= 0 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
353 |
t_insert(items, { start = last_item_start; length = prev_pos - last_item_start }); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
354 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
355 |
return items; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
356 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
357 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
358 |
local function store_list_index(username, host, datastore, index) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
359 |
local data = { index_magic }; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
360 |
for i, v in ipairs(index) do |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
361 |
data[i + 1] = string.pack(index_fmt, v.start + v.length); |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
362 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
363 |
local filename = getpath(username, host, datastore, "lidx"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
364 |
return atomic_store(filename, t_concat(data)); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
365 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
366 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
367 |
local index_mt = { |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
368 |
__index = function(t, i) |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
369 |
if type(i) ~= "number" or i % 1 ~= 0 or i < 0 then |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
370 |
return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
371 |
end |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
372 |
if i <= 0 then |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
373 |
return 0 |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
374 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
375 |
local fh = t.file; |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
376 |
local pos = (i - 1) * index_item_size; |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
377 |
if fh:seek("set", pos) ~= pos then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
378 |
return nil |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
379 |
end |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
380 |
local data = fh:read(index_item_size * 2); |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
381 |
if not data or #data ~= index_item_size * 2 then |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
382 |
return nil |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
383 |
end |
13141
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
384 |
local start, next_pos = string.unpack(index_fmt .. index_fmt, data); |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
385 |
if pos == 0 then |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
386 |
start = 0 |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
387 |
end |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13138
diff
changeset
|
388 |
local length = next_pos - start; |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
389 |
local v = { start = start; length = length }; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
390 |
t[i] = v; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
391 |
return v; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
392 |
end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
393 |
__len = function(t) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
394 |
-- Account for both the header and the fence post error |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
395 |
return floor(t.file:seek("end") / index_item_size) - 1; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
396 |
end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
397 |
} |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
398 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
399 |
local function get_list_index(username, host, datastore) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
400 |
log("debug", "Loading index for (%s%s/%s)", username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
401 |
local index_filename = getpath(username, host, datastore, "lidx"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
402 |
local ih = io_open(index_filename); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
403 |
if ih then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
404 |
local magic = ih:read(#index_magic); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
405 |
if magic ~= index_magic then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
406 |
log("warn", "Index %q has wrong version number (got %q, expected %q)", index_filename, magic, index_magic); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
407 |
-- wrong version or something |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
408 |
ih:close(); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
409 |
ih = nil; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
410 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
411 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
412 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
413 |
if ih then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
414 |
return setmetatable({ file = ih }, index_mt); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
415 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
416 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
417 |
local index, err = build_list_index(username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
418 |
if not index then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
419 |
return index, err |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
420 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
421 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
422 |
-- TODO How to handle failure to store the index? |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
423 |
local dontcare = store_list_index(username, host, datastore, index); -- luacheck: ignore 211/dontcare |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
424 |
return index; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
425 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
426 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
427 |
local function list_load_one(fh, start, length) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
428 |
if fh:seek("set", start) ~= start then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
429 |
return nil |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
430 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
431 |
local raw_data = fh:read(length) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
432 |
if not raw_data or #raw_data ~= length then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
433 |
return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
434 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
435 |
local item; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
436 |
local data, err, errno = envload(raw_data, "@list", { |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
437 |
item = function(i) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
438 |
item = i; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
439 |
end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
440 |
}); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
441 |
if not data then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
442 |
return data, err, errno |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
443 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
444 |
local success, ret = pcall(data); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
445 |
if not success then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
446 |
return success, ret; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
447 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
448 |
return item; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
449 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
450 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
451 |
local indexed_list_mt = { |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
452 |
__index = function(t, i) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
453 |
if type(i) ~= "number" or i % 1 ~= 0 or i < 1 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
454 |
return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
455 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
456 |
local ix = t.index[i]; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
457 |
if not ix then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
458 |
return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
459 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
460 |
local item = list_load_one(t.file, ix.start, ix.length); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
461 |
return item; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
462 |
end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
463 |
__len = function(t) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
464 |
return #t.index; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
465 |
end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
466 |
} |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
467 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
468 |
local function list_load(username, host, datastore) |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
469 |
local items = {}; |
7928
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
470 |
local data, err, errno = envloadfile(getpath(username, host, datastore, "list"), {item = function(i) t_insert(items, i); end}); |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
471 |
if not data then |
7928
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
472 |
if errno == ENOENT then |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
473 |
-- No such file, ok to ignore |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
474 |
return nil; |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7737
diff
changeset
|
475 |
end |
7999
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7998
diff
changeset
|
476 |
log("error", "Failed to load %s storage ('%s') for user: %s@%s", datastore, err, username or "nil", host or "nil"); |
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7998
diff
changeset
|
477 |
return nil, "Error reading storage"; |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
478 |
end |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
479 |
|
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
480 |
local success, ret = pcall(data); |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
481 |
if not success then |
4993
5243b74a4cbb
Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents:
4452
diff
changeset
|
482 |
log("error", "Unable to load %s storage ('%s') for user: %s@%s", datastore, ret, username or "nil", host or "nil"); |
3723
72a917d910a4
util.datamanager: Return an error string when pcall fails on a loaded list file.
Waqas Hussain <waqas20@gmail.com>
parents:
3722
diff
changeset
|
483 |
return nil, "Error reading storage"; |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
484 |
end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
485 |
return items; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
486 |
end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
487 |
|
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
488 |
local function list_open(username, host, datastore) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
489 |
if not index_magic then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
490 |
log("warn", "Falling back from lazy loading to to loading full list for %s storage for user: %s@%s", datastore, username or "nil", host or "nil"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
491 |
return list_load(username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
492 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
493 |
local filename = getpath(username, host, datastore, "list"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
494 |
local file, err, errno = io_open(filename); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
495 |
if not file then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
496 |
if errno == ENOENT then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
497 |
return nil; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
498 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
499 |
return file, err, errno; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
500 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
501 |
local index, err = get_list_index(username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
502 |
if not index then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
503 |
file:close() |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
504 |
return index, err; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
505 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
506 |
return setmetatable({ file = file; index = index }, indexed_list_mt); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
507 |
end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
508 |
|
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
509 |
local type_map = { |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
510 |
keyval = "dat"; |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
511 |
list = "list"; |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
512 |
} |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
513 |
|
7677
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7676
diff
changeset
|
514 |
local function users(host, store, typ) -- luacheck: ignore 431/store |
11373
87105a9a11df
util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents:
11292
diff
changeset
|
515 |
typ = "."..(type_map[typ or "keyval"] or typ); |
10669
69acda92d5a9
util.datamanager: Fix iterating over "users" (thanks marc0s)
Kim Alvefur <zash@zash.se>
parents:
8558
diff
changeset
|
516 |
local store_dir = format("%s/%s/%s", data_path, encode(host), store_encode(store)); |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
517 |
|
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
518 |
local mode, err = lfs.attributes(store_dir, "mode"); |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
519 |
if not mode then |
6647
6cb6855f60df
util.datamanager: Fix traceback due to %s in log message
Kim Alvefur <zash@zash.se>
parents:
6578
diff
changeset
|
520 |
return function() log("debug", "%s", err or (store_dir .. " does not exist")) end |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
521 |
end |
7677
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7676
diff
changeset
|
522 |
local next, state = lfs.dir(store_dir); -- luacheck: ignore 431/next 431/state |
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7676
diff
changeset
|
523 |
return function(state) -- luacheck: ignore 431/state |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
524 |
for node in next, state do |
11373
87105a9a11df
util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents:
11292
diff
changeset
|
525 |
if node:sub(-#typ, -1) == typ then |
87105a9a11df
util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents:
11292
diff
changeset
|
526 |
return decode(node:sub(1, -#typ-1)); |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
527 |
end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
528 |
end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
529 |
end, state; |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
530 |
end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
531 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
532 |
local function stores(username, host, typ) |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
533 |
typ = type_map[typ or "keyval"]; |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
534 |
local store_dir = format("%s/%s/", data_path, encode(host)); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
535 |
|
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
536 |
local mode, err = lfs.attributes(store_dir, "mode"); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
537 |
if not mode then |
11292
5fd1f1b544a0
util.datamanager: Fix not passing variables as format string to logger (thanks semgrep)
Kim Alvefur <zash@zash.se>
parents:
10670
diff
changeset
|
538 |
return function() log("debug", "Could not iterate over stores in %s: %s", store_dir, err); end |
5032
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
539 |
end |
7677
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7676
diff
changeset
|
540 |
local next, state = lfs.dir(store_dir); -- luacheck: ignore 431/next 431/state |
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7676
diff
changeset
|
541 |
return function(state) -- luacheck: ignore 431/state |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
542 |
for node in next, state do |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
543 |
if not node:match"^%." then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
544 |
if username == true then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
545 |
if lfs.attributes(store_dir..node, "mode") == "directory" then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
546 |
return decode(node); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
547 |
end |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
548 |
elseif username then |
7676
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7435
diff
changeset
|
549 |
local store_name = decode(node); |
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7435
diff
changeset
|
550 |
if lfs.attributes(getpath(username, host, store_name, typ), "mode") then |
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7435
diff
changeset
|
551 |
return store_name; |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
552 |
end |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
553 |
elseif lfs.attributes(node, "mode") == "file" then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
554 |
local file, ext = node:match("^(.*)%.([dalist]+)$"); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
555 |
if ext == typ then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
556 |
return decode(file) |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
557 |
end |
5032
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
558 |
end |
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
559 |
end |
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
560 |
end |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
561 |
end, state; |
5032
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
562 |
end |
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
563 |
|
5103
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
564 |
local function do_remove(path) |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
565 |
local ok, err = os_remove(path); |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
566 |
if not ok and lfs.attributes(path, "mode") then |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
567 |
return ok, err; |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
568 |
end |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
569 |
return true |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
570 |
end |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
571 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
572 |
local function purge(username, host) |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
573 |
local host_dir = format("%s/%s/", data_path, encode(host)); |
6684
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6647
diff
changeset
|
574 |
local ok, iter, state, var = pcall(lfs.dir, host_dir); |
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6647
diff
changeset
|
575 |
if not ok then |
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6647
diff
changeset
|
576 |
return ok, iter; |
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6647
diff
changeset
|
577 |
end |
5095
dddbcd62183a
util.datamanager: Collect errors when deleting all stores of a user, but ignore "no such file"
Kim Alvefur <zash@zash.se>
parents:
5069
diff
changeset
|
578 |
local errs = {}; |
6684
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6647
diff
changeset
|
579 |
for file in iter, state, var do |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
580 |
if lfs.attributes(host_dir..file, "mode") == "directory" then |
7676
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7435
diff
changeset
|
581 |
local store_name = decode(file); |
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7435
diff
changeset
|
582 |
local ok, err = do_remove(getpath(username, host, store_name)); |
5103
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
583 |
if not ok then errs[#errs+1] = err; end |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
584 |
|
7676
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7435
diff
changeset
|
585 |
local ok, err = do_remove(getpath(username, host, store_name, "list")); |
5103
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
586 |
if not ok then errs[#errs+1] = err; end |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
587 |
end |
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
588 |
end |
5095
dddbcd62183a
util.datamanager: Collect errors when deleting all stores of a user, but ignore "no such file"
Kim Alvefur <zash@zash.se>
parents:
5069
diff
changeset
|
589 |
return #errs == 0, t_concat(errs, ", "); |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
590 |
end |
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
591 |
|
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
592 |
return { |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
593 |
set_data_path = set_data_path; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
594 |
add_callback = add_callback; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
595 |
remove_callback = remove_callback; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
596 |
getpath = getpath; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
597 |
load = load; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
598 |
store = store; |
7005
9ab0d5e69c41
util.datamanager: Add append to public api
Kim Alvefur <zash@zash.se>
parents:
7004
diff
changeset
|
599 |
append_raw = append; |
7932
4017ea6d2599
util.datamanager: Expose atomic_store as store_raw
Kim Alvefur <zash@zash.se>
parents:
7931
diff
changeset
|
600 |
store_raw = atomic_store; |
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
601 |
list_append = list_append; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
602 |
list_store = list_store; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
603 |
list_load = list_load; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
604 |
users = users; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
605 |
stores = stores; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
606 |
purge = purge; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
607 |
path_decode = decode; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
608 |
path_encode = encode; |
13138
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
609 |
|
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
610 |
build_list_index = build_list_index; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12979
diff
changeset
|
611 |
list_open = list_open; |
6780
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6684
diff
changeset
|
612 |
}; |